From 9308f0d3b102ea8f639a368c6a26b02cddf6fe89 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 09:49:06 +0530 Subject: [PATCH 001/111] build(dev): stop gating on the prose check The communication rules are guidelines, not gates. `make docs-lint` no longer holds PROSE_PATHS to zero issues, and the `prose` count leaves .quality-baseline.json and dev/quality_baseline.py. The new `make docs-prose [FILES=...]` target reports the issues and never fails. Co-Authored-By: Claude Opus 5.5 --- .quality-baseline.json | 8 +------ Makefile | 35 +++++++++++++++--------------- dev/quality_baseline.py | 25 ++++++--------------- dev/tests/test_quality_baseline.py | 11 +++------- 4 files changed, 28 insertions(+), 51 deletions(-) diff --git a/.quality-baseline.json b/.quality-baseline.json index 9ecff81b..b848058c 100644 --- a/.quality-baseline.json +++ b/.quality-baseline.json @@ -1,5 +1,5 @@ { - "_comment": "Quality baseline: one full-tree violation count per tool. A baseline only goes down; raising a count needs the owner. Compared by dev/quality_baseline.py from make format-check, typecheck and docs-lint. The coverage floor lives in .coveragerc (fail_under). The prose entry counts dev/prose_check.py issues (docs/COMMUNICATION_RULES.md rules). is the version in dev/tools/pyproject.toml or dev/tools/package.json.", + "_comment": "Quality baseline: one full-tree violation count per tool. A baseline only goes down; raising a count needs the owner. Compared by dev/quality_baseline.py from make format-check, typecheck and docs-lint. The coverage floor lives in .coveragerc (fail_under). is the version in dev/tools/pyproject.toml or dev/tools/package.json.", "measured": "2026-11-08", "counts": { "ruff": { @@ -19,12 +19,6 @@ "what": "markdownlint-cli2 issues over every tracked Markdown file (the DOCS_LINT_PATHS ratchet is a subset and clean)", "command": "git ls-files -z '*.md' | xargs -0 npx markdownlint-cli2@", "verb": "make docs-lint" - }, - "prose": { - "count": 1624, - "what": "dev/prose_check.py issues (long sentences, vague words, explanation pages without a diagram) over the tracked Markdown in PROSE_SCOPE (the PROSE_PATHS ratchet is a subset and clean)", - "command": "$(PROSE_SCOPE) | xargs -0 python3 dev/prose_check.py --quiet", - "verb": "make docs-lint" } } } diff --git a/Makefile b/Makefile index fe81115e..17759bef 100644 --- a/Makefile +++ b/Makefile @@ -60,18 +60,11 @@ DOCS_LINT_PATHS := 'implement/SKILL.md' 'implement/state.md' 'implement/playbook 'docs/explanation/user-overview/03-trust-and-safety.md' 'docs/explanation/contributor-deep-dive/04-stack-profiles.md' 'docs/explanation/contributor-deep-dive/05-evaluation.md' 'docs/how-to/add-a-stack-profile.md' 'docs/reference/harness-notes.md' \ 'audit/engine/docs/how-to/read-the-report.md' -# The Markdown that the prose check (dev/prose_check.py, the plain-language rules of docs/COMMUNICATION_RULES.md) -# holds to zero issues. A ratchet like DOCS_LINT_PATHS: a page joins once it is clean, and the list only grows. -# Outside it, the full-tree count may not rise above the `prose` entry of .quality-baseline.json. -PROSE_PATHS := docs/COMMUNICATION_RULES.md docs/STYLE_GUIDE.md docs/architecture/adrs/0004-plain-language-rules-and-a-prose-check.md \ - docs/explanation/contributor-deep-dive/04-stack-profiles.md \ - docs/explanation/contributor-deep-dive/05-evaluation.md \ - docs/how-to/add-a-stack-profile.md \ - docs/reference/harness-notes.md \ - audit/engine/docs/how-to/read-the-report.md +# The prose check (dev/prose_check.py, the plain-language rules of docs/COMMUNICATION_RULES.md) is advisory: it +# reports warnings and notices and never fails a target. `make docs-prose` runs it; CI annotates changed Markdown. PROSE_CHECK := $(PYTHON) dev/prose_check.py -# The tracked Markdown the prose count covers: the markdownlint scope, without fixtures, test data and the worked -# example's copied reports. +# The tracked Markdown `make docs-prose` reads when FILES is not set: the markdownlint scope, without fixtures, +# test data and the worked example's copied reports. PROSE_SCOPE := git ls-files -z -- '*.md' ':(exclude,glob)**/fixtures/**' ':(exclude,glob)**/testdata/**' \ ':(exclude,glob)**/server-worked-example/digital-assistant-server[-.]*.md' @@ -84,12 +77,12 @@ TYPED_SOURCES := $(shell find audit/tools implement/tools dev -name '*.py' -not .DEFAULT_GOAL := help .NOTPARALLEL: -.PHONY: help setup format format-check format\:check lint typecheck test coverage verify-rules crosscheck recipe-lint check security update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-mermaid changelog-fix workflow-lint commit-lint +.PHONY: help setup format format-check format\:check lint typecheck test coverage verify-rules crosscheck recipe-lint check security update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint help: @echo "verbs: format format:check (format-check) lint typecheck test check security" @echo "extra: setup (checks prerequisites, installs nothing) coverage self-audit update-audit-baseline verify-rules crosscheck recipe-lint workflow-lint commit-lint update-secrets-baseline validate-skills" - @echo "docs: docs-lint docs-lint-all docs-mermaid FILES=... changelog-fix" + @echo "docs: docs-lint docs-lint-all docs-prose [FILES=...] docs-mermaid FILES=... changelog-fix" # There is nothing to install into the project: the runtime code is standard-library Python and the dev tools run # through uvx and npx at pinned versions. This target checks the prerequisites without a prompt, exits 1 naming the @@ -196,20 +189,26 @@ validate-skills: $(SKILLS_REF) validate dev/dist/skills/agentic-readiness-audit $(SKILLS_REF) validate dev/dist/skills/agentic-readiness-implement -# Lints the Markdown in DOCS_LINT_PATHS and checks the prose in PROSE_PATHS; fails on any violation. Then compares -# both full-tree counts with .quality-baseline.json. Needs Node (npx); the first run fetches the pinned CLI. CI runs -# it in the `docs` job when a Markdown file changed. +# Lints the Markdown in DOCS_LINT_PATHS; fails on any violation. Then compares the full-tree count with +# .quality-baseline.json. The prose check is not part of it: see docs-prose. Needs Node (npx); the first run +# fetches the pinned CLI. CI runs it in the `docs` job when a Markdown file changed. docs-lint: $(MARKDOWNLINT) $(DOCS_LINT_PATHS) - $(PROSE_CHECK) $(PROSE_PATHS) $(QUALITY_BASELINE) markdownlint -- sh -c "git ls-files -z '*.md' | xargs -0 $(MARKDOWNLINT)" - $(QUALITY_BASELINE) prose -- sh -c "$(PROSE_SCOPE) | xargs -0 $(PROSE_CHECK) --quiet" # Reports violations in every tracked Markdown file (fixtures excluded by the config). Never fails: the debt # outside DOCS_LINT_PATHS is recorded, not gated. docs-lint-all: -git ls-files -z '*.md' | xargs -0 $(MARKDOWNLINT) + +# Reports the prose check's warnings and notices (docs/COMMUNICATION_RULES.md) in FILES, or in all tracked Markdown +# when FILES is not set. Advisory: it never fails. +docs-prose: +ifdef FILES + -$(PROSE_CHECK) $(FILES) +else -$(PROSE_SCOPE) | xargs -0 $(PROSE_CHECK) +endif # Rewrites CHANGELOG.md in place with the markdownlint fixes (release-please writes `*` bullets and double blank # lines). release-please.yml runs it on the release pull request; it fails when a violation is left that --fix diff --git a/dev/quality_baseline.py b/dev/quality_baseline.py index 1db4f7b6..f0ac8f25 100644 --- a/dev/quality_baseline.py +++ b/dev/quality_baseline.py @@ -5,8 +5,8 @@ quality_baseline.py check TOOL [--baseline FILE] -- COMMAND... -``TOOL`` is ``ruff``, ``mypy``, ``markdownlint`` or ``prose`` (``dev/prose_check.py``). The command after ``--`` is the tool's own read-only full-tree run; -its output is parsed into one number and compared with the entry for ``TOOL`` in ``.quality-baseline.json``: +``TOOL`` is ``ruff``, ``mypy`` or ``markdownlint``. The command after ``--`` is the tool's own read-only full-tree +run; its output is parsed into one number and compared with the entry for ``TOOL`` in ``.quality-baseline.json``: * count above the baseline: exit 1, naming the tool and both numbers; * count equal: exit 0; @@ -28,7 +28,8 @@ from pathlib import Path DEFAULT_BASELINE = Path(".quality-baseline.json") -TOOLS = ("ruff", "mypy", "markdownlint", "prose") +# The prose check (dev/prose_check.py) is advisory, so it has no count here (ADR 0004). +TOOLS = ("ruff", "mypy", "markdownlint") _MARKDOWNLINT_SUMMARY = re.compile(r"^Summary:\s+(\d+)\s+issues?\b", re.MULTILINE) @@ -56,18 +57,7 @@ def count_markdownlint(output: str) -> int: raise ValueError("no 'Summary: N issues' line in markdownlint output") -_PROSE_SUMMARY = re.compile(r"^Summary: (\d+) issues? in \d+ files?$", re.MULTILINE) - - -def count_prose(output: str) -> int: - """The ``Summary: N issues in M files`` line of ``dev/prose_check.py``.""" - match = _PROSE_SUMMARY.search(output) - if not match: - raise ValueError("no 'Summary: N issues in M files' line in prose_check output") - return int(match.group(1)) - - -PARSERS = {"ruff": count_ruff, "mypy": count_mypy, "markdownlint": count_markdownlint, "prose": count_prose} +PARSERS = {"ruff": count_ruff, "mypy": count_mypy, "markdownlint": count_markdownlint} def load_baseline(path: Path, tool: str) -> int: @@ -117,9 +107,8 @@ def main(argv: list[str] | None = None) -> int: print(f"quality baseline: cannot run {command[0]}: {exc}", file=sys.stderr) return 2 try: - # ruff's JSON and prose_check's summary are on stdout alone; mypy and markdownlint-cli2 split their report - # across both streams. - text = run.stdout if args.tool in ("ruff", "prose") else run.stdout + "\n" + run.stderr + # ruff's JSON is on stdout alone; mypy and markdownlint-cli2 split their report across both streams. + text = run.stdout if args.tool == "ruff" else run.stdout + "\n" + run.stderr current = PARSERS[args.tool](text) except ValueError as exc: print(f"quality baseline: cannot read the {args.tool} output: {exc}", file=sys.stderr) diff --git a/dev/tests/test_quality_baseline.py b/dev/tests/test_quality_baseline.py index cb5d20da..92343eed 100644 --- a/dev/tests/test_quality_baseline.py +++ b/dev/tests/test_quality_baseline.py @@ -52,14 +52,9 @@ def test_markdownlint_without_summary_is_an_error(self) -> None: with self.assertRaises(ValueError): quality_baseline.count_markdownlint("npm ERR! network\n") - def test_prose_summary(self) -> None: - self.assertEqual(quality_baseline.count_prose("a.md:3: PL001 x\nSummary: 1711 issues in 167 files\n"), 1711) - self.assertEqual(quality_baseline.count_prose("Summary: 1 issue in 1 file\n"), 1) - self.assertEqual(quality_baseline.count_prose("Summary: 0 issues in 2 files\n"), 0) - - def test_prose_without_summary_is_an_error(self) -> None: - with self.assertRaises(ValueError): - quality_baseline.count_prose("Traceback (most recent call last):\n") + def test_prose_is_advisory_and_has_no_count(self) -> None: + self.assertNotIn("prose", quality_baseline.TOOLS) + self.assertNotIn("prose", quality_baseline.PARSERS) class Compare(unittest.TestCase): From d36a954e19b4c2ee4e8c93aeb7712e5c8cb4e319 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 09:50:10 +0530 Subject: [PATCH 002/111] feat(dev): give prose-check issues a severity and always exit 0 Each issue is now a warning or a notice. Vague words, missing diagrams, long steps and sentences over 40 words are warnings; other sentences over 25 words are notices. The check exits 0 whatever it finds. `--format github` prints annotations, and the CI `docs` job uses it on the changed Markdown. `--min-severity warning` hides the notices. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 9 ++++- Makefile | 7 ++-- dev/prose_check.py | 76 ++++++++++++++++++++++++++--------- dev/tests/test_prose_check.py | 55 ++++++++++++++++++++----- 4 files changed, 112 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81377a22..5957c221 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ # CI runs the same verbs a human or agent runs locally (see the Makefile): `make check` and `make security`. # Python 3.11 is the supported minimum and 3.14 the newest release (Q8); Windows is supported through WSL only. # The `docs` job runs `make docs-lint` and the Mermaid check when a Markdown file changed; it needs Node, so it -# stays out of `make check` and the matrix. +# stays out of `make check` and the matrix. It also annotates the prose of the changed files, which never fails. # It runs on every pull request and on every push to the two protected branches, `dev` and `main` # (docs/reference/branch-protection.md). name: CI @@ -115,6 +115,13 @@ jobs: echo "Changed Markdown:" cat "$list" if [[ -s "$list" ]]; then echo "any=true" >> "$GITHUB_OUTPUT"; else echo "any=false" >> "$GITHUB_OUTPUT"; fi + # The prose check (docs/COMMUNICATION_RULES.md) is advisory: it annotates the changed Markdown with warnings and + # notices and always exits 0. Fixtures and test data are skipped, as in `make docs-prose`. + - if: steps.changed.outputs.any == 'true' + name: Annotate prose in changed Markdown + run: | + { grep -v -E '(^|/)(fixtures|testdata)/' "$RUNNER_TEMP/changed-md.txt" || true; } \ + | xargs -r -d '\n' python3 dev/prose_check.py --format github - if: steps.changed.outputs.any == 'true' uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: diff --git a/Makefile b/Makefile index 17759bef..c90f0be0 100644 --- a/Makefile +++ b/Makefile @@ -202,12 +202,13 @@ docs-lint-all: -git ls-files -z '*.md' | xargs -0 $(MARKDOWNLINT) # Reports the prose check's warnings and notices (docs/COMMUNICATION_RULES.md) in FILES, or in all tracked Markdown -# when FILES is not set. Advisory: it never fails. +# when FILES is not set. Advisory: the check exits 0 whatever it finds. Add PROSE_ARGS=--min-severity=warning to +# hide the notices. docs-prose: ifdef FILES - -$(PROSE_CHECK) $(FILES) + $(PROSE_CHECK) $(PROSE_ARGS) $(FILES) else - -$(PROSE_SCOPE) | xargs -0 $(PROSE_CHECK) + $(PROSE_SCOPE) | xargs -0 $(PROSE_CHECK) $(PROSE_ARGS) endif # Rewrites CHANGELOG.md in place with the markdownlint fixes (release-please writes `*` bullets and double blank diff --git a/dev/prose_check.py b/dev/prose_check.py index da01d866..4f47523a 100644 --- a/dev/prose_check.py +++ b/dev/prose_check.py @@ -3,18 +3,23 @@ """Check Markdown prose against the plain-language rules in ``docs/COMMUNICATION_RULES.md``. - prose_check.py FILE... + prose_check.py [--format text|github] [--min-severity notice|warning] [--quiet] FILE... -Three rules, each reported as ``path:line: CODE message``: +The check is advisory: it never fails. Three rules, each reported as ``path:line: CODE severity: message``: -* ``PL001`` a sentence is longer than 25 words, or longer than 20 words in a numbered step; -* ``PL002`` a vague or padded word from :data:`VAGUE` (``simply``, ``in order to`` and the others); -* ``PL003`` an explanation or tutorial page (see :func:`needs_diagram`) has no Mermaid block. +* ``PL001`` a sentence is longer than 25 words (a notice), longer than 40 words (a warning), or longer than 20 words + in a numbered step (a warning); +* ``PL002`` a vague or padded word from :data:`VAGUE` (``simply``, ``in order to`` and the others), a warning; +* ``PL003`` an explanation or tutorial page (see :func:`needs_diagram`) has no Mermaid block, a warning. + +A *warning* is worth fixing in the change that touches the text. A *notice* is a hint: fix it when it helps the +reader. ``--format github`` prints each issue as a GitHub Actions annotation (``::warning`` or ``::notice``), which +shows on the pull request without failing the job. ``--min-severity warning`` hides the notices. Only prose is read: paragraphs, list items and blockquotes. Fenced code, tables, headings, HTML comments and inline code spans are skipped. A region between ```` and ```` is skipped, for -a deliberate bad example. The last line is ``Summary: N issues``; the exit code is 1 when N is above zero, so the -Makefile can hold a path list to zero and ``quality_baseline.py`` can ratchet the full-tree count. +a deliberate bad example. The last line is ``Summary: N issues in M files (W warnings, K notices)``. The exit code +is 0 whatever the count; it is 2 only when a file cannot be read. Dev-only and standard library only. The sentence splitter is a heuristic: it splits at ``.``, ``!`` or ``?`` before a space and a capital letter, a digit, a code span or an opening bracket. @@ -30,6 +35,12 @@ MAX_WORDS = 25 MAX_WORDS_STEP = 20 +# A sentence this long is hard to follow even for a patient reader, so it is a warning, not a notice. +MAX_WORDS_WARNING = 40 + +WARNING = "warning" +NOTICE = "notice" +SEVERITIES = (NOTICE, WARNING) VAGUE = ( "simply", @@ -76,10 +87,17 @@ class Issue: path: str line: int code: str + severity: str message: str def __str__(self) -> str: - return f"{self.path}:{self.line}: {self.code} {self.message}" + return f"{self.path}:{self.line}: {self.code} {self.severity}: {self.message}" + + def github(self) -> str: + """The issue as a GitHub Actions workflow command (an annotation on the file and line).""" + message = self.message.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + path = self.path.replace("%", "%25").replace(",", "%2C").replace(":", "%3A") + return f"::{self.severity} file={path},line={self.line},title={self.code}::{message}" @dataclass @@ -189,11 +207,14 @@ def check_text(path: str, content: str) -> list[Issue]: words = sum(1 for token in sentence.split() if _WORD.search(token)) if words > limit: kind = "step" if block.step else "sentence" - issues.append(Issue(path, block.line, "PL001", f"{kind} has {words} words (limit {limit})")) + severity = WARNING if block.step or words > MAX_WORDS_WARNING else NOTICE + message = f"{kind} has {words} words (limit {limit})" + issues.append(Issue(path, block.line, "PL001", severity, message)) for match in _VAGUE_RE.finditer(prose): - issues.append(Issue(path, block.line, "PL002", f"vague word '{match.group(1)}'")) + issues.append(Issue(path, block.line, "PL002", WARNING, f"vague word '{match.group(1)}'")) if needs_diagram(path) and not has_mermaid: - issues.append(Issue(path, 1, "PL003", "explanation or tutorial page without a Mermaid diagram")) + message = "explanation or tutorial page without a Mermaid diagram" + issues.append(Issue(path, 1, "PL003", WARNING, message)) return issues @@ -201,22 +222,37 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else None) parser.add_argument("files", nargs="*", type=Path, help="Markdown files to check") parser.add_argument("--quiet", action="store_true", help="print the summary line only") + parser.add_argument("--format", choices=("text", "github"), default="text", help="issue format (default: text)") + parser.add_argument( + "--min-severity", choices=SEVERITIES, default=NOTICE, help="hide issues below this severity (default: notice)" + ) args = parser.parse_args(argv) - total = 0 + shown = SEVERITIES[SEVERITIES.index(args.min_severity) :] + counts = dict.fromkeys(SEVERITIES, 0) for file in args.files: try: content = file.read_text(encoding="utf-8") except OSError as exc: print(f"prose_check: cannot read {file}: {exc}", file=sys.stderr) return 2 - issues = check_text(file.as_posix(), content) - total += len(issues) - if not args.quiet: - for issue in issues: - print(issue) - files = len(args.files) - print(f"Summary: {total} issue{'' if total == 1 else 's'} in {files} file{'' if files == 1 else 's'}") - return 1 if total else 0 + issues = [i for i in check_text(file.as_posix(), content) if i.severity in shown] + for issue in issues: + counts[issue.severity] += 1 + if not args.quiet: + print(issue.github() if args.format == "github" else issue) + print(summary(counts, len(args.files))) + return 0 + + +def _plural(n: int, word: str) -> str: + return f"{n} {word}{'' if n == 1 else 's'}" + + +def summary(counts: dict[str, int], files: int) -> str: + """``Summary: N issues in M files (W warnings, K notices)``.""" + total = sum(counts.values()) + detail = ", ".join(_plural(counts[s], s) for s in (WARNING, NOTICE)) + return f"Summary: {_plural(total, 'issue')} in {_plural(files, 'file')} ({detail})" if __name__ == "__main__": diff --git a/dev/tests/test_prose_check.py b/dev/tests/test_prose_check.py index 111f2735..8519c621 100644 --- a/dev/tests/test_prose_check.py +++ b/dev/tests/test_prose_check.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # Copyright (C) 2026 K. C. Ramakrishna -"""Unit tests for ``prose_check.py``: the three rules, what is skipped, and the summary and exit code.""" +"""Unit tests for ``prose_check.py``: the three rules, their severities, what is skipped, the output and exit code.""" from __future__ import annotations @@ -54,6 +54,18 @@ def test_a_code_span_counts_as_one_word(self) -> None: def test_a_link_counts_as_its_text(self) -> None: self.assertEqual(_codes("Read [the guide](" + "a/" * 40 + "b.md) first.\n"), []) + def test_a_long_sentence_is_a_notice(self) -> None: + issues = pc.check_text("a.md", _words(40) + "\n") + self.assertEqual([(i.code, i.severity) for i in issues], [("PL001", pc.NOTICE)]) + + def test_a_very_long_sentence_is_a_warning(self) -> None: + issues = pc.check_text("a.md", _words(41) + "\n") + self.assertEqual([(i.code, i.severity) for i in issues], [("PL001", pc.WARNING)]) + + def test_a_long_step_is_a_warning(self) -> None: + issues = pc.check_text("a.md", "1. " + _words(21) + "\n") + self.assertEqual([i.severity for i in issues], [pc.WARNING]) + def test_the_line_number_is_the_block_start(self) -> None: issues = pc.check_text("a.md", "# Title\n\nShort.\n\n" + _words(30) + "\n") self.assertEqual(issues[0].line, 5) @@ -81,6 +93,7 @@ def test_each_vague_word_is_reported(self) -> None: issues = pc.check_text("a.md", "Simply run it in order to win, etc.\n") self.assertEqual([i.code for i in issues], ["PL002"] * 3) self.assertIn("'Simply'", issues[0].message) + self.assertEqual({i.severity for i in issues}, {pc.WARNING}) def test_a_word_inside_another_is_not_reported(self) -> None: self.assertEqual(_codes("Adjust the justification.\n"), []) @@ -91,7 +104,8 @@ def test_a_vague_word_in_a_code_span_is_not_reported(self) -> None: class Diagrams(unittest.TestCase): def test_an_explanation_page_without_mermaid_fails(self) -> None: - self.assertEqual(_codes("Text.\n", "docs/explanation/user-overview/01-x.md"), ["PL003"]) + issues = pc.check_text("docs/explanation/user-overview/01-x.md", "Text.\n") + self.assertEqual([(i.code, i.severity) for i in issues], [("PL003", pc.WARNING)]) def test_an_explanation_page_with_mermaid_passes(self) -> None: text = "Text.\n\n```mermaid\nflowchart LR\n A --> B\n```\n" @@ -105,7 +119,7 @@ def test_which_pages_need_a_diagram(self) -> None: class Main(unittest.TestCase): - def _run(self, *texts: str, quiet: bool = False) -> tuple[int, str]: + def _run(self, *texts: str, quiet: bool = False, extra: tuple[str, ...] = ()) -> tuple[int, str]: with tempfile.TemporaryDirectory() as tmp: files = [] for n, text in enumerate(texts): @@ -114,23 +128,42 @@ def _run(self, *texts: str, quiet: bool = False) -> tuple[int, str]: files.append(str(path)) out = io.StringIO() with contextlib.redirect_stdout(out): - code = pc.main([*files, *(["--quiet"] if quiet else [])]) + code = pc.main([*files, *(["--quiet"] if quiet else []), *extra]) return code, out.getvalue() def test_clean_files_exit_0_with_a_summary(self) -> None: code, out = self._run("Short.\n", "Also short.\n") self.assertEqual(code, 0) - self.assertEqual(out.strip(), "Summary: 0 issues in 2 files") + self.assertEqual(out.strip(), "Summary: 0 issues in 2 files (0 warnings, 0 notices)") - def test_issues_exit_1_and_are_listed(self) -> None: - code, out = self._run(_words(30) + "\n") - self.assertEqual(code, 1) - self.assertIn("PL001", out) - self.assertTrue(out.rstrip().endswith("Summary: 1 issue in 1 file")) + def test_issues_are_listed_and_still_exit_0(self) -> None: + code, out = self._run(_words(30) + " Simply.\n") + self.assertEqual(code, 0) + self.assertIn("PL001 notice: sentence has 30 words (limit 25)", out) + self.assertIn("PL002 warning: vague word 'Simply'", out) + self.assertTrue(out.rstrip().endswith("Summary: 2 issues in 1 file (1 warning, 1 notice)")) def test_quiet_prints_only_the_summary(self) -> None: code, out = self._run(_words(30) + "\n", quiet=True) - self.assertEqual((code, out.strip()), (1, "Summary: 1 issue in 1 file")) + self.assertEqual((code, out.strip()), (0, "Summary: 1 issue in 1 file (0 warnings, 1 notice)")) + + def test_min_severity_warning_hides_notices(self) -> None: + code, out = self._run(_words(30) + " Simply.\n", extra=("--min-severity", "warning")) + self.assertEqual(code, 0) + self.assertNotIn("PL001", out) + self.assertIn("PL002", out) + self.assertTrue(out.rstrip().endswith("Summary: 1 issue in 1 file (1 warning, 0 notices)")) + + def test_github_format_prints_annotations(self) -> None: + code, out = self._run(_words(30) + " Simply.\n", extra=("--format", "github")) + self.assertEqual(code, 0) + lines = out.splitlines() + self.assertRegex(lines[0], r"^::notice file=.*0\.md,line=1,title=PL001::sentence has 30 words \(limit 25\)$") + self.assertRegex(lines[1], r"^::warning file=.*0\.md,line=1,title=PL002::vague word 'Simply'$") + + def test_github_format_escapes_the_path(self) -> None: + issue = pc.Issue("a,b:c.md", 3, "PL002", pc.WARNING, "x") + self.assertEqual(issue.github(), "::warning file=a%2Cb%3Ac.md,line=3,title=PL002::x") def test_a_missing_file_exits_2(self) -> None: err = io.StringIO() From b736b0ddf2e4a03aab07ecdac74d77281c7f683d Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 09:51:04 +0530 Subject: [PATCH 003/111] docs: describe the prose check as advisory COMMUNICATION_RULES.md, STYLE_GUIDE.md, dev-tooling.md, CONTRIBUTING.md and the contribute how-to now say that the prose check reports warnings and notices and never blocks. ADR 0004 records the reversal of the blocking check and the tools reviewed on 2026-11-09. Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 1 + docs/COMMUNICATION_RULES.md | 3 ++ docs/STYLE_GUIDE.md | 12 ++++--- ...-plain-language-rules-and-a-prose-check.md | 35 ++++++++++++------- docs/how-to/contribute-a-change.md | 3 ++ docs/reference/dev-tooling.md | 4 +-- 6 files changed, 38 insertions(+), 20 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d11427dc..fe650af2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,6 +48,7 @@ Run these before you propose a change. CI runs the same commands. | `make commit-lint` | Checks that each commit subject on your branch follows Conventional Commits (`RANGE=origin/dev..HEAD` by default). CI runs the same check on every pull request | | `make security` | A secrets scan against `.secrets.baseline` | | `make docs-lint` | Markdown lint over the ratchet path list in the `Makefile`. Run it when Markdown changed | +| `make docs-prose FILES=` | The plain-language check from [`docs/COMMUNICATION_RULES.md`](docs/COMMUNICATION_RULES.md). Advisory: it prints warnings and notices and never fails. Fix the warnings on the lines you changed | | `make docs-mermaid FILES=` | Renders every Mermaid block in the files. Run it when you added or changed one | | `make coverage` | The engine's and the tools' suites under coverage.py; fails below the floor in `.coveragerc`. The floor only goes up | | `make self-audit` | The engine on this repository against `.audit-baseline.json`; exit 2 on a regression. See [Self-Audit](docs/reference/self-audit.md) | diff --git a/docs/COMMUNICATION_RULES.md b/docs/COMMUNICATION_RULES.md index 2d78930a..97cf2f12 100644 --- a/docs/COMMUNICATION_RULES.md +++ b/docs/COMMUNICATION_RULES.md @@ -19,6 +19,9 @@ These rules come from the writing rules of ASD-STE100 (Simplified Technical Engl that help us. We do not use its fixed word list: technical names are fine. `dev/prose_check.py` checks the rules marked *checked*. A person or an agent applies the others. +The rules are guidelines. The check reports warnings and notices, and it never blocks a change +([ADR 0004](architecture/adrs/0004-plain-language-rules-and-a-prose-check.md)). +Fix a warning in the change that touches the text. Treat a notice as a hint. 1. **Bottom line first.** Put the answer, the result or the decision you need in the first two lines. Put the detail after it. diff --git a/docs/STYLE_GUIDE.md b/docs/STYLE_GUIDE.md index 734af592..fe84e183 100644 --- a/docs/STYLE_GUIDE.md +++ b/docs/STYLE_GUIDE.md @@ -53,7 +53,8 @@ The rules are in [`COMMUNICATION_RULES.md`](COMMUNICATION_RULES.md) sections 2 a - **Define a term on first use,** or link it to [`docs/glossary.md`](glossary.md). - **Keep a page to one topic.** When a page grows past about 300 lines, split it. -`make docs-lint` runs the prose check. Section 6 says which pages must be clean. +`make docs-prose FILES=` runs the prose check. It reports warnings and notices and never fails. +CI shows them as annotations on the pull request. ## 4. Format @@ -95,13 +96,14 @@ Paths are relative to `docs/`. | Tool | What it checks | Where it must be clean | |---|---|---| | `markdownlint-cli2` | Markdown format | `DOCS_LINT_PATHS` in the `Makefile` | -| `dev/prose_check.py` | Sentence length, vague words, diagrams on explanation pages | `PROSE_PATHS` in the `Makefile` | +| `dev/prose_check.py` | Sentence length, vague words, diagrams on explanation pages | Nowhere: advisory warnings and notices | | `dev/mermaid_check.py` | Each Mermaid block renders | The files you pass to `make docs-mermaid` | | `dev/recipe_lint.py` | Links resolve inside the repository | Recipes, `docs/` and the root pages | -Both path lists are ratchets: a page joins once it is clean, and the lists only grow. Outside the lists, the -total count of issues may not go up (`.quality-baseline.json`). When you edit a page outside the lists, fix its -issues and add it to the list. +`DOCS_LINT_PATHS` is a ratchet: a page joins once it is clean, and the list only grows. Outside the list, the +total count of Markdown issues may not go up (`.quality-baseline.json`). When you edit a page outside the list, fix +its issues and add it to the list. The prose check has no list and no count. Fix its warnings on the lines you +edit. ## 7. Agents writing docs diff --git a/docs/architecture/adrs/0004-plain-language-rules-and-a-prose-check.md b/docs/architecture/adrs/0004-plain-language-rules-and-a-prose-check.md index 13376959..a35faf9e 100644 --- a/docs/architecture/adrs/0004-plain-language-rules-and-a-prose-check.md +++ b/docs/architecture/adrs/0004-plain-language-rules-and-a-prose-check.md @@ -2,7 +2,7 @@ * **Status:** Accepted * **Date:** 2026-11-08 -* **Last reviewed:** 2026-11-08 +* **Last reviewed:** 2026-11-09 * **Deciders:** [@nistapp] --- @@ -26,6 +26,7 @@ Three questions needed an answer: 1. Where do the rules live, so that every harness reads them? 2. How much of STE100 do we adopt? 3. Does a tool check the rules, and which tool? +4. Does the check block a change, or only advise? --- @@ -41,18 +42,22 @@ Three questions needed an answer: [`AGENTS.md`](../../../AGENTS.md) carries a six-line summary, because every harness loads it. The style guide ([`docs/STYLE_GUIDE.md`](../../STYLE_GUIDE.md)) adds only the rules for doc pages and links to the rest. -**Check.** [`dev/prose_check.py`](../../../dev/prose_check.py) is a standard-library script with three rules: +**Check.** [`dev/prose_check.py`](../../../dev/prose_check.py) is a standard-library script with three rules. Each +issue has a severity: -* `PL001`: a sentence of more than 25 words, or more than 20 in a numbered step; -* `PL002`: a vague or padded word, such as `simply` or `in order to`; -* `PL003`: an explanation page or a tutorial without a Mermaid diagram. +| Rule | Reports | Severity | +|---|---|---| +| `PL001` | A sentence of more than 25 words | Notice | +| `PL001` | A sentence of more than 40 words, or more than 20 in a numbered step | Warning | +| `PL002` | A vague or padded word, such as `simply` or `in order to` | Warning | +| `PL003` | An explanation page or a tutorial without a Mermaid diagram | Warning | -`make docs-lint` runs it two ways: +**Advisory, never blocking.** The rules are guidelines. The check exits 0 whatever it finds. No gate, path list or +baseline count holds it. A writer fixes a warning in the change that touches the text and treats a notice as a hint. -* on the `PROSE_PATHS` list in the `Makefile`, which must have zero issues; -* on all tracked Markdown, whose count may not rise above the `prose` entry of `.quality-baseline.json`. - -Both are ratchets. A page joins `PROSE_PATHS` once it is clean. +* `make docs-prose [FILES=...]` prints the issues. `PROSE_ARGS=--min-severity=warning` hides the notices. +* The CI `docs` job runs it with `--format github` on the changed Markdown. The issues show as annotations on the + pull request, and the job does not fail on them. **Scope.** The rules apply to this repository: agent messages and docs. How the audit and implement recipes talk to their users, and whether target repositories get a matching requirement, is listed for v0.3.4. @@ -67,7 +72,10 @@ their users, and whether target repositories get a matching requirement, is list | Rules only in `AGENTS.md` | Every line costs context in every session, and `AGENTS.md` is already long. | | Rules only in `COMMUNICATION_RULES.md` | Agents often skip a linked file. The short summary in `AGENTS.md` keeps the core in view. | | Vale with a custom STE style (the v0.3.3 A4 pilot) | A Go binary: not on PyPI or npm, so it cannot be pinned the ADR 0003 way. Our three rules need about 200 lines of Python. | +| textlint, with `textlint-rule-terminology` (reviewed 2026-11-09) | Pinnable through `dev/tools/package.json`, and its rules can be warnings. Not needed yet: revisit when passive voice or glossary terms need a check. | +| proselint, write-good, alex, LanguageTool (reviewed 2026-11-09) | proselint is opinionated and hard to configure. write-good and alex are narrow. LanguageTool needs Java. | | Rules without a check | Rules without a check drift. Chat cannot be checked, but docs can. | +| A blocking check: a zero-issue path list (`PROSE_PATHS`) and a ratchet count in `.quality-baseline.json` (shipped 2026-11-08, reversed 2026-11-09) | Too strict for guidelines. On 2026-11-09, 1,574 of 1,624 issues were `PL001`, and a long sentence is often the clearest one. Blocking on it slowed work without a matching gain. | --- @@ -76,15 +84,16 @@ their users, and whether target repositories get a matching requirement, is list ### Positive * Every harness that reads `AGENTS.md` gets the core rules, and the full rules are one link away. -* New and edited pages in `PROSE_PATHS` stay readable; the full-tree count can only go down. +* Writers see the issues on their own lines, in the terminal or on the pull request, without being blocked. +* Severities point attention at the issues that matter most: vague words, long steps and missing diagrams. * No new pinned tool, and no network. ### Negative / Trade-offs * The sentence splitter is a heuristic. It can miss a split, for example after a code span that ends with a full stop. Rewording the sentence fixes it. -* The full-tree count starts high (about 1,680 issues on 2026-11-08). Most pages join the list only when someone edits - them. +* Nothing stops the count from rising. The full tree had 1,624 issues on 2026-11-09. Review, not a gate, keeps new + text readable. * The check cannot see passive voice, one meaning per word or the bottom-line-first rule. People and agents apply those. diff --git a/docs/how-to/contribute-a-change.md b/docs/how-to/contribute-a-change.md index 4e731da1..9100ba3e 100644 --- a/docs/how-to/contribute-a-change.md +++ b/docs/how-to/contribute-a-change.md @@ -156,8 +156,11 @@ If you changed Markdown, also run: ```bash make docs-lint +make docs-prose FILES= ``` +`docs-prose` never fails. Fix the warnings it reports on the lines you changed. + If you added or changed a Mermaid diagram, also run this for each file: ```bash diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 0f9f5f22..9f4f0068 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -22,7 +22,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | check-jsonschema | `.github/workflows/*.yml` and `.github/dependabot.yml` that do not match their SchemaStore schemas | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | | detect-secrets | Key-shaped strings not in `.secrets.baseline` | `make security` | `security` | `dev/tools/pyproject.toml` | | markdownlint-cli2 | Markdown rule violations (`.markdownlint-cli2.jsonc`) | `make docs-lint`; `make changelog-fix` rewrites `CHANGELOG.md` with the fixes | `docs`; `format-changelog` in `release-please.yml` | `dev/tools/package.json` | -| `dev/prose_check.py` | Long sentences, vague words, explanation pages without a diagram ([`COMMUNICATION_RULES.md`](../COMMUNICATION_RULES.md)) | `make docs-lint` | `docs` | This repository | +| `dev/prose_check.py` | Long sentences, vague words, explanation pages without a diagram ([`COMMUNICATION_RULES.md`](../COMMUNICATION_RULES.md)). Advisory: warnings and notices, never a failure | `make docs-prose [FILES=...]` | `docs` (annotations on changed Markdown) | This repository | | mermaid-cli | A Mermaid block that does not render | `make docs-mermaid FILES=` | `docs` | `dev/tools/package.json` | | skills-ref | A skill bundle that breaks the Agent Skills specification | `make validate-skills` | Release only | `dev/tools/pyproject.toml` | | `dev/commit_lint.py` | A commit subject that is not Conventional Commits | `make commit-lint` | `commits` | This repository | @@ -42,7 +42,7 @@ fails its gate. When a count goes down, lower the baseline in the same change. | Ratchet | File | Checked by | | --- | --- | --- | -| ruff, mypy, Markdown lint and prose counts | `.quality-baseline.json` | `make format:check`, `make typecheck`, `make docs-lint` | +| ruff, mypy and Markdown lint counts | `.quality-baseline.json` | `make format:check`, `make typecheck`, `make docs-lint` | | Known key-shaped strings | `.secrets.baseline` | `make security` | | Coverage floor | `.coveragerc` (`fail_under`) | `make coverage` | | Readiness findings | `.audit-baseline.json` | `make self-audit` | From 110f2f7789d9f0402d362a98511399de5383f63f Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 10:20:16 +0530 Subject: [PATCH 004/111] docs(stacks): recommend each stack's own hook and commit tools Add the owner's stack-adaptive rule to the stack-profile README and a hook manager and commit-message rule to each profile's Tooling defaults: Husky with lint-staged for Node, pre-commit for Python, lefthook for mixed or Java repositories. Roadmap-Step: C0 Co-Authored-By: Claude Opus 5.5 --- audit/stacks/README.md | 11 +++++++++-- audit/stacks/java.md | 4 +++- audit/stacks/python.md | 4 +++- audit/stacks/typescript-node.md | 4 +++- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/audit/stacks/README.md b/audit/stacks/README.md index 0cc2be4d..066acb9a 100644 --- a/audit/stacks/README.md +++ b/audit/stacks/README.md @@ -16,6 +16,12 @@ runner/dependency scanner are the stack's defaults, and so on. Checks and playbo toolchain; a profile is where that knowledge lives, so the same check file reads correctly for Node, Java or Python. +**Stack-adaptive rule (owner, 2026-11-09).** A check accepts any recognised tool that does the job for the +stack. A playbook keeps the tool a repository already uses. When there is none, it recommends the tool that +matches the stack, from the profile's Tooling defaults, never this framework's own choice by default. For +hooks: Husky with lint-staged for Node, pre-commit for Python, lefthook for a mixed repository. A quiet +release history alone does not rule a tool out; an archived repository or an open security advisory does. + ## 2. Anatomy Every profile follows this fixed shape, in this order: @@ -28,8 +34,9 @@ Detection signals files and dependencies that identify the stack and its fr Command-surface map how each verb is usually realised, and the runner (npm scripts / Makefile / Taskfile / Gradle / Maven) Tooling defaults recommended formatter, linter, type checker, test runner, coverage tool, - dependency scanner, with "alternatives you may find" so an existing choice is - respected, never replaced without being asked + dependency scanner, hook manager and commit-message rule, with "alternatives + you may find" so an existing choice is respected, never replaced without being + asked Execution determinism version pinning and lockfile expectations for this stack CI idiom how to run the verbs in the common CI providers for this stack Baseline / ratchet how to adopt "no new violations" for this stack diff --git a/audit/stacks/java.md b/audit/stacks/java.md index d83a0f45..3fd3ccd6 100644 --- a/audit/stacks/java.md +++ b/audit/stacks/java.md @@ -41,7 +41,9 @@ SpotBugs is a static-analysis alternative. Type checker: none separate from the Gradle's built-in `test` task (`useJUnitPlatform()` in the build file). Dependency scanner: the OWASP Dependency-Check Gradle plugin (`org.owasp.dependencycheck`), registering `dependencyCheckAnalyze` — a task name that does not match `security` literally, so a repository applying the plugin still reads -`CMD-01` `FAIL` for that verb unless a `security` task delegates to it (see Pitfalls). +`CMD-01` `FAIL` for that verb unless a `security` task delegates to it (see Pitfalls). Hook manager: none +is standard; a committed `.githooks/` with `core.hooksPath` set by a build task, or lefthook (no JVM plugin +needed). Commit rule: a `commit-msg` script or lefthook stage. ## Execution determinism diff --git a/audit/stacks/python.md b/audit/stacks/python.md index 31741cb2..77249bcf 100644 --- a/audit/stacks/python.md +++ b/audit/stacks/python.md @@ -31,7 +31,9 @@ Formatter: Ruff's own formatter, or Black. Linter: Ruff (replacing Flake8/Pylint Flake8/Pylint on an older one; config in `pyproject.toml`'s `[tool.ruff]`/`[tool.black]` or standalone `ruff.toml`/`.flake8`/`.pylintrc`. Type checker: mypy or Pyright, `[tool.mypy]`/`[tool.pyright]` or `mypy.ini`'s `strict = true`. Test runner: pytest. Dependency scanner: `pip-audit` (works against any of -pip/uv/poetry's resolved environment) or `safety check`. +pip/uv/poetry's resolved environment) or `safety check`. Hook manager: pre-commit +(`.pre-commit-config.yaml`); lefthook in a mixed repository. Commit rule: a pre-commit `commit-msg` hook +(conventional-pre-commit, commitizen or gitlint). ## Execution determinism diff --git a/audit/stacks/typescript-node.md b/audit/stacks/typescript-node.md index d1e9bdbf..201b2eaa 100644 --- a/audit/stacks/typescript-node.md +++ b/audit/stacks/typescript-node.md @@ -41,7 +41,9 @@ plugin-dependent rules keeps a minimal ESLint config alongside it rather than sw checker: `tsc` reading `tsconfig.json`'s `compilerOptions.strict`. Test runner: Vitest or Jest, both seen across the corpus (§Verified) — a third option, Node's own built-in test runner, is untested here. Dependency scanner: the package manager's own `audit` command; `audit-ci` or Snyk are alternatives that -pin a fixed severity threshold in config rather than a CLI flag. +pin a fixed severity threshold in config rather than a CLI flag. Hook manager: Husky + lint-staged +(`prepare: husky`), or simple-git-hooks; lefthook when there is no root `package.json`. Commit rule: +commitlint in `.husky/commit-msg`. ## Execution determinism From 304420e69970da6ec0a7fb6a3ad592616a6cdcad Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 10:20:16 +0530 Subject: [PATCH 005/111] docs(implement): choose the hook manager by the target's stack The hooks playbook recommended lefthook everywhere and dropped Husky for a quiet release history. It now keeps an existing manager, recommends the stack's own tool when there is none (Husky with lint-staged, pre-commit, lefthook), restores Husky with a Node section, and matches the commit rule to the stack. Tool currency rechecked on 2026-11-09. Roadmap-Step: C0 Co-Authored-By: Claude Opus 5.5 --- .../playbooks/add-hooks-and-commit-rules.md | 54 ++++++++++++------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/implement/playbooks/add-hooks-and-commit-rules.md b/implement/playbooks/add-hooks-and-commit-rules.md index 1fc82989..fc92b5c3 100644 --- a/implement/playbooks/add-hooks-and-commit-rules.md +++ b/implement/playbooks/add-hooks-and-commit-rules.md @@ -31,24 +31,29 @@ elsewhere (the pull-request title) by the owner's decision. ## Inputs to ask -1. **Hook manager.** Detect first and use what exists. If nothing, offer this set. Currency checked 2026-10-31 - (GitHub releases API, npm registry); re-check when applied. - - | Tool | Version, date | Pros | Cons | - |---|---|---|---| - | lefthook (recommended) | v2.2.2, 2026-10-13 (about 9 million npm downloads a week) | One Go binary, no Node or Python needed, polyglot, one committed `lefthook.yml`, parallel hooks | A binary to install per developer; fewer ready-made hooks | - | pre-commit | v4.7.0, 2026-10-13 | Large hook catalogue, mature | Needs Python on every machine; pins hook repositories by revision | - | husky + lint-staged | husky 9.1.7, **2024-11-18** | Node standard | **Dropped: no release in over 12 months.** Needs a root Node manifest; runs only after `npm install` | - | A Gradle git-hooks plugin | not verified | Java only | Not checked for maintenance, no help for a Node repository; do not offer unless a maintained one is verified at apply time | - - Installing the binary is a developer step the owner approves, not a repository dependency. +1. **Hook manager.** Detect first and keep what exists. Never replace a working manager. If nothing exists, + recommend the manager that matches the stack (the stack profile's Tooling defaults, `audit/stacks/`), not this + framework's own choice. Offer the others as alternatives. Currency checked 2026-11-09 (npm registry and + downloads API, PyPI, GitHub API); re-check when applied. + + | Stack | Recommend | Version, date | Pros | Cons | + |---|---|---|---|---| + | Node or TypeScript with a root `package.json` | Husky + lint-staged | husky 9.1.7, 2024-11-18; lint-staged 17.6.0, 2026-09-26 (about 50 and 41 million npm downloads a week) | The Node default; installed by the package manager, pinned by the lockfile; `prepare: husky` wires it | Runs only after `npm install`; needs a root manifest | + | Python | pre-commit | v4.7.0, 2026-10-12 | Large hook catalogue, mature, the Python default | Needs Python on every machine; pins hook repositories by revision | + | Mixed stacks, Node with no root manifest, Java, or no profile | lefthook | v2.2.2, 2026-10-13 (about 9.5 million npm downloads a week) | One Go binary, no Node or Python needed, one committed `lefthook.yml`, parallel hooks | A binary to install per developer; fewer ready-made hooks | + + Alternatives you may find, and keep: simple-git-hooks (Node, 2.14.0, 2026-08-28), or a committed + `.githooks/` folder. A build task sets `core.hooksPath` for it, often in Gradle or Maven repositories. A quiet + release history alone is no reason to reject a tool. Husky had no release since 2024-11-18. Its repository is + not archived, and it is the most used manager (checked 2026-11-09). Reject a tool only when it is archived or + has an open security advisory. Installing a binary is a developer step the owner approves, not a repository dependency. 2. **Hook content.** Default below; the owner may remove, never add a slow hook silently. 3. **Commit rule** (skip when already skipped in the state file): | Option | What | Engine reads it as | |---|---|---| | A. Document the PR-title rule | The repository squash-merges and a CI check rejects non-conventional PR titles; the PR title is what lands on the default branch. Write this in the contributing doc and record the finding `skipped` with that reason | TOOL-06 stays `FAIL` (the engine does not read CI; "TOOL-06's own blind spot"); owner-accepted | - | B. commitlint in a `commit-msg` stage | Config file plus a hook stage | `PASS`. Needs Node and `@commitlint/cli` (21.2.4); in a repository with no root manifest the version cannot be pinned by a lockfile | + | B. A commit-message rule in a `commit-msg` stage, matched to the stack | Node: commitlint (`@commitlint/cli` 21.2.4) in `.husky/commit-msg`. Python: a pre-commit `commit-msg` hook (conventional-pre-commit, commitizen or gitlint). Others: the manager's `commit-msg` stage | `PASS`. commitlint needs Node; with no root manifest its version cannot be pinned by a lockfile, so prefer the Python or lefthook route there | | C. Both | A and B | `PASS`; carries B's costs | Recommend A where squash merges and a title check exist (the history that matters is the squash commit); @@ -64,8 +69,9 @@ elsewhere (the pull-request title) by the owner's decision. - the formatter's check on **staged files only** (the T1 changed-files form, never the full tree), - the secret scanner on staged changes, at the version already pinned in CI (one pin, shared). No tests, no full lint, no type check, no network access in a hook: those stay in the gate and in CI. -3. **Install documentation.** Add the one-time install command (`lefthook install` or the manager's equivalent) - and the binary install line to the instruction file and the contributing doc. Say that CI is the real gate. +3. **Install documentation.** Add the one-time install command to the instruction file and the contributing doc. + That is `npm install` for Husky, `pre-commit install` or `lefthook install`. For a binary manager, add the + binary install line too. Say that CI is the real gate. Never advise `--no-verify`; when a hook fails, fix the cause. 4. **Commit rule** per the owner's option. For A: the contributing doc names the PR-title workflow and the type list it enforces. For B or C: add the config and the `commit-msg` stage, with the same type list as the @@ -79,9 +85,16 @@ No Node is needed; the binary manager runs the Gradle formatter task limited to changed-files verb. If the formatter only supports a whole-project run, hook it as report-only or leave the formatter out and say why (never run a full-tree format in a hook). +### Node with a root manifest + +Use Husky + lint-staged. Add `husky` and `lint-staged` as dev dependencies (the lockfile pins them), set +`"prepare": "husky"` in `package.json`, and write `.husky/pre-commit` as `npx lint-staged`. Configure lint-staged +to run the formatter's check on the staged files only. The engine reads `.husky/` as wired only when a script +names `husky` (`TOOL-05`). For the commit rule, `.husky/commit-msg` runs `npx --no -- commitlint --edit "$1"`. + ### Node (npm, per-app layout) -With no root manifest a Node-based manager has no home, which is another reason the binary manager is the default. +With no root manifest a Node-based manager has no home, so lefthook is the recommendation here. The hook runs each app's changed-files verb from that app's directory (`layout.command_runner`). With lefthook (IR07.3): @@ -110,7 +123,7 @@ With lefthook (IR07.3): | Check | Command or file | Expected | |---|---|---| -| TOOL-05 | `test -f lefthook.yml` (or the manager's config) and `git config core.hooksPath` or the install step in the docs | The engine's wiring file exists; the install step is documented | +| TOOL-05 | `test -f ` (`.husky/pre-commit`, `.pre-commit-config.yaml`, `lefthook.yml`) and `git config core.hooksPath` or the install step in the docs | The engine's wiring file exists; the install step is documented | | TOOL-05 (behaviour) | In a scratch worktree, stage a file with a deliberate format violation and a fake secret, run the hook | The commit is refused with a clear message; a clean staged change passes in a few seconds | | TOOL-06 | Option A: the contributing doc names the title check and the state records `skipped` with the owner's reason. Option B or C: the config and the `commit-msg` stage exist; a bad message is refused | As stated | @@ -118,7 +131,8 @@ The engine reads TOOL-06 as a file or stage check; option A leaves it `FAIL` by ## Rollback -`git revert` the commits (config, docs, rule each revert alone). Developers run `lefthook uninstall`. +`git revert` the commits (config, docs, rule each revert alone). Developers run the manager's uninstall +(`lefthook uninstall`, `pre-commit uninstall`; for Husky, `git config --unset core.hooksPath`). ## Do-not-do @@ -126,7 +140,7 @@ The engine reads TOOL-06 as a file or stage check; option A leaves it `FAIL` by - Do not run tests, a full lint, a full-tree format or any network call in a hook. - Do not advise `--no-verify` or add a bypass variable. - Do not re-propose a `skipped` commit-convention finding. -- Do not pick a manager by what looks familiar: detect, then offer the table. +- Do not pick a manager by what looks familiar or by what this framework uses: detect, then recommend by stack. - Do not duplicate the secret-scanner version: reuse the CI pin. ## State updates @@ -135,10 +149,10 @@ Beyond the shared lines in [Stage 3](../SKILL.md#stage-3-apply) (`in_progress`, [Stage 4](../SKILL.md#stage-4-close-the-theme) (the ending): ```bash -python3 implement/tools/progress.py pref set hooks.manager \ +python3 implement/tools/progress.py pref set hooks.manager \ --target --phase --reason "" --by owner python3 implement/tools/progress.py skip F-nnn --target --phase --by owner \ --reason "" python3 implement/tools/progress.py set-status F-nnn done --target --phase \ - --commit --verification-command "test -f lefthook.yml" --verification-result "present; install documented" + --commit --verification-command "test -f " --verification-result "present; install documented" ``` From 683ddb3b40af71788e7d09f02632eb055f4e5646 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 10:42:51 +0530 Subject: [PATCH 006/111] feat(dev): register codebase-memory-mcp in one project .mcp.json The repository failed its own IDX-01 BLOCKER: no project MCP config. .mcp.json now names the bare codebase-memory-mcp command, so each person's install runs and no version is written in the repository. Claude Code and VS Code read the file; other harnesses copy the entry. Approval stays with each person: .claude/settings.json pre-approves no project server, and dev/tests/test_mcp_config.py checks both. The self-audit baseline drops the IDX-01 finding (score 0.8934), with the absolute paths neutralised again, and docs/reference/self-audit.md ends owner exception D21. Roadmap-Step: B1 Co-Authored-By: Claude Opus 5.5 --- .audit-baseline.json | 64 +- .mcp.json | 7 + .secrets.baseline | 24 +- AGENTS.md | 3 +- .../examples/self-audit/2026-11-09.json | 1580 +++++++++++++++++ dev/tests/test_mcp_config.py | 58 + docs/reference/dev-tooling.md | 24 + docs/reference/self-audit.md | 31 +- 8 files changed, 1731 insertions(+), 60 deletions(-) create mode 100644 .mcp.json create mode 100644 audit/schema/examples/self-audit/2026-11-09.json create mode 100644 dev/tests/test_mcp_config.py diff --git a/.audit-baseline.json b/.audit-baseline.json index cc9eac64..b2c6f613 100644 --- a/.audit-baseline.json +++ b/.audit-baseline.json @@ -9,10 +9,10 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "no project-level MCP configuration found (user-level configuration not read: --no-user-config)", + "summary": "codebase-memory-mcp registered in: .mcp.json (project)", "tier": "A", "title": "codebase-memory-mcp registered for this repository", - "verdict": "FAIL" + "verdict": "PASS" }, { "data": {}, @@ -22,7 +22,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "index store could not be located; searched: user-level locations (not read: --no-user-config), /home/kc/Projects/Nistapp-agentic-frameworks/codebase-agentic-readiness-framework/.codebase-memory/graph.db.zst", + "summary": "index store could not be located; searched: user-level locations (not read: --no-user-config), /.codebase-memory/graph.db.zst", "tier": "A", "title": "An index exists for this checkout", "verdict": "UNKNOWN" @@ -161,7 +161,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "20 boundary rules reference real paths", + "summary": "21 boundary rules reference real paths", "tier": "B", "title": "Architectural boundaries declared", "verdict": "PASS" @@ -262,7 +262,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "39 markdown file(s) under docs/", + "summary": "43 markdown file(s) under docs/", "tier": "A", "title": "docs/ exists", "verdict": "PASS" @@ -301,7 +301,7 @@ "scored": true, "severity": "COSMETIC", "status": "implemented", - "summary": "3 ADR(s) indexed in docs/architecture/adrs/README.md", + "summary": "5 ADR(s) indexed in docs/architecture/adrs/README.md", "tier": "A", "title": "ADR directory and index", "verdict": "PASS" @@ -625,6 +625,8 @@ "docs/architecture/adrs/README.md", ".env", "RELEASE_PROCESS.md", + "CHANGELOG.md", + ".release-please-manifest.json", ".github/rulesets/", "docs/reference/branch-protection.md", "audit/tools/", @@ -801,7 +803,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "test verb on Makefile (cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests); 82 test file(s)", + "summary": "test verb on Makefile (cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests); 85 test file(s)", "tier": "A", "title": "Test verb exists, suite non-empty", "verdict": "PASS" @@ -834,7 +836,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "skip/xfail census across 82 test file(s): python 1; total 1", + "summary": "skip/xfail census across 85 test file(s): python 1; total 1", "tier": "B", "title": "Skip / xfail census", "verdict": "PASS" @@ -919,10 +921,10 @@ }, { "data": { - "denominator": 1138, + "denominator": 1152, "kind": "ratio", "method": "python: module-level def/class not starting with '_'", - "numerator": 453, + "numerator": 456, "unit": "public symbols" }, "id": "NAV-05", @@ -931,7 +933,7 @@ "scored": false, "severity": "COSMETIC", "status": "implemented", - "summary": "453/1138 public symbols documented (40%); method: python: module-level def/class not starting with '_'", + "summary": "456/1152 public symbols documented (40%); method: python: module-level def/class not starting with '_'", "tier": "B", "title": "Public-symbol documentation coverage", "verdict": "PARTIAL" @@ -1274,22 +1276,6 @@ } ], "findings": [ - { - "check": "IDX-01", - "evidence": [ - { - "line": null, - "note": null, - "path": "" - } - ], - "id": "IDX-01", - "phase": 1, - "remediation": "Register the server in a project config (for example `opencode.json`) or a user config (`~/.config/opencode/opencode.json`).", - "severity": "BLOCKER", - "statement": "An agent cannot discover the codebase through an MCP index because no MCP configuration names codebase-memory-mcp for this repository.", - "verdict": "FAIL" - }, { "check": "SEC-01", "evidence": [ @@ -1451,9 +1437,9 @@ } ], "inventory": { - "files_inspected": 325, + "files_inspected": 336, "files_set_aside": 460, - "files_skipped": 2250, + "files_skipped": 3, "set_aside": [ { "files": 3, @@ -1531,22 +1517,22 @@ } ], "provenance": { - "framework_revision": "bdcbe3d516552fb17e1606cd8c77ae86635f688d", - "git_branch": "chore/review-for-release", + "framework_revision": "320fad8969a38d09331ece11634cc36e124db725", + "git_branch": "feat/v0.3.3-track-b-mcp", "git_dirty": true, - "git_sha": "bdcbe3d516552fb17e1606cd8c77ae86635f688d", + "git_sha": "320fad8969a38d09331ece11634cc36e124db725", "phase": 1, "probes_enabled": false, "ruleset_hash": "970e6792f98eeeb5", "ruleset_revision": "2026-11-06.1", - "scanned_at": "2026-11-08T07:19:00+00:00", - "target": "/home/kc/Projects/Nistapp-agentic-frameworks/codebase-agentic-readiness-framework", + "scanned_at": "2026-11-09T05:04:17+00:00", + "target": ".", "tool": "python-agentic-audit", "user_config": { "paths_read": [], "read_allowed": false }, - "version": "0.1.0" + "version": "0.3.2" }, "schema_version": "2", "stack": { @@ -1568,12 +1554,12 @@ "test_frameworks": [] }, "summary": { - "blockers": 2, + "blockers": 1, "checks": { "attest": 0, - "fail": 1, + "fail": 0, "partial": 6, - "pass": 58, + "pass": 59, "unknown": 4 }, "checks_applicable": 69, @@ -1582,7 +1568,7 @@ "cosmetic": 0, "coverage_note": "55 of 55 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", "degraders": 3, - "phase1_score": 0.8689 + "phase1_score": 0.8934 }, "unattested": [ "index-in-use", diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 00000000..382cd77a --- /dev/null +++ b/.mcp.json @@ -0,0 +1,7 @@ +{ + "mcpServers": { + "codebase-memory-mcp": { + "command": "codebase-memory-mcp" + } + } +} diff --git a/.secrets.baseline b/.secrets.baseline index dae87024..13fd8c9c 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -131,16 +131,16 @@ { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", - "hashed_secret": "63215b3c1959cb7cfcaaad73ca8b8ebbbcf1c805", + "hashed_secret": "8614d586c51ed65bb882b415a3c861a2c92b157f", "is_verified": false, - "line_number": 1534 + "line_number": 1520 }, { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", "hashed_secret": "228ea6c3984c8d1226e0ffc33455f16dce652d01", "is_verified": false, - "line_number": 1540 + "line_number": 1526 } ], "audit/catalogue.json": [ @@ -447,6 +447,22 @@ "line_number": 1518 } ], + "audit/schema/examples/self-audit/2026-11-09.json": [ + { + "type": "Hex High Entropy String", + "filename": "audit/schema/examples/self-audit/2026-11-09.json", + "hashed_secret": "8614d586c51ed65bb882b415a3c861a2c92b157f", + "is_verified": false, + "line_number": 1520 + }, + { + "type": "Hex High Entropy String", + "filename": "audit/schema/examples/self-audit/2026-11-09.json", + "hashed_secret": "228ea6c3984c8d1226e0ffc33455f16dce652d01", + "is_verified": false, + "line_number": 1526 + } + ], "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json": [ { "type": "Hex High Entropy String", @@ -1449,5 +1465,5 @@ } ] }, - "generated_at": "2026-11-08T11:27:15Z" + "generated_at": "2026-11-09T05:09:30Z" } diff --git a/AGENTS.md b/AGENTS.md index ba543d64..0a6e7a14 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -151,4 +151,5 @@ A change is done when all of this holds: - The engine's design, checks and ADRs: `audit/engine/docs/architecture/README.md`. - The tool contract and the tool catalogue: `audit/tools/README.md`, `audit/tools/CATALOGUE.md`. - Tool ecosystem and index tooling: `docs/framework/tool-ecosystem.md`. If `codebase-memory-mcp` is available, query it - before reading whole files. + before reading whole files. `.mcp.json` registers it; Claude Code and VS Code read that file. Another harness + copies the entry into its own config. No gate needs an MCP server (`docs/reference/dev-tooling.md`). diff --git a/audit/schema/examples/self-audit/2026-11-09.json b/audit/schema/examples/self-audit/2026-11-09.json new file mode 100644 index 00000000..b2c6f613 --- /dev/null +++ b/audit/schema/examples/self-audit/2026-11-09.json @@ -0,0 +1,1580 @@ +{ + "candidate_components": [], + "checks": [ + { + "data": {}, + "id": "IDX-01", + "pack": "IDX", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "codebase-memory-mcp registered in: .mcp.json (project)", + "tier": "A", + "title": "codebase-memory-mcp registered for this repository", + "verdict": "PASS" + }, + { + "data": {}, + "id": "IDX-02", + "pack": "IDX", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "index store could not be located; searched: user-level locations (not read: --no-user-config), /.codebase-memory/graph.db.zst", + "tier": "A", + "title": "An index exists for this checkout", + "verdict": "UNKNOWN" + }, + { + "data": {}, + "id": "IDX-03", + "pack": "IDX", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "index freshness is a Tier-C probe; re-run with --run-gates to compare the indexed revision against HEAD", + "tier": "C", + "title": "Index freshness", + "verdict": "UNKNOWN" + }, + { + "data": {}, + "id": "AGT-01", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "3 nested instruction file(s)", + "tier": "A", + "title": "Instruction file present, canonical, and non-trivial", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-02", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "single-root repository: no component-level instruction files required", + "tier": "A", + "title": "Component coverage by instruction files", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-03", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "docs/, the style guide and the scratch contract are named", + "tier": "B", + "title": "Documentation contract named", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-04", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "definition of done ties a doc trigger to the same change set", + "tier": "B", + "title": "Definition of done stated", + "verdict": "PASS" + }, + { + "data": { + "kind": "verb_surface", + "missing": [], + "resolved_count": 7, + "verbs": [ + { + "command": "$(RUFF) format audit implement dev", + "resolved": true, + "runner": "Makefile", + "verb": "format" + }, + { + "command": "make format-check", + "resolved": true, + "runner": "Makefile", + "verb": "format:check" + }, + { + "command": "$(RUFF) check audit implement dev", + "resolved": true, + "runner": "Makefile", + "verb": "lint" + }, + { + "command": "$(MYPY) --config-file mypy.ini $(TYPED_SOURCES) && $(QUALITY_BASELINE) mypy -- $(MYPY) --config-file mypy.ini $(TYPED_SOURCES)", + "resolved": true, + "runner": "Makefile", + "verb": "typecheck" + }, + { + "command": "cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests", + "resolved": true, + "runner": "Makefile", + "verb": "test" + }, + { + "command": "make format-check && make typecheck && make test && make verify-rules && make crosscheck && make recipe-lint && make workflow-lint", + "resolved": true, + "runner": "Makefile", + "verb": "check" + }, + { + "command": "git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline", + "resolved": true, + "runner": "Makefile", + "verb": "security" + } + ] + }, + "id": "AGT-05", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "all 7 verbs named in AGENTS.md and resolved on a runner", + "tier": "B", + "title": "Seven verbs named", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-06", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "21 boundary rules reference real paths", + "tier": "B", + "title": "Architectural boundaries declared", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-07", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "prohibitions cover tests, generated files, the default branch and secrets", + "tier": "B", + "title": "Do-not-do / deny rules present", + "verdict": "PASS" + }, + { + "data": { + "kind": "harness_matrix", + "reached": 20, + "reached_tools": [ + "Augment Code", + "Bolt (StackBlitz)", + "Charm Crush", + "Cline", + "Cursor", + "Devin (Cognition)", + "Firebase Studio (ex-IDX)", + "GitHub Copilot / VS Code", + "Goose (Block)", + "JetBrains Junie", + "Jules (Google)", + "Kiro (AWS)", + "OpenAI Codex CLI", + "Roo Code", + "Sourcegraph Amp", + "Trae", + "Warp", + "Windsurf / Devin Desktop", + "Zed", + "opencode" + ], + "total": 28, + "unreached_tools": [ + "Aider", + "Amazon Q Developer", + "Anthropic Claude Code", + "Continue", + "Google Gemini CLI", + "Qwen Code", + "Replit Agent", + "Void" + ] + }, + "id": "AGT-08", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "20/28 harnesses reached; AGENTS.md:46 states the supported harnesses (Anthropic Claude Code, Cursor, Google Gemini CLI, OpenAI Codex CLI, opencode)", + "tier": "B", + "title": "No competing instruction files, and the reach matrix is stated", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-09", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "the branch model and the release boundary are stated", + "tier": "B", + "title": "Branch and release boundary declared", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-10", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "", + "tier": "B", + "title": "Instruction indirection resolved", + "verdict": "PASS" + }, + { + "data": {}, + "id": "DOC-01", + "pack": "DOC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "43 markdown file(s) under docs/", + "tier": "A", + "title": "docs/ exists", + "verdict": "PASS" + }, + { + "data": {}, + "id": "DOC-02", + "pack": "DOC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "docs/STYLE_GUIDE.md present and named in AGENTS.md", + "tier": "A", + "title": "docs/STYLE_GUIDE.md exists", + "verdict": "PASS" + }, + { + "data": {}, + "id": "DOC-03", + "pack": "DOC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "artefacts/ present, ignored by artefacts/.gitignore, and untracked", + "tier": "B", + "title": "artefacts/ exists and is ignored", + "verdict": "PASS" + }, + { + "data": {}, + "id": "DOC-04", + "pack": "DOC", + "phase": 1, + "scored": true, + "severity": "COSMETIC", + "status": "implemented", + "summary": "5 ADR(s) indexed in docs/architecture/adrs/README.md", + "tier": "A", + "title": "ADR directory and index", + "verdict": "PASS" + }, + { + "data": { + "kind": "verb_surface", + "missing": [], + "resolved_count": 7, + "verbs": [ + { + "command": "$(RUFF) format audit implement dev", + "resolved": true, + "runner": "Makefile", + "verb": "format" + }, + { + "command": "make format-check", + "resolved": true, + "runner": "Makefile", + "verb": "format:check" + }, + { + "command": "$(RUFF) check audit implement dev", + "resolved": true, + "runner": "Makefile", + "verb": "lint" + }, + { + "command": "$(MYPY) --config-file mypy.ini $(TYPED_SOURCES) && $(QUALITY_BASELINE) mypy -- $(MYPY) --config-file mypy.ini $(TYPED_SOURCES)", + "resolved": true, + "runner": "Makefile", + "verb": "typecheck" + }, + { + "command": "cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests", + "resolved": true, + "runner": "Makefile", + "verb": "test" + }, + { + "command": "make format-check && make typecheck && make test && make verify-rules && make crosscheck && make recipe-lint && make workflow-lint", + "resolved": true, + "runner": "Makefile", + "verb": "check" + }, + { + "command": "git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline", + "resolved": true, + "runner": "Makefile", + "verb": "security" + } + ] + }, + "id": "CMD-01", + "pack": "CMD", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "format -> Makefile; format:check -> Makefile; lint -> Makefile; typecheck -> Makefile; test -> Makefile; check -> Makefile; security -> Makefile", + "tier": "B", + "title": "Seven verbs resolvable on one runner", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CMD-02", + "pack": "CMD", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "check (Makefile) composes: format-check, typecheck, test", + "tier": "B", + "title": "check composes the read-only verbs", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CMD-03", + "pack": "CMD", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "one runner defines the gate", + "tier": "B", + "title": "No divergent second definition", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CMD-04", + "pack": "CMD", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "no read-only verb invokes a write verb", + "tier": "B", + "title": "Write verbs separated from read-only verbs", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TOOL-01", + "pack": "TOOL", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "formatter ruff.toml + Makefile format verb", + "tier": "B", + "title": "Formatter configured", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TOOL-02", + "pack": "TOOL", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "linter ruff.toml + Makefile lint verb", + "tier": "B", + "title": "Linter configured", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TOOL-03", + "pack": "TOOL", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "strict config: mypy.ini", + "tier": "A", + "title": "Type checker configured", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TOOL-04", + "pack": "TOOL", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "security verb on Makefile runs no dependency audit (git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline)", + "tier": "A", + "title": "Dependency / security scan verb", + "verdict": "PARTIAL" + }, + { + "data": {}, + "id": "TOOL-05", + "pack": "TOOL", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "wired: lefthook.yml", + "tier": "B", + "title": "Local hook chain active", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TOOL-06", + "pack": "TOOL", + "phase": 1, + "scored": true, + "severity": "COSMETIC", + "status": "implemented", + "summary": "commit rule: lefthook.yml:commit-msg", + "tier": "A", + "title": "Commit convention enforced", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CI-01", + "pack": "CI", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "PR-triggered: .github/workflows/ci.yml (github); .github/workflows/pr-source-branch.yml (github)", + "tier": "B", + "title": "Pipeline present, triggered on PRs", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CI-02", + "pack": "CI", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "CI verbs: check, security; local read-only verbs not exercised: format:check, lint, test, typecheck", + "tier": "B", + "title": "Local\u2194CI parity", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CI-03", + "pack": "CI", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "no CI-only steps", + "tier": "B", + "title": "CI-only steps flagged", + "verdict": "PASS" + }, + { + "data": {}, + "id": "BASE-01", + "pack": "BASE", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "baseline/suppression mechanism present: .audit-baseline.json, .quality-baseline.json, .secrets.baseline", + "tier": "A", + "title": "Baseline artifact exists", + "verdict": "PASS" + }, + { + "data": {}, + "id": "BASE-02", + "pack": "BASE", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "counting surface present but no committed count/budget: .audit-baseline.json, .quality-baseline.json, .secrets.baseline, Makefile:lint, Makefile:security, Makefile:typecheck; method: static configuration scan; the target's own tools are never run", + "tier": "B", + "title": "Current violation counts measured", + "verdict": "PARTIAL" + }, + { + "data": {}, + "id": "BASE-03", + "pack": "BASE", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "ratchet present: baseline compared in CI (.github/workflows/ci.yml)", + "tier": "B", + "title": "No-new-violations mechanism", + "verdict": "PASS" + }, + { + "data": {}, + "id": "BASE-04", + "pack": "BASE", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "coverage threshold configured: .coveragerc", + "tier": "A", + "title": "Coverage floor configured", + "verdict": "PASS" + }, + { + "data": { + "kind": "path_list", + "paths": [ + "Makefile", + "audit/engine/", + "AGENTS.md", + ".python-version", + "dev/tools/pyproject.toml", + "dev/tools/package.json", + "lefthook.yml", + "dev", + ".github/", + "dev/commit_lint.py", + ".coveragerc", + ".quality-baseline.json", + ".secrets.baseline", + ".skip", + "README.md", + "docs/", + "greenfield/", + "docs/framework/", + "docs/STYLE_GUIDE.md", + "docs/README.md", + "spec/", + "REQ-...", + "spec/README.md", + "audit/", + "SKILL.md", + "checks/", + "prompts/", + "catalogue.json", + "stacks/", + "schema/", + "tools/", + "audit/AGENTS.md", + "audit/engine/AGENTS.md", + "implement/", + "playbooks/", + "state.md", + "progress.py", + "implement/docs/architecture/adrs/", + "implement/AGENTS.md", + "dev/", + "dev/dist/skills/", + "dev/eval/", + "artefacts/", + "docs/architecture/adrs/README.md", + ".env", + "RELEASE_PROCESS.md", + "CHANGELOG.md", + ".release-please-manifest.json", + ".github/rulesets/", + "docs/reference/branch-protection.md", + "audit/tools/", + "implement/tools/", + "audit/engine/tests/", + "audit/tools/tests/", + "implement/tools/tests/", + "dev/tests/" + ], + "truncated": false + }, + "id": "CON-01", + "pack": "CON", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "the prohibitions section of AGENTS.md names paths that exist: Makefile, audit/engine/, AGENTS.md, .python-version, dev/tools/pyproject.toml, dev/tools/package.json, lefthook.yml, dev (+44 more)", + "tier": "B", + "title": "Allow/deny artifact present", + "verdict": "PASS" + }, + { + "data": { + "kind": "path_list", + "paths": [ + "audit/engine/tests", + "audit/tools/tests", + "dev/tests", + "implement/tools/tests" + ], + "truncated": false + }, + "id": "CON-02", + "pack": "CON", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "the prohibitions section of AGENTS.md names the test directories (audit/engine/tests/, audit/tools/tests/, dev/tests/, implement/tools/tests/)", + "tier": "B", + "title": "Test directories protected", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CON-03", + "pack": "CON", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "artefacts/ present, ignored by artefacts/.gitignore, and untracked", + "tier": "B", + "title": "artefacts/ excluded from index and agent reads", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CON-04", + "pack": "CON", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "draft not emitted; pass --emit-baseline to write do-not-do.draft.md beside the report", + "tier": "\u2014", + "title": "Draft allow/deny emitted", + "verdict": "UNKNOWN" + }, + { + "data": {}, + "id": "EXEC-01", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "pinned: node (.nvmrc); python (.python-version)", + "tier": "A", + "title": "Toolchain pinned / wrapper committed", + "verdict": "PASS" + }, + { + "data": {}, + "id": "EXEC-02", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "consistency unverified: dev/tools/package-lock.json, dev/tools/uv.lock", + "tier": "B", + "title": "Lockfile committed and consistent", + "verdict": "PARTIAL" + }, + { + "data": {}, + "id": "EXEC-03", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "documented setup command: AGENTS.md: make setup", + "tier": "A", + "title": "Non-interactive setup path", + "verdict": "PASS" + }, + { + "data": { + "kind": "env_keys", + "missing": [], + "referenced": [ + "TMPDIR" + ], + "templates": [] + }, + "id": "EXEC-04", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "1 referenced key(s) declared", + "tier": "B", + "title": "Env-var inventory complete", + "verdict": "PASS" + }, + { + "data": {}, + "id": "EXEC-05", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "no tracked generated or vendored paths", + "tier": "B", + "title": "No generated artifacts committed in-tree", + "verdict": "PASS" + }, + { + "data": {}, + "id": "EXEC-06", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "15 documented command(s) resolve", + "tier": "B", + "title": "Documented commands resolve", + "verdict": "PASS" + }, + { + "data": {}, + "id": "EXEC-07", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "lint: ruff.toml; types: mypy.ini", + "tier": "B", + "title": "No competing configs for one concern", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TST-01", + "pack": "TST", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "test verb on Makefile (cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests); 85 test file(s)", + "tier": "A", + "title": "Test verb exists, suite non-empty", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TST-02", + "pack": "TST", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "probe not run: probes disabled (pass --run-gates)", + "tier": "C", + "title": "Suite status known", + "verdict": "UNKNOWN" + }, + { + "data": { + "breakdown": { + "python": 1 + }, + "kind": "counter", + "unit": "skip/xfail markers", + "value": 1 + }, + "id": "TST-03", + "pack": "TST", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "skip/xfail census across 85 test file(s): python 1; total 1", + "tier": "B", + "title": "Skip / xfail census", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TST-04", + "pack": "TST", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "fast path documented: AGENTS.md:36 `python3 -m unittest tests.test_cmd_pack -k verbs` (scope: README.md, AGENTS.md, CONTRIBUTING.md)", + "tier": "B", + "title": "Fast path documented", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TST-05", + "pack": "TST", + "phase": 1, + "scored": true, + "severity": "COSMETIC", + "status": "implemented", + "summary": "coverage configured: .coveragerc", + "tier": "A", + "title": "Coverage configuration present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "NAV-01", + "pack": "NAV", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "single-root repository declares package.json", + "tier": "A", + "title": "Declared component boundaries", + "verdict": "PASS" + }, + { + "data": {}, + "id": "NAV-02", + "pack": "NAV", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "entry point(s): audit/engine/audit/__main__.py, audit/engine/build.py, audit/tools/engine.py, audit/tools/facts.py, audit/tools/render_report.py, audit/tools/report.py", + "tier": "B", + "title": "Obvious entry points", + "verdict": "PASS" + }, + { + "data": {}, + "id": "NAV-03", + "pack": "NAV", + "phase": 1, + "scored": true, + "severity": "COSMETIC", + "status": "implemented", + "summary": "oversized(>524288 B): 0; catch-all dirs(>50 files): 0; binaries: 0; minified: 0", + "tier": "B", + "title": "Structural red flags", + "verdict": "PASS" + }, + { + "data": {}, + "id": "NAV-04", + "pack": "NAV", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "docs/architecture/; an ADR directory", + "tier": "A", + "title": "Architecture orientation artifact", + "verdict": "PASS" + }, + { + "data": { + "denominator": 1152, + "kind": "ratio", + "method": "python: module-level def/class not starting with '_'", + "numerator": 456, + "unit": "public symbols" + }, + "id": "NAV-05", + "pack": "NAV", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "456/1152 public symbols documented (40%); method: python: module-level def/class not starting with '_'", + "tier": "B", + "title": "Public-symbol documentation coverage", + "verdict": "PARTIAL" + }, + { + "data": { + "covered": [], + "kind": "secret_shapes", + "missing": [], + "root_only": [ + "dotenv", + "dotenv variant" + ], + "shapes": [ + { + "covered": false, + "label": "dotenv", + "probe": ".env", + "root_only": true, + "suggested": ".env" + }, + { + "covered": false, + "label": "dotenv variant", + "probe": ".env.local", + "root_only": true, + "suggested": ".env.*" + } + ] + }, + "id": "SEC-01", + "pack": "SEC", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "no tracked .env, but ignore rules are incomplete: root-anchored only: dotenv, dotenv variant (the rule covers the root, not nested files such as `a/.env`)", + "tier": "B", + "title": ".env untracked and ignored", + "verdict": "PARTIAL" + }, + { + "data": { + "allowlisted": 2, + "allowlisted_by": [ + ".secrets.baseline" + ], + "files": [], + "kind": "credential_matrix", + "source": "tracked text files", + "suppressed": [ + { + "lines": [ + 38 + ], + "path": "audit/engine/tests/test_report_render.py" + }, + { + "lines": [ + 41 + ], + "path": "audit/engine/tests/test_sec_pack.py" + } + ], + "total": 0 + }, + "id": "SEC-02", + "pack": "SEC", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "0 credential-shaped lines outside the allowlist in tracked text files; 2 allowlisted by `.secrets.baseline`", + "tier": "B", + "title": "No key-shaped strings in tracked files", + "verdict": "PASS" + }, + { + "data": { + "covered": [ + "ssh private key", + "PEM", + "key", + "PKCS#12", + "PFX", + "keystore", + "certificate (.crt)", + "certificate (.cer)", + "AWS credentials" + ], + "kind": "secret_shapes", + "missing": [], + "root_only": [ + "dotenv", + "dotenv variant" + ], + "shapes": [ + { + "covered": false, + "label": "dotenv", + "probe": ".env", + "root_only": true, + "suggested": ".env" + }, + { + "covered": false, + "label": "dotenv variant", + "probe": ".env.local", + "root_only": true, + "suggested": ".env.*" + }, + { + "covered": true, + "label": "ssh private key", + "probe": "id_rsa", + "root_only": false, + "suggested": "id_rsa*" + }, + { + "covered": true, + "label": "PEM", + "probe": "server.pem", + "root_only": false, + "suggested": "*.pem" + }, + { + "covered": true, + "label": "key", + "probe": "server.key", + "root_only": false, + "suggested": "*.key" + }, + { + "covered": true, + "label": "PKCS#12", + "probe": "server.p12", + "root_only": false, + "suggested": "*.p12" + }, + { + "covered": true, + "label": "PFX", + "probe": "server.pfx", + "root_only": false, + "suggested": "*.pfx" + }, + { + "covered": true, + "label": "keystore", + "probe": "app.keystore", + "root_only": false, + "suggested": "*.keystore" + }, + { + "covered": true, + "label": "certificate (.crt)", + "probe": "server.crt", + "root_only": false, + "suggested": "*.crt" + }, + { + "covered": true, + "label": "certificate (.cer)", + "probe": "server.cer", + "root_only": false, + "suggested": "*.cer" + }, + { + "covered": true, + "label": "AWS credentials", + "probe": ".aws/credentials", + "root_only": false, + "suggested": ".aws/*" + } + ] + }, + "id": "SEC-03", + "pack": "SEC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "covered: ssh private key, PEM, key, PKCS#12, PFX, keystore, certificate (.crt), certificate (.cer), AWS credentials; root-anchored only: dotenv, dotenv variant (the rule covers the root, not nested files such as `a/.env`)", + "tier": "B", + "title": "Ignore patterns cover secret shapes", + "verdict": "PARTIAL" + }, + { + "data": {}, + "id": "HYG-01", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "README.md present and non-placeholder", + "tier": "A", + "title": "README present and non-placeholder", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-02", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "CODEOWNERS present at .github/CODEOWNERS", + "tier": "A", + "title": "CODEOWNERS present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-03", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "pull-request template present at .github/pull_request_template.md", + "tier": "A", + "title": "Pull-request template present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-04", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "2 issue template(s) present", + "tier": "A", + "title": "Issue templates present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-05", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "CONTRIBUTING present at CONTRIBUTING.md", + "tier": "A", + "title": "CONTRIBUTING present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-06", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "SECURITY present at SECURITY.md", + "tier": "A", + "title": "SECURITY present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-07", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "CHANGELOG.md last changed 0 day(s) before HEAD", + "tier": "A", + "title": "CHANGELOG present and recent", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-08", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "release document present at RELEASE_PROCESS.md", + "tier": "A", + "title": "Release process documented", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-09", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "dependency automation configured at .github/dependabot.yml", + "tier": "A", + "title": "Dependency update automation configured", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-10", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "license present at LICENSE", + "tier": "A", + "title": "License present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-11", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "branch protection declared as code at docs/reference/branch-protection.md", + "tier": "B", + "title": "Branch protection declared as code", + "verdict": "PASS" + } + ], + "components": [ + { + "agents_md": "present", + "declared": true, + "declared_by": "implicit", + "manifest": "package.json", + "name": "(root)", + "path": "." + } + ], + "findings": [ + { + "check": "SEC-01", + "evidence": [ + { + "line": null, + "note": "__pycache__", + "path": ".gitignore" + }, + { + "line": null, + "note": "*.py[cod]", + "path": ".gitignore" + }, + { + "line": null, + "note": ".mypy_cache", + "path": ".gitignore" + }, + { + "line": null, + "note": ".ruff_cache", + "path": ".gitignore" + }, + { + "line": null, + "note": "/dev/dist", + "path": ".gitignore" + }, + { + "line": null, + "note": "/audit/engine/dist", + "path": ".gitignore" + }, + { + "line": null, + "note": "/dist", + "path": ".gitignore" + }, + { + "line": null, + "note": "*.pyz", + "path": ".gitignore" + } + ], + "id": "SEC-01", + "phase": 1, + "remediation": "Add the .env shape without a leading slash (for example `.env` and `.env.*`) to .gitignore so an environment file cannot be committed by accident, at any depth.", + "severity": "BLOCKER", + "statement": "An agent cannot trust that a future .env stays out of history because ignore rules are incomplete (root-anchored only: dotenv, dotenv variant (the rule covers the root, not nested files such as `a/.env`)).", + "verdict": "PARTIAL" + }, + { + "check": "BASE-02", + "evidence": [ + { + "line": null, + "note": null, + "path": ".audit-baseline.json" + }, + { + "line": null, + "note": null, + "path": ".quality-baseline.json" + }, + { + "line": null, + "note": null, + "path": ".secrets.baseline" + }, + { + "line": null, + "note": null, + "path": "Makefile:lint" + }, + { + "line": null, + "note": null, + "path": "Makefile:security" + }, + { + "line": null, + "note": null, + "path": "Makefile:typecheck" + } + ], + "id": "BASE-02", + "phase": 1, + "remediation": "Commit the count or configure a budget (`--max-warnings N`, `max-issues`, or a `violation budget`) so a regression has a baseline number to compare against.", + "severity": "DEGRADER", + "statement": "An agent cannot state how many violations exist today because a counting surface exists but no numeric violation budget or count is committed.", + "verdict": "PARTIAL" + }, + { + "check": "SEC-03", + "evidence": [ + { + "line": null, + "note": "__pycache__", + "path": ".gitignore" + }, + { + "line": null, + "note": "*.py[cod]", + "path": ".gitignore" + }, + { + "line": null, + "note": ".mypy_cache", + "path": ".gitignore" + }, + { + "line": null, + "note": ".ruff_cache", + "path": ".gitignore" + }, + { + "line": null, + "note": "/dev/dist", + "path": ".gitignore" + }, + { + "line": null, + "note": "/audit/engine/dist", + "path": ".gitignore" + }, + { + "line": null, + "note": "/dist", + "path": ".gitignore" + }, + { + "line": null, + "note": "*.pyz", + "path": ".gitignore" + } + ], + "id": "SEC-03", + "phase": 1, + "remediation": "Add the missing shapes to .gitignore \u2014 for example `.env.*`, `*.pem`, `*.key`, `*.p12`, `*.pfx`, `*.keystore`, `id_rsa*`, `*.crt`, `*.cer`, `.aws/*`.", + "severity": "DEGRADER", + "statement": "An agent cannot trust that secret shapes cannot be committed because ignore rules are incomplete (root-anchored only: dotenv, dotenv variant (the rule covers the root, not nested files such as `a/.env`)).", + "verdict": "PARTIAL" + }, + { + "check": "TOOL-04", + "evidence": [ + { + "line": null, + "note": "git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline", + "path": "Makefile" + } + ], + "id": "TOOL-04", + "phase": 1, + "remediation": "Make the security verb run a dependency audit (npm audit, pip-audit, osv-scanner, cargo audit, govulncheck, snyk).", + "severity": "DEGRADER", + "statement": "An agent cannot know whether a dependency carries a known vulnerability because a security verb exists but it does not run a dependency-audit check.", + "verdict": "PARTIAL" + } + ], + "inventory": { + "files_inspected": 336, + "files_set_aside": 460, + "files_skipped": 3, + "set_aside": [ + { + "files": 3, + "path": "audit/engine/examples" + }, + { + "files": 276, + "path": "audit/engine/tests/fixtures" + }, + { + "files": 32, + "path": "audit/schema/examples" + }, + { + "files": 92, + "path": "dev/eval/expected/fixtures" + }, + { + "files": 13, + "path": "dev/eval/implement-fixtures/testdata" + }, + { + "files": 5, + "path": "docs/templates" + }, + { + "files": 39, + "path": "greenfield/templates" + } + ], + "truncated": false + }, + "not_applicable": [], + "probes": [ + { + "command": [ + "cd", + "$(ENGINE)", + "&&", + "$(PYTHON)", + "-m", + "unittest", + "discover", + "-s", + "tests", + "&&", + "$(PYTHON)", + "-m", + "unittest", + "discover", + "-s", + "audit/tools/tests", + "&&", + "$(PYTHON)", + "-m", + "unittest", + "discover", + "-s", + "implement/tools/tests", + "&&", + "$(PYTHON)", + "-m", + "unittest", + "discover", + "-s", + "dev/tests" + ], + "duration_s": 0.0, + "exit_code": null, + "output_tail": "", + "redactions": 0, + "refused_reason": "probes disabled (pass --run-gates)", + "timed_out": false, + "verb": "test" + } + ], + "provenance": { + "framework_revision": "320fad8969a38d09331ece11634cc36e124db725", + "git_branch": "feat/v0.3.3-track-b-mcp", + "git_dirty": true, + "git_sha": "320fad8969a38d09331ece11634cc36e124db725", + "phase": 1, + "probes_enabled": false, + "ruleset_hash": "970e6792f98eeeb5", + "ruleset_revision": "2026-11-06.1", + "scanned_at": "2026-11-09T05:04:17+00:00", + "target": ".", + "tool": "python-agentic-audit", + "user_config": { + "paths_read": [], + "read_allowed": false + }, + "version": "0.3.2" + }, + "schema_version": "2", + "stack": { + "ci_providers": [ + "github" + ], + "ecosystems": [ + "node", + "python" + ], + "hook_managers": [ + "lefthook" + ], + "notes": [], + "package_managers": [ + "npm", + "uv" + ], + "test_frameworks": [] + }, + "summary": { + "blockers": 1, + "checks": { + "attest": 0, + "fail": 0, + "partial": 6, + "pass": 59, + "unknown": 4 + }, + "checks_applicable": 69, + "checks_implemented": 69, + "checks_scoreable": 55, + "cosmetic": 0, + "coverage_note": "55 of 55 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", + "degraders": 3, + "phase1_score": 0.8934 + }, + "unattested": [ + "index-in-use", + "baseline-adequacy", + "suite-trustworthiness", + "doc-accuracy", + "gate-honoured" + ] +} diff --git a/dev/tests/test_mcp_config.py b/dev/tests/test_mcp_config.py new file mode 100644 index 00000000..b8443409 --- /dev/null +++ b/dev/tests/test_mcp_config.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna + +"""The project MCP file, ``.mcp.json``, registers ``codebase-memory-mcp`` and nothing that runs without consent. + +The file is the one source for agent tools in this repository. Claude Code and VS Code read it directly; other +harnesses copy its entry. It names a bare command, so each person's own install runs and no version is written +here (the one-place pin rule of ``test_tool_pins.py``). Approval stays with each person: the committed Claude Code +settings never pre-approve a project server. +""" + +from __future__ import annotations + +import json +import unittest + +import _dev_paths # noqa: F401 (puts dev/ on sys.path) +from _dev_paths import REPO + +MCP_FILE = REPO / ".mcp.json" +CLAUDE_SETTINGS = REPO / ".claude" / "settings.json" +SERVER = "codebase-memory-mcp" +PRE_APPROVAL_KEYS = ("enableAllProjectMcpServers", "enabledMcpjsonServers") + + +def servers() -> dict[str, dict[str, object]]: + data = json.loads(MCP_FILE.read_text(encoding="utf-8")) + result: dict[str, dict[str, object]] = data["mcpServers"] + return result + + +class McpFileRegistersTheIndex(unittest.TestCase): + def test_only_the_index_server_is_listed(self) -> None: + self.assertEqual(set(servers()), {SERVER}) + + def test_the_command_is_a_bare_name(self) -> None: + entry = servers()[SERVER] + command = entry["command"] + self.assertEqual(command, SERVER, "use the bare command: no path, no version, no package runner") + self.assertNotIn("args", entry) + + def test_no_server_carries_environment_values(self) -> None: + for name, entry in servers().items(): + with self.subTest(server=name): + self.assertNotIn("env", entry) + self.assertNotIn("envFile", entry) + + +class ApprovalStaysWithEachPerson(unittest.TestCase): + def test_committed_settings_do_not_pre_approve_project_servers(self) -> None: + settings = json.loads(CLAUDE_SETTINGS.read_text(encoding="utf-8")) + for key in PRE_APPROVAL_KEYS: + with self.subTest(key=key): + self.assertNotIn(key, settings) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 9f4f0068..0e0c5680 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -78,6 +78,30 @@ another file repeats a version. To change a version, follow [Change a tool version](../../RELEASE_PROCESS.md#change-a-tool-version). +## Agent tools (MCP) + +MCP servers make an agent faster. No gate, CI job or release step needs one. Anything that must be right runs +through `make`. + +`.mcp.json` at the repository root registers one server, `codebase-memory-mcp` (the code index). It names the bare +command, so each person installs the server and keeps its version. No version is written in the repository. +`dev/tests/test_mcp_config.py` checks the file, and checks that `.claude/settings.json` approves no project server +for you. + +How each harness treats a project MCP file (checked 2026-11-09 in each harness's own docs): + +| Harness | Reads | First start in this repository | +| --- | --- | --- | +| Claude Code | `.mcp.json` | Asks you to approve the server, once per person | +| VS Code with Copilot | `.mcp.json` or `.vscode/mcp.json` | Starts it when the folder is trusted | +| Cursor | `.cursor/mcp.json` | You turn each server on or off | +| Gemini CLI | `.gemini/settings.json` (`mcpServers`) | Starts it after you trust the folder | +| opencode | `opencode.json` (`mcp`) | Connects it with no prompt | +| Codex CLI | `.codex/config.toml` (`[mcp_servers]`) | Loads it only for a trusted project | + +For a harness that does not read `.mcp.json`, copy the entry into its own file, outside the repository or +uncommitted. If the server is not installed, it fails to start and the agent reads files instead. + ## Settings on GitHub These checks run on GitHub, outside the repository files: diff --git a/docs/reference/self-audit.md b/docs/reference/self-audit.md index 4226d8ad..e8eec815 100644 --- a/docs/reference/self-audit.md +++ b/docs/reference/self-audit.md @@ -1,7 +1,7 @@ # Self-Audit > **Audience:** Everyone -> **Source of truth:** [`schema/examples/self-audit/2026-11-06.json`](../../audit/schema/examples/self-audit/2026-11-06.json) +> **Source of truth:** [`schema/examples/self-audit/2026-11-09.json`](../../audit/schema/examples/self-audit/2026-11-09.json) > (the engine's report for this repository) and the root [`.audit-baseline.json`](../../.audit-baseline.json) > (the same report, kept as the CI ratchet). This page restates the report; the report wins. > **Status:** Live. Regenerated by hand when the baseline changes (see [Keep it true](#keep-it-true)). @@ -16,12 +16,12 @@ score from falling. | Field | Value | | --- | --- | -| Phase-1 score | **0.8402** | -| Date | 2026-11-06 | -| Engine | `python-agentic-audit` 0.1.0 | +| Phase-1 score | **0.8934** | +| Date | 2026-11-09 | +| Engine | `python-agentic-audit` 0.3.2 | | Ruleset | `2026-11-06.1` (hash `970e6792f98eeeb5`) | | Applicable checks | 69 (55 scoreable) | -| Findings | 7: 2 BLOCKER, 3 DEGRADER, 2 COSMETIC | +| Findings | 4: 1 BLOCKER, 3 DEGRADER | | Scope | the whole repository except `artefacts/` and `dist/`; no user-level configuration read | `HYG-05`, `HYG-07`, `HYG-08` and `HYG-09` closed with `CONTRIBUTING.md`, the root changelog, `RELEASE_PROCESS.md` and @@ -33,14 +33,14 @@ The score was 0.6762 on 2026-11-05, before the repository adopted the Phase 1 re | Verdict | Checks | | --- | --: | -| PASS | 55 | -| PARTIAL | 5 | -| FAIL | 3 | -| UNKNOWN | 6 | +| PASS | 59 | +| PARTIAL | 6 | +| FAIL | 0 | +| UNKNOWN | 4 | | ATTEST | 0 | | **Total** | **69** | -`UNKNOWN` and `ATTEST` earn no credit. Six checks are `UNKNOWN` because their evidence needs a Tier-C probe +`UNKNOWN` and `ATTEST` earn no credit. Four checks are `UNKNOWN` because their evidence needs a Tier-C probe (`--run-gates`, not used here) or the user-level configuration that `--no-user-config` leaves unread, so this score understates the repository rather than certifying it. @@ -50,17 +50,16 @@ Every check that is not `PASS`, with the reason it is open. | Check | Verdict | Severity | Why it is open | | --- | --- | --- | --- | -| `IDX-01` | FAIL | BLOCKER | **Owner exception** (D21): no project `.mcp.json`. The index is optional here, and a project file would make every contributor's harness offer to run a binary. Review in v0.3 | | `SEC-01` | PARTIAL | BLOCKER | `/.env` is root-anchored on purpose: the engine's fixtures hold `.env` files as data and must stay committable (comment in `.gitignore`). The check reads a root-only rule as incomplete | | `SEC-03` | PARTIAL | DEGRADER | Same cause as `SEC-01` for the dotenv shapes; the key, certificate and keystore shapes are covered | | `BASE-02` | PARTIAL | DEGRADER | The T5 playbook was applied (D32): `.quality-baseline.json` holds the counts (ruff 0, mypy 0, Markdown lint outside the ratchet 735) and `make format-check`, `make typecheck` and `make docs-lint` fail when one goes up. The check still reads PARTIAL by design: it credits only a numeric budget flag (`--max-warnings N`), and none of these tools has one. Engine issue [#45](https://github.com/Nistapp/codebase-agentic-readiness-framework/issues/45) tracks it; the coverage floor (91) is unchanged and `BASE-04` passes | | `TOOL-04` | PARTIAL | DEGRADER | **Owner exception** (D33): the repository has no third-party dependencies, so there is nothing for a dependency audit to scan; `make security` is a secrets scan | -| `HYG-02`, `HYG-06` | FAIL | COSMETIC | Need values only the owner has: `CODEOWNERS` names and a private disclosure contact for `SECURITY.md`. Stay open until supplied (D22) | -| `NAV-05` | PARTIAL | COSMETIC | Informational, no work (D34): 44% of public symbols documented (638 of 1438) | -| `IDX-02`, `IDX-03`, `TST-02`, `CON-04` | UNKNOWN | mixed | Need a probe, user-level configuration, or `--emit-baseline`; not run here | -| `EXEC-02`, `NAV-01` | UNKNOWN | mixed | The repository declares no dependencies and no components, so there is nothing for the check to read | +| `NAV-05` | PARTIAL | COSMETIC | Informational, no work (D34): 40% of public symbols documented (456 of 1152) | +| `IDX-02`, `IDX-03`, `TST-02`, `CON-04` | UNKNOWN | mixed | Need a probe, the user-level index store, or `--emit-baseline`; not run here | +| `EXEC-02` | PARTIAL | BLOCKER | The two lockfiles of the pinned dev tools (`dev/tools/uv.lock`, `dev/tools/package-lock.json`) cannot be checked against their manifests without running the package managers. `dev/tests/test_tool_pins.py` checks that they agree; the engine does not read that test | -Recorded decisions that keep a finding open or exempt: `IDX-01` and `TOOL-04` are owner exceptions (D21, D33); +Recorded decisions that keep a finding open or exempt: `TOOL-04` is an owner exception (D33). The `IDX-01` exception +(D21) ended on 2026-11-09: `.mcp.json` now registers `codebase-memory-mcp`, and each person still approves it. `AGT-08` stays as it is for v0.2, because a stated harness list closes the gap even when a named harness does not read `AGENTS.md` natively, and is revisited in v0.3 (D31); `NAV-05` is informational and unscored (D34). The engine has no way to record an owner exception, so this page is the record. Everything else is either a deliberate design choice stated in the From e6e2c91dc75a20202d1ec1749bb3187df6d1dee5 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 10:58:08 +0530 Subject: [PATCH 007/111] test(audit): let an engine check follow an agent-only check's number The catalogue test required every agent-only check to be numbered above every engine check in its family, so no engine check could ever join the CI family after the agent-only CI-04. The documented rule is narrower: an engine check takes the next free number and must not take an agent check's ID. The test now checks that rule, and the explainer says it plainly. Roadmap-Step: C1.1 Co-Authored-By: Claude Opus 5.5 --- audit/tools/tests/test_catalogue.py | 6 ++++-- .../contributor-deep-dive/03-check-catalogue-and-rubrics.md | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/audit/tools/tests/test_catalogue.py b/audit/tools/tests/test_catalogue.py index c8d50cf5..031d5756 100644 --- a/audit/tools/tests/test_catalogue.py +++ b/audit/tools/tests/test_catalogue.py @@ -167,7 +167,10 @@ def test_no_catalogue_check_claims_an_engine_id_the_engine_lacks(self) -> None: if c["engine_id"] is not None: self.assertIn(c["engine_id"], engine_ids, f"{c['id']} names an engine check that does not exist") - def test_agent_only_checks_are_numbered_after_the_engines_in_their_family(self) -> None: + def test_agent_only_checks_never_share_an_id_with_an_engine_check(self) -> None: + """An agent-only check took the next free number when it was added; an engine check added later takes the + next free number after it. Neither may take the other's ID (docs/explanation/contributor-deep-dive/ + 03-check-catalogue-and-rubrics.md).""" engine_ids = {e["id"] for e in self.engine} agent = [c for c in self.checks if c["engine_id"] is None] self.assertTrue(agent, "expected at least one agent-only check") @@ -176,7 +179,6 @@ def test_agent_only_checks_are_numbered_after_the_engines_in_their_family(self) self.assertNotIn(c["id"], engine_ids, f"{c['id']} reuses an engine ID") family_numbers = [int(e["id"].split("-")[1]) for e in self.engine if e["pack"] == c["family"]] self.assertTrue(family_numbers, f"{c['id']}: the family {c['family']} has no engine checks") - self.assertGreater(int(c["id"].split("-")[1]), max(family_numbers), f"{c['id']} is not after the engine's") def test_requirements_and_the_catalogue_agree_both_ways(self) -> None: requirements = requirement_fields() diff --git a/docs/explanation/contributor-deep-dive/03-check-catalogue-and-rubrics.md b/docs/explanation/contributor-deep-dive/03-check-catalogue-and-rubrics.md index d37da867..8df23b35 100644 --- a/docs/explanation/contributor-deep-dive/03-check-catalogue-and-rubrics.md +++ b/docs/explanation/contributor-deep-dive/03-check-catalogue-and-rubrics.md @@ -37,7 +37,7 @@ checks, all informational and unscored). The catalogue itself, Where the entries come from: - **Engine checks** are seeded from the engine's own list, `python3 -m audit --list-checks --format json` ([`cli.py`](../../../audit/engine/audit/cli.py)). After seeding the file is edited by hand. The engine's ID is the check's ID, and `engine_id` repeats it. -- **Agent-only checks** are the ones no deterministic rule can settle. Today there are two: `AGT-11` (instructions are specific and not inferable, for [REQ-SH-015](../../../spec/readiness-requirements.md#req-sh-015--instructions-are-specific-and-not-inferable)) and `CI-04` (CI enforces the same gate as the local check, for [REQ-SH-035](../../../spec/readiness-requirements.md#req-sh-035--ci-enforces-the-gate)). They are numbered after the engine's highest number in their family, so an engine check added later cannot collide with one: the engine takes the next free number and the test fails if it takes an agent check's. +- **Agent-only checks** are the ones no deterministic rule can settle. Today there are two: `AGT-11` (instructions are specific and not inferable, for [REQ-SH-015](../../../spec/readiness-requirements.md#req-sh-015--instructions-are-specific-and-not-inferable)) and `CI-04` (CI enforces the same gate as the local check, for [REQ-SH-035](../../../spec/readiness-requirements.md#req-sh-035--ci-enforces-the-gate)). Each took the next free number in its family when it was added. An engine check added later takes the next free number after it, and the test fails only if an engine check takes an agent check's ID. - **`status`** is the engine's own status for an engine check. An agent-only check is `planned` until its rubric exists. A check that is planned or blocked is reported `UNKNOWN`. `scored` is also the engine's: `CON-01` was `blocked` and unscored until catalogue 0.2.0, when the engine began to read the prohibitions section. ## 2. Methods From 4c9c82daf1c5f1a28cfc1e9d0a294268428e8648 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 10:58:08 +0530 Subject: [PATCH 008/111] feat(audit): record a check newer than the engine's ruleset as UNKNOWN A catalogue entry may now carry since_ruleset, the engine ruleset that added it. When import-engine reads a report from an older ruleset, each such check is recorded as UNKNOWN with no engine verdict and a warning, so the draft can still be finalised and the check earns no credit. Any other check a report lacks stays missing, as before. Roadmap-Step: C1.1 Co-Authored-By: Claude Opus 5.5 --- audit/schema/SCHEMA.md | 1 + audit/schema/catalogue.schema.json | 5 ++ audit/tools/_engine_adapter.py | 51 +++++++++++++++++++ audit/tools/tests/test_engine_adapter.py | 64 ++++++++++++++++++++++++ 4 files changed, 121 insertions(+) diff --git a/audit/schema/SCHEMA.md b/audit/schema/SCHEMA.md index ba88276e..819cfd9a 100644 --- a/audit/schema/SCHEMA.md +++ b/audit/schema/SCHEMA.md @@ -405,6 +405,7 @@ Every field of the engine's schema v2 is either carried or named as not carried. | `probes[]` | `evidence[]` with `kind: command` and `captured_by: tool`, on the check that ran the probe, after the file evidence gathered from its findings: `command` (the engine's argument list, written as a shell would read it), `exit_code`, `output_tail` (when there is one), `timed_out`, `duration_s` and `redactions`. The engine's probe record does not name its check, so the adapter holds a table of verb to check, today `test` to `TST-02`, and a test fails if the engine gains another check that builds a probe plan. `verb` is not carried. A probe with a `refused_reason` ran nothing, and its check's summary says why, so it is not carried and needs no warning. A probe that ran, whose verb or check the adapter does not know, is a warning. | | `unattested[]` | `unattested[]`, in the engine's order: each of the five fixed items as the requirement in the table above, with `method` `attested`, `property` the engine's item name and `why` the engine's own description of it (read from the engine, never copied). An item the table does not have is a warning and is not carried. Each requirement of a check the engine marks `ATTEST` follows; see the verdict table. | | `not_applicable[]` | `checks[]` with `verdict: NOT_APPLICABLE` and `engine_verdict: NOT_APPLICABLE`, the title, phase, severity and `scored` flag from the catalogue and a `summary` that says the engine did not run it. They follow the engine's checks. | +| (a check newer than the report's ruleset) | A catalogue check whose `since_ruleset` is later than the report's `provenance.ruleset_revision` cannot be in the report. It becomes a check with `method: automated`, `verdict: UNKNOWN`, no `engine_verdict`, no evidence, the catalogue's title, phase, severity and `scored` flag, and a `summary` that says to scan again with a newer engine; a warning names it. UNKNOWN earns no credit, and the draft can still be finished. Any other check the report lacks stays missing, and so does every check when the report gives no ruleset revision. | | `ratchet` | Not carried. A re-audit compares reports by finding `key`. | | (none) | `method: automated` and `disposition: pending`; everything under `run_context`; `provenance` other than the engine's and the target's; `coverage.scopes_examined` and `scopes_skipped`, which start empty because the engine reports no scopes | diff --git a/audit/schema/catalogue.schema.json b/audit/schema/catalogue.schema.json index ae42df52..93066e5b 100644 --- a/audit/schema/catalogue.schema.json +++ b/audit/schema/catalogue.schema.json @@ -66,6 +66,11 @@ "type": ["string", "null"], "pattern": "^[A-Z]{2,5}-[0-9]{2,3}$" }, + "since_ruleset": { + "description": "The engine ruleset revision that added this engine check; set for checks added after ruleset 2026-11-06.1. When an engine report from an older ruleset lacks the check, report.py import-engine records it as UNKNOWN, so the draft can still be finished. Absent for older checks and for agent-only checks.", + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}\\.[0-9]+$" + }, "playbooks": { "description": "The playbooks that remediate this check's findings. A playbook may be named before it is written.", "type": "array", diff --git a/audit/tools/_engine_adapter.py b/audit/tools/_engine_adapter.py index d77b1fee..e58c2384 100644 --- a/audit/tools/_engine_adapter.py +++ b/audit/tools/_engine_adapter.py @@ -55,6 +55,9 @@ } NOT_APPLICABLE_SUMMARY = "The engine did not run this check: it does not apply to the detected stack." BLOCKED_SUMMARY = "The engine did not evaluate this check: its ruleset had blocked it." +NOT_IN_RULESET_SUMMARY = ( + "The engine did not run this check: its ruleset is older than the catalogue. Scan again with a newer engine." +) STACK_LISTS = ("ecosystems", "package_managers", "ci_providers", "test_frameworks", "hook_managers") @@ -287,6 +290,15 @@ def safe_key(engine_id: str) -> str: # --- The conversion ----------------------------------------------------------------------------------------- +RULESET_REVISION = re.compile(r"^([0-9]{4}-[0-9]{2}-[0-9]{2})\.([0-9]+)$") + + +def ruleset_order(revision: Any) -> tuple[str, int] | None: + """A ruleset revision (`2026-11-09.1`) as a pair that sorts in time order, or `None` when it is not one.""" + found = RULESET_REVISION.match(revision) if isinstance(revision, str) else None + return (found.group(1), int(found.group(2))) if found else None + + @dataclass(frozen=True) class Conversion: """What an engine report becomes: the engine's own record, the target it says it scanned, and the checks. @@ -585,6 +597,43 @@ def not_applicable(self, number: int, check_id: str) -> dict[str, Any] | None: "searches": [], } + def not_in_ruleset(self, reported: Collection[str]) -> list[dict[str, Any]]: + """The catalogue's engine checks that are newer than the report's ruleset, as UNKNOWN with no engine verdict. + + A catalogue entry with `since_ruleset` names the engine ruleset that added it. When the report's ruleset is + older, the report cannot hold the check, so it is recorded as UNKNOWN, which earns no credit, and the draft can + still be finished; a scan with a newer engine evaluates it. Any other check the report lacks stays missing, and + so does every check when the report gives no ruleset revision. + """ + revision = ruleset_order(self.engine["provenance"].get("ruleset_revision")) + if revision is None: + return [] + missing: list[dict[str, Any]] = [] + for entry in self.entries.values(): + check_id = entry["id"] + since = ruleset_order(entry.get("since_ruleset")) + if since is None or revision >= since or entry["phase"] != PHASE or check_id in reported: + continue + self.warnings.append( + f"{check_id}: the engine's ruleset {self.engine['provenance']['ruleset_revision']} is older than the " + f"check ({entry['since_ruleset']}), so it is recorded as UNKNOWN; scan again with a newer engine" + ) + missing.append( + { + "id": check_id, + "title": entry["title"], + "phase": entry["phase"], + "severity": entry["severity"], + "scored": entry["scored"], + "method": "automated", + "verdict": "UNKNOWN", + "summary": NOT_IN_RULESET_SUMMARY, + "evidence": [], + "searches": [], + } + ) + return missing + def inventory(self) -> dict[str, Any]: """The engine's file counts, in the report's order. A scan the engine cut short is said.""" raw = self.engine["inventory"] @@ -697,6 +746,8 @@ def run(self) -> Conversion: skipped = self.not_applicable(number, check_id) if skipped is not None: checks.append(skipped) + reported = {raw["id"] for raw in raw_checks} | set(self.engine["not_applicable"]) + checks.extend(self.not_in_ruleset(reported)) for check_id in sorted(set(commands) - {made["id"] for made in checks if made["verdict"] != "NOT_APPLICABLE"}): self.warnings.append(f"probe: {check_id} ran it, and the report holds no such check, so it is not carried") if self.problems: diff --git a/audit/tools/tests/test_engine_adapter.py b/audit/tools/tests/test_engine_adapter.py index 0ef5dd56..80fb9a29 100644 --- a/audit/tools/tests/test_engine_adapter.py +++ b/audit/tools/tests/test_engine_adapter.py @@ -1155,6 +1155,70 @@ def test_a_scan_the_engine_cut_short_is_said(self) -> None: ) +class NewerCheckTests(unittest.TestCase): + """A catalogue check added after the report's ruleset (`since_ruleset`) is recorded as UNKNOWN; others stay missing.""" + + def catalogue(self, since: str | None) -> dict[str, Any]: + catalogue: dict[str, Any] = copy.deepcopy(dict(CATALOGUE)) + for entry in catalogue["checks"]: + entry.pop("since_ruleset", None) + if entry["id"] == "TOOL-02" and since is not None: + entry["since_ruleset"] = since + return catalogue + + def converted(self, since: str | None, **provenance: Any) -> adapter.Conversion: + engine = er.report(TARGET, [er.check("TOOL-01", "PASS")]) + engine["provenance"].update(provenance) + engine["provenance"] = {k: v for k, v in engine["provenance"].items() if v is not None} + return adapter.convert(engine, catalogue=self.catalogue(since)) + + def test_a_check_newer_than_the_reports_ruleset_is_recorded_as_unknown_with_no_engine_verdict(self) -> None: + conversion = self.converted("2026-11-09.1", ruleset_revision="2026-09-28.13") + self.assertEqual([c["id"] for c in conversion.checks], ["TOOL-01", "TOOL-02"]) + added = conversion.checks[1] + self.assertEqual( + (added["verdict"], added["method"], added["summary"]), + ("UNKNOWN", "automated", adapter.NOT_IN_RULESET_SUMMARY), + ) + self.assertNotIn("engine_verdict", added) + self.assertEqual(added["evidence"], []) + self.assertEqual( + conversion.warnings, + [ + "TOOL-02: the engine's ruleset 2026-09-28.13 is older than the check (2026-11-09.1), so it is " + "recorded as UNKNOWN; scan again with a newer engine" + ], + ) + + def test_a_check_the_reports_ruleset_already_has_stays_missing(self) -> None: + for revision in ("2026-11-09.1", "2026-11-09.2", "2026-12-01.1"): + with self.subTest(revision=revision): + conversion = self.converted("2026-11-09.1", ruleset_revision=revision) + self.assertEqual(([c["id"] for c in conversion.checks], conversion.warnings), (["TOOL-01"], [])) + + def test_a_check_with_no_since_ruleset_stays_missing(self) -> None: + conversion = self.converted(None, ruleset_revision="2026-09-28.13") + self.assertEqual([c["id"] for c in conversion.checks], ["TOOL-01"]) + + def test_a_report_without_a_ruleset_revision_adds_nothing(self) -> None: + conversion = self.converted("2026-11-09.1", ruleset_revision=None) + self.assertEqual(([c["id"] for c in conversion.checks], conversion.warnings), (["TOOL-01"], [])) + + def test_a_check_the_report_lists_as_not_applicable_is_not_added_again(self) -> None: + engine = er.report(TARGET, [er.check("TOOL-01", "PASS")], not_applicable=["TOOL-02"]) + conversion = adapter.convert(engine, catalogue=self.catalogue("2026-11-09.1")) + self.assertEqual([c["verdict"] for c in conversion.checks], ["PASS", "NOT_APPLICABLE"]) + + def test_revisions_sort_by_date_then_by_number(self) -> None: + for earlier, later in (("2026-11-09.9", "2026-11-09.10"), ("2026-10-31.4", "2026-11-01.1")): + first, second = adapter.ruleset_order(earlier), adapter.ruleset_order(later) + assert first is not None and second is not None + self.assertLess(first, second) + for text in (None, "", "2026-11-09", "v1", 7): + with self.subTest(text=text): + self.assertIsNone(adapter.ruleset_order(text)) + + class EngineScoreTests(unittest.TestCase): def test_the_engines_score_is_read_from_its_summary(self) -> None: engine = er.report(TARGET, summary={"phase1_score": 0.7311, "blockers": 4}) From b0b2085997260dfbc9972e05bb9611b9149576d6 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 11:17:30 +0530 Subject: [PATCH 009/111] feat(dev): translate .mcp.json into each harness's MCP config dev/mcp_config.py reads the one .mcp.json and prints or merges the entry for Cursor, Gemini CLI, opencode (mcp.servers, local, command array) and Codex ([mcp_servers] TOML). Claude Code and VS Code read .mcp.json as it is, so it writes nothing for them. A merge never replaces a different entry. It also records the first-run answer per harness in .agentic-readiness/setup.json and checks whether each server's command is on PATH, printing the install routes (the upstream installer with --skip-config) without running them. Tests cover one format per harness, merges into existing JSON and TOML, the refusals, the record and the check. opencode v2.0.24 resolves the generated project file (checked with `opencode debug config`). Roadmap-Step: B2 Co-Authored-By: Claude Opus 5.5 --- dev/mcp_config.py | 326 ++++++++++++++++++++++++++++++ dev/tests/test_mcp_config_tool.py | 234 +++++++++++++++++++++ docs/reference/dev-tooling.md | 14 +- 3 files changed, 572 insertions(+), 2 deletions(-) create mode 100644 dev/mcp_config.py create mode 100644 dev/tests/test_mcp_config_tool.py diff --git a/dev/mcp_config.py b/dev/mcp_config.py new file mode 100644 index 00000000..697206db --- /dev/null +++ b/dev/mcp_config.py @@ -0,0 +1,326 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna + +"""Translate this repository's ``.mcp.json`` into each harness's own MCP config, and record the first-run answer. + + mcp_config.py show --harness [--scope project|user] print where the entry goes and the entry itself + mcp_config.py write --harness [--scope project|user] merge the entry into that file (needs consent) + mcp_config.py check is each server's command on PATH; how to install it + mcp_config.py status --harness has the first-run offer been answered for it + mcp_config.py record --harness --answer accepted|declined [--scope project|user] + +``.mcp.json`` is the one source (ADR 0006). Claude Code and VS Code read it as it is, so ``write`` has nothing to +do for them. For the other harnesses, ``write`` merges the entry and never replaces an entry that differs: it +stops and prints both. The answer goes into ``.agentic-readiness/setup.json``, which git ignores. Nothing here +installs a server or runs one, except ``check``, which runs `` --version`` for a command on ``PATH``. + +Exit 0 on success, 1 when ``write`` refuses or ``check`` finds a missing command, 2 on a usage or file error. +Standard library only; this file runs in this repository and is never shipped. +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import re +import shutil +import subprocess +import sys +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Sequence + +REPO = Path(__file__).resolve().parents[1] +SOURCE = ".mcp.json" +RECORD = Path(".agentic-readiness") / "setup.json" + +HARNESSES = ("claude-code", "vscode", "cursor", "gemini", "opencode", "codex") +SCOPES = ("project", "user") +ANSWERS = ("accepted", "declined") +BARE_KEY = re.compile(r"^[A-Za-z0-9_-]+$") + +#: Install routes from each server's own documentation. ``--skip-config`` stops the upstream installer from +#: writing agent configs for every client it finds; this tool writes the one entry the person agreed to. +INSTALL_HINTS: dict[str, tuple[str, ...]] = { + "codebase-memory-mcp": ( + "macOS or Linux: curl -fsSL https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/install.sh" + " | bash -s -- --skip-config", + "With Node: npm install -g codebase-memory-mcp (downloads the native binary)", + "Windows, and the manual route: https://github.com/DeusData/codebase-memory-mcp#quick-start", + ), +} + + +class ConfigError(Exception): + """A file cannot be read or parsed, or the request makes no sense for the harness.""" + + +class Conflict(Exception): + """The target file already holds a different entry under the same name.""" + + +@dataclass(frozen=True) +class Target: + """Where one harness keeps its MCP config, and in which format.""" + + path: Path | None + native: str = "" + + +def load_servers(root: Path) -> dict[str, dict[str, Any]]: + """The ``mcpServers`` map of ``/.mcp.json``.""" + path = root / SOURCE + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ConfigError(f"cannot read {path}: {exc}") from exc + servers = data.get("mcpServers") + if not isinstance(servers, dict) or not servers: + raise ConfigError(f"{path} has no mcpServers") + for name, entry in servers.items(): + if not isinstance(entry, dict) or not isinstance(entry.get("command"), str): + raise ConfigError(f"{path}: server {name!r} has no command") + return servers + + +def target(harness: str, scope: str, root: Path, home: Path) -> Target: + """The file ``harness`` reads at ``scope``. A native target has no path: the harness reads ``.mcp.json``.""" + if harness == "claude-code": + if scope == "project": + return Target(None, "Claude Code reads .mcp.json. On the next start it asks you to approve each server.") + return Target(None, "Run: claude mcp add --scope user -- [args]. Claude Code keeps it.") + if harness == "vscode": + if scope == "project": + return Target(None, "VS Code reads .mcp.json. It starts the servers once you trust the folder.") + return Target(None, "Run 'MCP: Open User Configuration' in VS Code and add the entry under 'servers'.") + paths = { + ("cursor", "project"): root / ".cursor" / "mcp.json", + ("cursor", "user"): home / ".cursor" / "mcp.json", + ("gemini", "project"): root / ".gemini" / "settings.json", + ("gemini", "user"): home / ".gemini" / "settings.json", + ("opencode", "project"): root / "opencode.json", + ("opencode", "user"): home / ".config" / "opencode" / "opencode.json", + ("codex", "project"): root / ".codex" / "config.toml", + ("codex", "user"): home / ".codex" / "config.toml", + } + try: + return Target(paths[(harness, scope)]) + except KeyError: + raise ConfigError(f"unknown harness or scope: {harness} {scope}") from None + + +def translate(harness: str, entry: dict[str, Any]) -> dict[str, Any]: + """One ``.mcp.json`` server entry in the shape ``harness`` expects.""" + command: str = entry["command"] + args: list[str] = list(entry.get("args", [])) + env: dict[str, str] = dict(entry.get("env", {})) + if harness == "opencode": + result: dict[str, Any] = {"type": "local", "command": [command, *args]} + if env: + result["environment"] = env + return result + result = {"type": "stdio", "command": command} if harness == "cursor" else {"command": command} + if args: + result["args"] = args + if env: + result["env"] = env + return result + + +def _json_path(harness: str) -> tuple[str, ...]: + """The keys that lead to the server map in a harness's JSON config.""" + return ("mcp", "servers") if harness == "opencode" else ("mcpServers",) + + +def render(harness: str, servers: dict[str, dict[str, Any]]) -> str: + """The config text for ``servers`` alone, as the person would paste it.""" + translated = {name: translate(harness, entry) for name, entry in servers.items()} + if harness == "codex": + return "".join(_toml_table(name, entry) for name, entry in translated.items()) + document: dict[str, Any] = translated + for key in reversed(_json_path(harness)): + document = {key: document} + return json.dumps(document, indent=2) + "\n" + + +def _toml_value(value: Any) -> str: + if isinstance(value, str): + return json.dumps(value) + if isinstance(value, list): + return "[" + ", ".join(_toml_value(item) for item in value) + "]" + if isinstance(value, dict): + return "{ " + ", ".join(f"{json.dumps(k)} = {_toml_value(v)}" for k, v in value.items()) + " }" + raise ConfigError(f"cannot write {value!r} as TOML") + + +def _toml_key(name: str) -> str: + return name if BARE_KEY.match(name) else json.dumps(name) + + +def _toml_table(name: str, entry: dict[str, Any]) -> str: + lines = [f"[mcp_servers.{_toml_key(name)}]"] + lines += [f"{key} = {_toml_value(value)}" for key, value in entry.items()] + return "\n".join(lines) + "\n" + + +def merge(harness: str, path: Path, servers: dict[str, dict[str, Any]]) -> list[str]: + """Add each server to ``path``; return the names added. Raises :class:`Conflict` before writing anything.""" + translated = {name: translate(harness, entry) for name, entry in servers.items()} + text = path.read_text(encoding="utf-8") if path.exists() else "" + if harness == "codex": + return _merge_toml(path, text, translated) + return _merge_json(harness, path, text, translated) + + +def _merge_json(harness: str, path: Path, text: str, translated: dict[str, dict[str, Any]]) -> list[str]: + try: + document = json.loads(text) if text.strip() else {} + except json.JSONDecodeError as exc: + raise ConfigError(f"{path} is not plain JSON ({exc}); paste the entry by hand") from exc + if not isinstance(document, dict): + raise ConfigError(f"{path} does not hold a JSON object") + if harness == "opencode" and not text.strip(): + document["$schema"] = "https://opencode.ai/config.json" + node = document + for key in _json_path(harness): + child = node.setdefault(key, {}) + if not isinstance(child, dict): + raise ConfigError(f"{path}: {key!r} is not an object") + node = child + added = _add(path, node, translated) + if added: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8") + return added + + +def _merge_toml(path: Path, text: str, translated: dict[str, dict[str, Any]]) -> list[str]: + try: + existing = tomllib.loads(text).get("mcp_servers", {}) + except tomllib.TOMLDecodeError as exc: + raise ConfigError(f"{path} is not valid TOML ({exc})") from exc + added = _add(path, dict(existing), translated) + if added: + block = "".join(_toml_table(name, translated[name]) for name in added) + separator = "" if not text or text.endswith("\n\n") else ("\n" if text.endswith("\n") else "\n\n") + result = text + separator + block + try: + tomllib.loads(result) + except tomllib.TOMLDecodeError as exc: + raise ConfigError(f"adding the entry would break {path} ({exc}); paste it by hand") from exc + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(result, encoding="utf-8") + return added + + +def _add(path: Path, node: dict[str, Any], translated: dict[str, dict[str, Any]]) -> list[str]: + for name, entry in translated.items(): + if name in node and node[name] != entry: + raise Conflict(f"{path} already has a different {name!r}:\n has: {node[name]}\n want: {entry}") + added = [name for name in translated if name not in node] + for name in added: + node[name] = translated[name] + return added + + +def read_record(root: Path) -> dict[str, Any]: + """The first-run answers, keyed by harness; empty when nothing was recorded.""" + path = root / RECORD + if not path.exists(): + return {} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ConfigError(f"cannot read {path}: {exc}") from exc + harnesses = data.get("harnesses", {}) if isinstance(data, dict) else {} + return harnesses if isinstance(harnesses, dict) else {} + + +def write_record(root: Path, harness: str, answer: str, scope: str, servers: list[str], day: datetime.date) -> None: + """Record one harness's answer, keeping the others.""" + harnesses = read_record(root) + harnesses[harness] = {"answer": answer, "date": day.isoformat(), "scope": scope, "servers": sorted(servers)} + path = root / RECORD + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"harnesses": harnesses}, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def check_commands(servers: dict[str, dict[str, Any]]) -> list[tuple[str, str | None, str]]: + """``(server, version line or None, note)`` for each server, from `` --version``.""" + results: list[tuple[str, str | None, str]] = [] + for name, entry in servers.items(): + found = shutil.which(entry["command"]) + if found is None: + results.append((name, None, "not on PATH")) + continue + try: + run = subprocess.run([found, "--version"], capture_output=True, text=True, timeout=15, check=False) + line = (run.stdout or run.stderr).strip().splitlines() + results.append((name, line[0] if line else "", found)) + except (OSError, subprocess.TimeoutExpired) as exc: + results.append((name, None, f"found at {found}, but --version failed: {exc}")) + return results + + +def main(argv: Sequence[str] | None = None, *, root: Path = REPO, home: Path | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="action", required=True) + for action in ("show", "write", "status", "record"): + command = sub.add_parser(action) + command.add_argument("--harness", choices=HARNESSES, required=True) + if action != "status": + command.add_argument("--scope", choices=SCOPES, default="project") + if action == "record": + command.add_argument("--answer", choices=ANSWERS, required=True) + sub.add_parser("check") + args = parser.parse_args(argv) + home = home if home is not None else Path.home() + + try: + if args.action == "status": + entry = read_record(root).get(args.harness) + print(f"asked: {entry['answer']} on {entry['date']}" if entry else "not asked") + return 0 + servers = load_servers(root) + if args.action == "check": + return _print_check(servers) + if args.action == "record": + write_record(root, args.harness, args.answer, args.scope, list(servers), datetime.date.today()) + print(f"recorded: {args.harness} {args.answer}") + return 0 + where = target(args.harness, args.scope, root, home) + if where.path is None: + print(where.native) + return 0 + if args.action == "show": + print(f"# {where.path}") + print(render(args.harness, servers), end="") + return 0 + added = merge(args.harness, where.path, servers) + print(f"{where.path}: added {', '.join(added)}" if added else f"{where.path}: already set, nothing to do") + return 0 + except Conflict as exc: + print(f"mcp_config: {exc}", file=sys.stderr) + return 1 + except (ConfigError, OSError) as exc: + print(f"mcp_config: {exc}", file=sys.stderr) + return 2 + + +def _print_check(servers: dict[str, dict[str, Any]]) -> int: + missing = False + for name, version, note in check_commands(servers): + if version is not None: + print(f"{name}: {version} ({note})") + continue + missing = True + print(f"{name}: {note}") + for hint in INSTALL_HINTS.get(name, ("See the server's own documentation.",)): + print(f" install: {hint}") + return 1 if missing else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev/tests/test_mcp_config_tool.py b/dev/tests/test_mcp_config_tool.py new file mode 100644 index 00000000..04570050 --- /dev/null +++ b/dev/tests/test_mcp_config_tool.py @@ -0,0 +1,234 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna + +"""Unit tests for ``mcp_config.py``: one translation per harness format, safe merges, the record and the check.""" + +from __future__ import annotations + +import contextlib +import io +import json +import os +import stat +import tempfile +import tomllib +import unittest +from pathlib import Path +from unittest import mock + +import _dev_paths # noqa: F401 (puts dev/ on sys.path) +import mcp_config + +SERVER: dict[str, object] = {"command": "codebase-memory-mcp"} +WITH_ARGS: dict[str, object] = {"command": "tool", "args": ["--stdio"], "env": {"MODE": "fast"}} + + +class Workspace(unittest.TestCase): + """A temporary repository root with a ``.mcp.json``, and a temporary home.""" + + servers: dict[str, dict[str, object]] = {"codebase-memory-mcp": SERVER} + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) / "repo" + self.home = Path(tmp.name) / "home" + self.root.mkdir() + self.home.mkdir() + (self.root / ".mcp.json").write_text(json.dumps({"mcpServers": self.servers}), encoding="utf-8") + + def run_tool(self, *argv: str) -> tuple[int, str, str]: + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = mcp_config.main(list(argv), root=self.root, home=self.home) + return code, out.getvalue(), err.getvalue() + + +class TranslatePerHarness(unittest.TestCase): + def test_cursor_uses_mcp_servers_with_stdio_type(self) -> None: + self.assertEqual( + json.loads(mcp_config.render("cursor", {"x": WITH_ARGS})), + {"mcpServers": {"x": {"type": "stdio", "command": "tool", "args": ["--stdio"], "env": {"MODE": "fast"}}}}, + ) + + def test_gemini_uses_mcp_servers(self) -> None: + self.assertEqual( + json.loads(mcp_config.render("gemini", {"x": WITH_ARGS})), + {"mcpServers": {"x": {"command": "tool", "args": ["--stdio"], "env": {"MODE": "fast"}}}}, + ) + + def test_opencode_uses_mcp_servers_local_with_command_array(self) -> None: + self.assertEqual( + json.loads(mcp_config.render("opencode", {"x": WITH_ARGS})), + { + "mcp": { + "servers": {"x": {"type": "local", "command": ["tool", "--stdio"], "environment": {"MODE": "fast"}}} + } + }, + ) + + def test_codex_writes_a_toml_table_that_parses_back(self) -> None: + text = mcp_config.render("codex", {"codebase-memory-mcp": SERVER, "x": WITH_ARGS}) + self.assertIn("[mcp_servers.codebase-memory-mcp]\n", text) + self.assertEqual( + tomllib.loads(text)["mcp_servers"], + {"codebase-memory-mcp": SERVER, "x": WITH_ARGS}, + ) + + def test_codex_quotes_a_key_that_is_not_bare(self) -> None: + text = mcp_config.render("codex", {"my server": SERVER}) + self.assertEqual(tomllib.loads(text)["mcp_servers"], {"my server": SERVER}) + + +class Targets(Workspace): + def test_native_harnesses_have_nothing_to_write(self) -> None: + for harness in ("claude-code", "vscode"): + for scope in ("project", "user"): + with self.subTest(harness=harness, scope=scope): + self.assertIsNone(mcp_config.target(harness, scope, self.root, self.home).path) + + def test_paths_per_harness_and_scope(self) -> None: + expected = { + ("cursor", "project"): self.root / ".cursor/mcp.json", + ("cursor", "user"): self.home / ".cursor/mcp.json", + ("gemini", "project"): self.root / ".gemini/settings.json", + ("gemini", "user"): self.home / ".gemini/settings.json", + ("opencode", "project"): self.root / "opencode.json", + ("opencode", "user"): self.home / ".config/opencode/opencode.json", + ("codex", "project"): self.root / ".codex/config.toml", + ("codex", "user"): self.home / ".codex/config.toml", + } + for (harness, scope), path in expected.items(): + with self.subTest(harness=harness, scope=scope): + self.assertEqual(mcp_config.target(harness, scope, self.root, self.home).path, path) + + def test_write_for_a_native_harness_writes_nothing(self) -> None: + code, out, _ = self.run_tool("write", "--harness", "claude-code") + self.assertEqual(code, 0) + self.assertIn("approve", out) + self.assertEqual(sorted(p.name for p in self.root.iterdir()), [".mcp.json"]) + + def test_show_prints_the_path_and_entry_without_writing(self) -> None: + code, out, _ = self.run_tool("show", "--harness", "gemini") + self.assertEqual(code, 0) + self.assertIn(".gemini/settings.json", out) + self.assertIn('"codebase-memory-mcp"', out) + self.assertFalse((self.root / ".gemini").exists()) + + +class WriteMerges(Workspace): + def test_creates_each_file_when_absent(self) -> None: + for harness in ("cursor", "gemini", "opencode", "codex"): + with self.subTest(harness=harness): + code, out, _ = self.run_tool("write", "--harness", harness) + self.assertEqual(code, 0) + self.assertIn("added codebase-memory-mcp", out) + self.assertEqual( + json.loads((self.root / "opencode.json").read_text())["$schema"], "https://opencode.ai/config.json" + ) + self.assertIn( + "codebase-memory-mcp", tomllib.loads((self.root / ".codex/config.toml").read_text())["mcp_servers"] + ) + + def test_keeps_other_settings_and_servers(self) -> None: + path = self.root / ".gemini" / "settings.json" + path.parent.mkdir() + path.write_text(json.dumps({"theme": "dark", "mcpServers": {"other": {"command": "o"}}}), encoding="utf-8") + self.assertEqual(self.run_tool("write", "--harness", "gemini")[0], 0) + data = json.loads(path.read_text()) + self.assertEqual(data["theme"], "dark") + self.assertEqual(set(data["mcpServers"]), {"other", "codebase-memory-mcp"}) + + def test_appends_to_an_existing_toml_file(self) -> None: + path = self.home / ".codex" / "config.toml" + path.parent.mkdir() + path.write_text('model = "x"\n\n[mcp_servers.other]\ncommand = "o"\n', encoding="utf-8") + self.assertEqual(self.run_tool("write", "--harness", "codex", "--scope", "user")[0], 0) + data = tomllib.loads(path.read_text()) + self.assertEqual(data["model"], "x") + self.assertEqual(set(data["mcp_servers"]), {"other", "codebase-memory-mcp"}) + + def test_second_write_changes_nothing(self) -> None: + self.run_tool("write", "--harness", "cursor") + before = (self.root / ".cursor/mcp.json").read_text() + code, out, _ = self.run_tool("write", "--harness", "cursor") + self.assertEqual(code, 0) + self.assertIn("already set", out) + self.assertEqual((self.root / ".cursor/mcp.json").read_text(), before) + + def test_refuses_to_replace_a_different_entry(self) -> None: + for harness, path, text in ( + ("cursor", ".cursor/mcp.json", json.dumps({"mcpServers": {"codebase-memory-mcp": {"command": "/opt/x"}}})), + ("codex", ".codex/config.toml", '[mcp_servers.codebase-memory-mcp]\ncommand = "/opt/x"\n'), + ): + with self.subTest(harness=harness): + file = self.root / path + file.parent.mkdir(exist_ok=True) + file.write_text(text, encoding="utf-8") + code, _, err = self.run_tool("write", "--harness", harness) + self.assertEqual(code, 1) + self.assertIn("already has a different", err) + self.assertEqual(file.read_text(), text) + + def test_refuses_a_file_that_is_not_plain_json(self) -> None: + path = self.root / "opencode.json" + path.write_text("{ // a comment\n}", encoding="utf-8") + code, _, err = self.run_tool("write", "--harness", "opencode") + self.assertEqual(code, 2) + self.assertIn("paste the entry by hand", err) + + +class FirstRunRecord(Workspace): + def test_status_is_not_asked_until_recorded(self) -> None: + self.assertEqual(self.run_tool("status", "--harness", "opencode")[1].strip(), "not asked") + + def test_record_keeps_one_answer_per_harness(self) -> None: + self.run_tool("record", "--harness", "opencode", "--answer", "accepted") + self.run_tool("record", "--harness", "claude-code", "--answer", "declined") + self.assertTrue(self.run_tool("status", "--harness", "opencode")[1].startswith("asked: accepted on ")) + self.assertTrue(self.run_tool("status", "--harness", "claude-code")[1].startswith("asked: declined on ")) + data = json.loads((self.root / ".agentic-readiness/setup.json").read_text()) + self.assertEqual(data["harnesses"]["opencode"]["servers"], ["codebase-memory-mcp"]) + self.assertEqual(data["harnesses"]["opencode"]["scope"], "project") + + +class CheckCommands(Workspace): + def test_missing_command_prints_install_routes_without_running_them(self) -> None: + with ( + mock.patch("mcp_config.shutil.which", return_value=None), + mock.patch("mcp_config.subprocess.run") as run, + ): + code, out, _ = self.run_tool("check") + self.assertEqual(code, 1) + self.assertIn("not on PATH", out) + self.assertIn("--skip-config", out) + run.assert_not_called() + + def test_present_command_reports_its_version(self) -> None: + bin_dir = self.root.parent / "bin" + bin_dir.mkdir() + script = bin_dir / "codebase-memory-mcp" + script.write_text("#!/bin/sh\necho 'codebase-memory-mcp 9.9.9'\n", encoding="utf-8") + script.chmod(script.stat().st_mode | stat.S_IEXEC) + with mock.patch.dict(os.environ, {"PATH": str(bin_dir)}): + code, out, _ = self.run_tool("check") + self.assertEqual(code, 0) + self.assertIn("codebase-memory-mcp 9.9.9", out) + + +class SourceErrors(Workspace): + def test_missing_source_is_a_usage_error(self) -> None: + (self.root / ".mcp.json").unlink() + self.assertEqual(self.run_tool("show", "--harness", "cursor")[0], 2) + + +class RepositorySource(unittest.TestCase): + def test_this_repository_translates_for_every_harness(self) -> None: + servers = mcp_config.load_servers(mcp_config.REPO) + for harness in ("cursor", "gemini", "opencode", "codex"): + with self.subTest(harness=harness): + self.assertIn("codebase-memory-mcp", mcp_config.render(harness, servers)) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 0e0c5680..b7d5df6a 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -99,8 +99,18 @@ How each harness treats a project MCP file (checked 2026-11-09 in each harness's | opencode | `opencode.json` (`mcp`) | Connects it with no prompt | | Codex CLI | `.codex/config.toml` (`[mcp_servers]`) | Loads it only for a trusted project | -For a harness that does not read `.mcp.json`, copy the entry into its own file, outside the repository or -uncommitted. If the server is not installed, it fails to start and the agent reads files instead. +For a harness that does not read `.mcp.json`, `dev/mcp_config.py` translates the entry into that harness's format: + +| Command | What it does | +| --- | --- | +| `python3 dev/mcp_config.py show --harness [--scope project\|user]` | Prints the target file and the entry. Writes nothing | +| `python3 dev/mcp_config.py write --harness [--scope project\|user]` | Adds the entry to the file. Stops if a different entry has the same name | +| `python3 dev/mcp_config.py check` | Is each server's command on `PATH`? Prints its version, or the install routes | +| `python3 dev/mcp_config.py status --harness ` | Has the person answered the setup offer for this harness? | +| `python3 dev/mcp_config.py record --harness --answer accepted\|declined` | Records the answer in `.agentic-readiness/setup.json` | + +The names are `claude-code`, `vscode`, `cursor`, `gemini`, `opencode` and `codex`. The tool installs nothing. If +the server is not installed, it fails to start and the agent reads files instead. ## Settings on GitHub From 6dce4b176ab53064d88e4a7eb237fe2947cc2f63 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 11:23:00 +0530 Subject: [PATCH 010/111] feat(dev): offer each contributor a one-time agent tool setup AGENTS.md now tells the agent, before its first change in a clone, to run `dev/mcp_config.py status` and, if the offer was never answered, to make it once: explain codebase-memory-mcp, ask, and with consent write the entry for the person's harness at project scope. Nothing is installed by the agent; the install routes carry --skip-config. No offer for audit or implement runs on another repository. docs/how-to/set-up-agent-tools.md holds the person's steps and the agent's procedure. ADR 0006 records the design. Git ignores the first-run record and the per-harness copies, so .mcp.json stays the one committed source. Roadmap-Step: B2 Co-Authored-By: Claude Opus 5.5 --- .gitignore | 10 ++ AGENTS.md | 6 + Makefile | 2 +- docs/README.md | 1 + .../adrs/0006-first-run-agent-tool-offer.md | 96 +++++++++++++ docs/architecture/adrs/README.md | 1 + docs/how-to/set-up-agent-tools.md | 135 ++++++++++++++++++ 7 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 docs/architecture/adrs/0006-first-run-agent-tool-offer.md create mode 100644 docs/how-to/set-up-agent-tools.md diff --git a/.gitignore b/.gitignore index 716406b2..e6f0f3f4 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,13 @@ id_rsa* .DS_Store Thumbs.db *.swp + +# agent tools: the first-run answer and the per-harness MCP configs that dev/mcp_config.py writes. `.mcp.json` is +# the one committed source (ADR 0006); these are each person's own copies. +/.agentic-readiness/ +/opencode.json +/opencode.jsonc +/.cursor/ +/.gemini/ +/.codex/ +/.vscode/mcp.json diff --git a/AGENTS.md b/AGENTS.md index 0a6e7a14..f7d3edeb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,12 @@ folder before you change it. The rest of this file is for changes to this repository. +## First session: offer the agent tools + +Before your first change in a clone, run `python3 dev/mcp_config.py status --harness `. If it prints +`not asked`, make the one-time offer in `docs/how-to/set-up-agent-tools.md` ("For agents"). Ask once. Never install a +server or write a harness config without the person's consent. + ## Commands The framework's seven verbs run through the root `Makefile`. CI runs the same commands. diff --git a/Makefile b/Makefile index c90f0be0..98b0000c 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ COVERAGE_DATA := $(CURDIR)/dev/dist/coverage/.coverage # The Markdown that `docs-lint` holds to zero violations. A ratchet: a path joins once it is clean, and the list # only grows. Everything else is reported by `docs-lint-all` and never fails. DOCS_LINT_PATHS := 'implement/SKILL.md' 'implement/state.md' 'implement/playbooks/*.md' 'implement/prompts/*.md' \ - 'docs/README.md' 'docs/STYLE_GUIDE.md' 'docs/architecture/**/*.md' 'docs/reference/branch-protection.md' 'docs/reference/dev-tooling.md' 'docs/how-to/contribute-a-change.md' 'docs/COMMUNICATION_RULES.md' 'CONTRIBUTING.md' 'CHANGELOG.md' 'RELEASE_PROCESS.md' 'SECURITY.md' \ + 'docs/README.md' 'docs/STYLE_GUIDE.md' 'docs/architecture/**/*.md' 'docs/reference/branch-protection.md' 'docs/reference/dev-tooling.md' 'docs/how-to/contribute-a-change.md' 'docs/how-to/set-up-agent-tools.md' 'docs/COMMUNICATION_RULES.md' 'CONTRIBUTING.md' 'CHANGELOG.md' 'RELEASE_PROCESS.md' 'SECURITY.md' \ 'docs/explanation/user-overview/03-trust-and-safety.md' 'docs/explanation/contributor-deep-dive/04-stack-profiles.md' 'docs/explanation/contributor-deep-dive/05-evaluation.md' 'docs/how-to/add-a-stack-profile.md' 'docs/reference/harness-notes.md' \ 'audit/engine/docs/how-to/read-the-report.md' diff --git a/docs/README.md b/docs/README.md index 52d8b175..59906a35 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,6 +20,7 @@ output to ``". To fix the findings, say "Start implementing the fix | Page | What it does | | --- | --- | | [Contribute a change](how-to/contribute-a-change.md) | Move one change from a work branch into `dev` through a pull request | +| [Set up the agent tools](how-to/set-up-agent-tools.md) | Give your coding agent the code index, or skip it; the one-time offer agents make | | [Install the skills](how-to/install-the-skills.md) | Put both skills from a release into a project, with no clone | | [Run an audit](how-to/run-an-audit.md) | Produce a confirmed report through the agent and the audit recipe | | [Implement an audit report](how-to/implement-an-audit-report.md) | Apply the confirmed findings, one theme per branch | diff --git a/docs/architecture/adrs/0006-first-run-agent-tool-offer.md b/docs/architecture/adrs/0006-first-run-agent-tool-offer.md new file mode 100644 index 00000000..02986a94 --- /dev/null +++ b/docs/architecture/adrs/0006-first-run-agent-tool-offer.md @@ -0,0 +1,96 @@ +# 0006. One Committed `.mcp.json`; Agents Offer Each Person a One-Time Setup for Their Harness + +* **Status:** Accepted +* **Date:** 2026-11-09 +* **Last reviewed:** 2026-11-09 +* **Deciders:** [@nistapp] + +--- + +## Context + +This repository asks target repositories to register the code index, `codebase-memory-mcp` (`IDX-01`, +`REQ-SH-001`). Until v0.3.3 it did not register the index for itself, and its own self-audit showed that. + +Six harnesses are in use. Each reads a different project file, and they act differently on the first start +(checked 2026-11-09 in each harness's own documentation; the table is in +[Development tooling](../../reference/dev-tooling.md#agent-tools-mcp)): + +* Claude Code reads `.mcp.json` and asks each person to approve each server. +* VS Code with Copilot also reads a root `.mcp.json`, and starts it in a trusted folder. +* Cursor (`.cursor/mcp.json`), Gemini CLI (`.gemini/settings.json`), opencode (`opencode.json`) and Codex CLI + (`.codex/config.toml`) do not read `.mcp.json`. +* opencode connects a project server with no prompt. + +An MCP server is a program that runs on the person's machine. Many people clone this repository only to audit +their own code; they do not need the server here. The owner decided on one committed file, not one per harness. + +--- + +## Decision + +1. **One committed source.** [`.mcp.json`](../../../.mcp.json) lists `codebase-memory-mcp` with its bare command: + no path, no version and no package runner. Each person installs the server and keeps its version. The + one-place pin rule (ADR 0003, `dev/tests/test_tool_pins.py`) needs no exception. +2. **Approval stays with each person.** The committed `.claude/settings.json` never sets + `enableAllProjectMcpServers` or `enabledMcpjsonServers`. `dev/tests/test_mcp_config.py` checks this. +3. **A one-time offer, from `AGENTS.md`.** Every harness reads `AGENTS.md`. Before the first change in a clone, + the agent runs `python3 dev/mcp_config.py status --harness `. If it prints `not asked`, the agent makes + the offer in [Set up the agent tools](../../how-to/set-up-agent-tools.md). It makes no offer for an audit or + an implement run on another repository. +4. **The agent names its harness.** It uses its own knowledge, confirmed by an environment variable where one is + known (`CLAUDECODE=1`). If it is not sure, it asks. The tool takes `--harness`; it does not guess. +5. **The answer is recorded per harness, outside git.** `.agentic-readiness/setup.json` is git-ignored. A + declined offer is recorded too, so the agent asks once per harness and clone. +6. **Project scope by default.** [`dev/mcp_config.py`](../../../dev/mcp_config.py) translates `.mcp.json` into each + harness's format and merges it into the project file, or the user file with `--scope user`. Git ignores the + project files it writes. It never replaces a different entry with the same name. For Claude Code and VS Code it + writes nothing and explains the approval. +7. **Nothing is installed by the tool or the agent.** `check` runs ` --version` for a command on `PATH`. + For a missing command it prints the upstream install routes. They carry `--skip-config`, so the installer + writes no config into other harnesses. The person runs the installer. +8. **No gate depends on MCP.** Anything that must be right runs through `make`. + +--- + +## Alternatives Considered + +| Alternative | Why it was rejected | +|---|---| +| A committed config file for each harness | The owner chose one file. Six copies drift, and opencode would start a committed server with no prompt | +| `npx -y codebase-memory-mcp@` in `.mcp.json` | The package downloads a native binary when it runs. The version would need a second place or a generator. It can differ from the version that built the person's index | +| A dedicated skill for the offer | Not every harness reads skills. Every harness reads `AGENTS.md` | +| Detect the harness from environment variables only | Only Claude Code's variable is confirmed. A wrong guess writes the wrong file | +| Record the answer in a tracked file | One person's answer would apply to every clone | +| Pre-approve the server in `.claude/settings.json` | It would start a program on every contributor's machine without asking | +| No offer: document the setup only | The setup is easy to miss, and agents then read whole files where a graph query would do | + +--- + +## Consequences + +### Positive + +* The self-audit passes `IDX-01`. +* A contributor gets the index in their own harness with one question, and is asked only once. +* `.mcp.json` stays the one source; the per-harness files are local copies. + +### Negative / Trade-offs + +* The offer depends on the agent following `AGENTS.md`; nothing enforces it. +* The translation follows each harness's current format (opencode `mcp.servers`, checked with opencode v2.0.24). A + harness that changes its format needs a change in `dev/mcp_config.py`. +* Each person installs a binary to get the benefit. + +--- + +## Lifecycle — Living, Current-State ADRs + +* **This ADR MUST describe only the shipped, current design.** There is no `Deprecated`, `Superseded`, or `Rejected` status. +* **Decision changed?** Overwrite the body, keep the file and its number, and fold the reversal into + *Alternatives Considered*. Do NOT create a superseding ADR and do NOT leave a tombstone stub. +* **Decision no longer relevant?** Delete the file, retire its number forever, and update every inbound reference + plus the index table in `docs/architecture/adrs/README.md` in the same change set. +* Archived wording: `git log --follow docs/architecture/adrs/0006-first-run-agent-tool-offer.md`. + +See [`docs/STYLE_GUIDE.md`](../../STYLE_GUIDE.md) § ADR Lifecycle for the canonical rules. diff --git a/docs/architecture/adrs/README.md b/docs/architecture/adrs/README.md index 447f32fc..38b821b5 100644 --- a/docs/architecture/adrs/README.md +++ b/docs/architecture/adrs/README.md @@ -13,6 +13,7 @@ The root router is [`docs/README.md`](../../README.md). | [0003](0003-dependabot-security-pull-requests-only.md) | Dependabot Opens Security Pull Requests Only; Major Versions Are Advisory | Accepted | 2026-11-08 | | [0004](0004-plain-language-rules-and-a-prose-check.md) | Plain-Language Rules in One File, Checked by a Standard-Library Prose Check | Accepted | 2026-11-08 | | [0005](0005-release-please-cuts-releases.md) | Release-Please Computes the Version, Writes the Changelog and Tags the Release | Accepted | 2026-11-08 | +| [0006](0006-first-run-agent-tool-offer.md) | One Committed `.mcp.json`; Agents Offer Each Person a One-Time Setup for Their Harness | Accepted | 2026-11-09 | ## Requirements: `spec/decisions/` diff --git a/docs/how-to/set-up-agent-tools.md b/docs/how-to/set-up-agent-tools.md new file mode 100644 index 00000000..2a9366a1 --- /dev/null +++ b/docs/how-to/set-up-agent-tools.md @@ -0,0 +1,135 @@ +# How to set up the agent tools + +> **Goal:** Give your coding agent the code index (`codebase-memory-mcp`) in this repository, or decide to skip it. +> **Audience:** Contributors, interns included, and AI coding agents. You know basic shell commands. +> **Status:** Live. + +--- + +An agent tool is an MCP server: a program that your coding agent (the harness) starts and asks questions. This +repository lists one, `codebase-memory-mcp`, in `.mcp.json`. It gives the agent a graph of the code: functions, +callers and dependencies. The agent then reads fewer whole files. + +The tool is optional. No gate, CI job or release step needs it. If you skip it, the agent searches and reads files +instead, and every check still gives the same result. The rules are in +[ADR 0006](../architecture/adrs/0006-first-run-agent-tool-offer.md). + +This page uses these terms: + +| Term | Meaning | +| --- | --- | +| Harness | The program that runs the agent: Claude Code, VS Code with Copilot, Cursor, Gemini CLI, opencode or Codex CLI | +| MCP server | A program that the harness starts to give the agent extra tools | +| Project scope | The setting applies to this clone only | +| User scope | The setting applies to every repository you open with that harness | + +## Prerequisites + +- Python 3.11 or newer on `PATH`. +- A clone of this repository. + +## Steps + +```mermaid +flowchart TD + A["1. Check the server is installed"] -->|"not installed"| B["2. Install it, or skip"] + A -->|"installed"| C["3. Register it for your harness"] + B -->|"installed"| C + B -->|"skip"| E["5. Record your answer"] + C --> D["4. Restart the harness and approve"] + D --> E +``` + +### Step 1: Check that the server is installed + +```bash +python3 dev/mcp_config.py check +``` + +The output gives the version, or `not on PATH` and the install routes. + +### Step 2: Install the server, or skip it + +Install it only if you want it. These are the routes from the server's own documentation: + +| System | Command | +| --- | --- | +| macOS or Linux | `curl -fsSL https://raw.githubusercontent.com/DeusData/codebase-memory-mcp/main/install.sh \| bash -s -- --skip-config` | +| Any, with Node | `npm install -g codebase-memory-mcp` | +| Windows | Follow the [quick start](https://github.com/DeusData/codebase-memory-mcp#quick-start) | + +Keep `--skip-config`. Without it, the installer also writes its own settings into every harness it finds. With it, +you install the program only, and step 3 adds the one entry you want. + +To skip the tool, go to step 5. + +### Step 3: Register the server for your harness + +Claude Code and VS Code read `.mcp.json` directly. For them, there is nothing to write. + +For another harness, look at the entry first, then write it: + +```bash +python3 dev/mcp_config.py show --harness opencode +python3 dev/mcp_config.py write --harness opencode +``` + +The names are `cursor`, `gemini`, `opencode` and `codex`. The default is project scope. Add `--scope user` to +register the server for every repository. Git ignores the project files that the tool writes +(`opencode.json`, `.cursor/`, `.gemini/`, `.codex/`), so they never go into a commit. If the file already has a +different entry with the same name, the tool stops and changes nothing. + +### Step 4: Restart the harness and approve the server + +| Harness | What you do | +| --- | --- | +| Claude Code | Approve `codebase-memory-mcp` when Claude Code asks. `/mcp` shows its state | +| VS Code with Copilot | Trust the folder. The server starts with no other prompt | +| Cursor | Turn the server on in the MCP settings | +| Gemini CLI | Trust the folder when Gemini CLI asks | +| opencode | Nothing. opencode connects the server with no prompt | +| Codex CLI | Trust the project. Codex loads `.codex/config.toml` only for a trusted project | + +### Step 5: Record your answer + +```bash +python3 dev/mcp_config.py record --harness opencode --answer accepted +``` + +Use `--answer declined` when you skip. The answer goes into `.agentic-readiness/setup.json`, which git ignores. +The agent then does not ask again for that harness. + +## Verification + +```bash +python3 dev/mcp_config.py status --harness opencode +``` + +The output is `asked: accepted on ` or `asked: declined on `. In the harness, the server shows as +connected (for example in Claude Code's `/mcp` list). + +## Undo + +- Remove the entry from the harness's file, or delete the git-ignored project file. +- In Claude Code, run `claude mcp reset-project-choices` to be asked to approve again. +- Delete `.agentic-readiness/setup.json` to get the offer again. + +## For agents: the first-run offer + +`AGENTS.md` sends you here when you start a change to this repository. Do not make the offer when the person only +wants to audit or implement another repository. + +1. Name your harness. Use your own knowledge first. `CLAUDECODE=1` in the environment confirms Claude Code. If + you are not sure, ask the person. +2. Run `python3 dev/mcp_config.py status --harness `. If it prints `asked: ...`, stop here. +3. Run `python3 dev/mcp_config.py check`. +4. Explain the server in two sentences: what it gives you and what it needs. Then ask one question: set it up now, + or not now. Name the scope: project scope by default. +5. If the person says no, record `--answer declined` and continue with their task. +6. If the person says yes: + - When the server is not installed, show the install routes from step 2. Never run an installer yourself. + - Run `show`, then `write` for the person's harness and scope. For Claude Code and VS Code, explain the + approval instead (step 4). + - Run `check` again, and tell the person to restart the harness. + - Record `--answer accepted`. +7. Ask once. Make the offer again only when the person asks for it. From a86ddab29fcfa308597ab2c2ff46759ea6582343 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 11:37:26 +0530 Subject: [PATCH 011/111] feat(engine): add CI-05, workflow files are linted An agent cannot run a workflow locally, so a provider-aware linter is its only check before a push. CI-05 (Phase 1, DEGRADER, REQ-SH-057) looks for one per CI provider: actionlint, zizmor or check-jsonschema for GitHub; check-jsonschema schemas, glab ci lint or circleci config validate for the others. Run from a local surface (runner, build file, hook, script) it passes; only in CI it is PARTIAL; none is FAIL; no workflow or Jenkins only is UNKNOWN. Lockfiles, Dependabot/Renovate configs and docs never count. Ruleset 2026-11-09.1, catalogue 0.3.0 with since_ruleset on CI-05, so an older engine's report imports CI-05 as UNKNOWN. Two one-defect fixtures; the other CI fixtures gain a workflow linter so each keeps one defect. The greenfield template lints its workflows (lint:workflows, actionlint in CI), so the generated scaffold still passes. The realistic drafts, the examples, the example implement state and the CI eval expectations are regenerated; implement theme T4 owns CI-05. Roadmap-Step: C1.2 Co-Authored-By: Claude Opus 5.5 --- .secrets.baseline | 41 ++-- audit/catalogue.json | 22 +- audit/checks/06-ci.md | 40 +++- audit/engine/audit/__init__.py | 2 +- audit/engine/audit/rules/checks/ci.py | 192 +++++++++++++++++- audit/engine/audit/rules/registry.py | 1 + .../02-check-catalogue.md | 10 +- .../user-overview/03-why-these-checks.md | 1 + audit/engine/docs/reference/changelog.md | 1 + audit/engine/tests/fixtures/README.md | 4 +- .../fixtures/ci-01-no-pr-trigger/package.json | 3 +- .../fixtures/ci-02-parity-break/package.json | 3 +- .../fixtures/ci-03-ci-only-step/package.json | 3 +- .../.github/workflows/ci.yml | 15 ++ .../fixtures/ci-05-ci-only-lint/package.json | 14 ++ .../.github/workflows/ci.yml | 14 ++ .../ci-05-no-workflow-lint/package.json | 14 ++ .../tests/fixtures/ci-ok-gh/package.json | 3 +- audit/engine/tests/test_ci_pack.py | 93 ++++++++- audit/engine/tests/test_fixture_matrix.py | 4 +- audit/engine/tests/test_hyg_pack.py | 4 +- audit/engine/tests/test_registry_status.py | 6 +- audit/engine/tests/test_smoke.py | 4 +- ...service.audit-report.20261010T094207Z.json | 2 +- ...e-service.audit-report.20261010T094207Z.md | 2 +- audit/schema/examples/minimal.json | 2 +- audit/schema/examples/minimal.md | 2 +- ...ess-state.example-service.in-progress.json | 4 +- ...iness-state.example-service.in-progress.md | 2 +- .../readiness-state.example-service.json | 4 +- .../readiness-state.example-service.md | 2 +- ...ntic-tdd.audit-draft.20261010T065738Z.json | 20 +- ...tant-sdk.audit-draft.20261010T065739Z.json | 20 +- ...t-server.audit-draft.20261010T065739Z.json | 20 +- audit/tools/tests/_engine_reports.py | 11 +- audit/tools/tests/test_import_engine.py | 34 +++- .../tools/tests/test_import_fixture_matrix.py | 4 +- audit/tools/tests/test_render_report.py | 6 +- audit/tools/tests/test_report.py | 31 +-- audit/tools/tests/test_summary.py | 8 +- audit/tools/tests/test_validate_report.py | 2 +- .../fixtures/ci-01-no-pr-trigger.json | 4 +- .../expected/fixtures/ci-02-parity-break.json | 4 +- .../expected/fixtures/ci-03-ci-only-step.json | 4 +- .../expected/fixtures/ci-05-ci-only-lint.json | 14 ++ .../fixtures/ci-05-no-workflow-lint.json | 14 ++ dev/eval/expected/fixtures/ci-ok-gh.json | 10 +- .../03-check-catalogue-and-rubrics.md | 2 +- .../how-the-checks-map-to-the-framework.md | 5 +- greenfield/README.md | 7 +- greenfield/templates/github/workflows/ci.yml | 5 + greenfield/templates/package.json | 1 + implement/playbooks/README.md | 2 +- implement/tools/progress.py | 2 +- spec/readiness-requirements.md | 10 + spec/traceability.md | 7 +- 56 files changed, 641 insertions(+), 120 deletions(-) create mode 100644 audit/engine/tests/fixtures/ci-05-ci-only-lint/.github/workflows/ci.yml create mode 100644 audit/engine/tests/fixtures/ci-05-ci-only-lint/package.json create mode 100644 audit/engine/tests/fixtures/ci-05-no-workflow-lint/.github/workflows/ci.yml create mode 100644 audit/engine/tests/fixtures/ci-05-no-workflow-lint/package.json create mode 100644 dev/eval/expected/fixtures/ci-05-ci-only-lint.json create mode 100644 dev/eval/expected/fixtures/ci-05-no-workflow-lint.json diff --git a/.secrets.baseline b/.secrets.baseline index dae87024..ae38561d 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -143,15 +143,6 @@ "line_number": 1540 } ], - "audit/catalogue.json": [ - { - "type": "Hex High Entropy String", - "filename": "audit/catalogue.json", - "hashed_secret": "2a99b730d632f7323b715d4b822bd2c2f2a6d60b", - "is_verified": false, - "line_number": 3 - } - ], "audit/engine/audit/rules/checks/con.py": [ { "type": "Secret Keyword", @@ -279,7 +270,7 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/readiness-state.example-service.in-progress.json", - "hashed_secret": "3225e9a0761733fb23cd5fd7dcaedf99abcb4085", + "hashed_secret": "9ba66d1f640d58e98891d9e7f69511c089f914e4", "is_verified": false, "line_number": 17 } @@ -288,7 +279,7 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/readiness-state.example-service.json", - "hashed_secret": "3225e9a0761733fb23cd5fd7dcaedf99abcb4085", + "hashed_secret": "9ba66d1f640d58e98891d9e7f69511c089f914e4", "is_verified": false, "line_number": 17 } @@ -749,7 +740,7 @@ { "type": "Hex High Entropy String", "filename": "dev/eval/expected/fixtures/ci-01-no-pr-trigger.json", - "hashed_secret": "dff0ffd7fbf36a165a5d8b4f0f6c6574f8e32deb", + "hashed_secret": "a34f7ff763127c41c659a8bc1b180f7ea91f14b0", "is_verified": false, "line_number": 4 } @@ -758,7 +749,7 @@ { "type": "Hex High Entropy String", "filename": "dev/eval/expected/fixtures/ci-02-parity-break.json", - "hashed_secret": "01974a5a6539c481fbfc23c0770bb71b0162364e", + "hashed_secret": "2111573a17af29924dc1bce2984de2611f11f7ef", "is_verified": false, "line_number": 4 } @@ -767,7 +758,25 @@ { "type": "Hex High Entropy String", "filename": "dev/eval/expected/fixtures/ci-03-ci-only-step.json", - "hashed_secret": "5afc046e4a38dce15b12f7c56fa92b6636fdbd3a", + "hashed_secret": "2a04cebada85f97678851bfac852408695405dc9", + "is_verified": false, + "line_number": 4 + } + ], + "dev/eval/expected/fixtures/ci-05-ci-only-lint.json": [ + { + "type": "Hex High Entropy String", + "filename": "dev/eval/expected/fixtures/ci-05-ci-only-lint.json", + "hashed_secret": "eb01d9022d33091ae612125118bd068803b71b7a", + "is_verified": false, + "line_number": 4 + } + ], + "dev/eval/expected/fixtures/ci-05-no-workflow-lint.json": [ + { + "type": "Hex High Entropy String", + "filename": "dev/eval/expected/fixtures/ci-05-no-workflow-lint.json", + "hashed_secret": "9286073d9a5530c003ae143da06089dc49687840", "is_verified": false, "line_number": 4 } @@ -776,7 +785,7 @@ { "type": "Hex High Entropy String", "filename": "dev/eval/expected/fixtures/ci-ok-gh.json", - "hashed_secret": "f52d3733c0e78ade949a63df9d00d718d76ee394", + "hashed_secret": "9e9463a6230b1b90241a4abf629cfc820d368289", "is_verified": false, "line_number": 4 } @@ -1449,5 +1458,5 @@ } ] }, - "generated_at": "2026-11-08T11:27:15Z" + "generated_at": "2026-11-09T05:58:50Z" } diff --git a/audit/catalogue.json b/audit/catalogue.json index 0b2b347b..d7446283 100644 --- a/audit/catalogue.json +++ b/audit/catalogue.json @@ -1,6 +1,6 @@ { - "catalogue_version": "0.2.8", - "fingerprint": "e33c95eda9f58b9d", + "catalogue_version": "0.3.0", + "fingerprint": "214aa1d1a94addb2", "checks": [ { "id": "IDX-01", @@ -546,6 +546,24 @@ "add-ci-parity" ] }, + { + "id": "CI-05", + "family": "CI", + "title": "Workflow files are linted", + "phase": 1, + "severity": "DEGRADER", + "scored": true, + "method": "automated", + "status": "implemented", + "req_ids": [ + "REQ-SH-057" + ], + "engine_id": "CI-05", + "since_ruleset": "2026-11-09.1", + "playbooks": [ + "add-ci-parity" + ] + }, { "id": "BASE-01", "family": "BASE", diff --git a/audit/checks/06-ci.md b/audit/checks/06-ci.md index 91788a7c..742ed176 100644 --- a/audit/checks/06-ci.md +++ b/audit/checks/06-ci.md @@ -1,7 +1,7 @@ # Continuous Integration (`CI`) -> Follows the anatomy in [`README.md`](README.md). Four checks: `CI-01` to `CI-03` are `method: -> automated`; `CI-04` is `method: agent` (no engine ID). See +> Follows the anatomy in [`README.md`](README.md). Five checks: `CI-01` to `CI-03` and `CI-05` are +> `method: automated`; `CI-04` is `method: agent` (no engine ID). See > [Why These Checks §7](../engine/docs/architecture/user-overview/03-why-these-checks.md#7-ci-ci) > for the consequence behind this family: the gate an agent *can* run locally must be the gate that > *decides*, or CI is a second, secret command surface nobody can reproduce. @@ -183,3 +183,39 @@ Pitfalls A workflow that passes every verb genuinely but is not wired in so `PASS` needs the branch-protection confirmation, not just a clean read of the workflow file; where that confirmation cannot be made (no access to the platform's settings), `PARTIAL` is the honest verdict, not a `PASS` on the file alone. + +## CI-05 — Workflow files are linted + +Why it matters An agent cannot run a workflow on its machine. A linter that knows the CI provider is its + only check before a push: without one, a broken expression, a wrong action input or a + script-injection risk is found only after the push + ([greenfield/README.md § 4.1 Workflows](../../greenfield/README.md#41-workflows)). +Severity DEGRADER +Engine `CI-05`. For each CI provider with workflow files, looks for a linter that knows that + provider: actionlint, zizmor or `check-jsonschema --builtin-schema vendor.github-workflows` + for GitHub; `check-jsonschema` with the provider's built-in schema (or the matching + pre-commit hook id, such as `check-gitlab-ci`) for GitLab, Azure, CircleCI and Buildkite; + `glab ci lint` and `circleci config validate` count too. A linter named in a local file (a + runner manifest, a build file, a hook config such as `.husky/pre-commit`, + `.pre-commit-config.yaml` or `lefthook.yml`, a script) is local; one named only in a + workflow file (a `run:` step or a `uses:` action) is CI only. Documentation, lockfiles and + Dependabot or Renovate configs never count: they name a tool without running it. +Method automated +Look for The command or hook that runs the linter, and the CI step that runs it. Accept the + stack's own route: a `package.json` script or Husky hook (Node), a pre-commit hook + (Python), a Makefile or Gradle task, lefthook. Check that the linter's target really is + the workflow folder. +Rubric PASS: every provider with workflow files has a linter that runs locally. + PARTIAL: a provider's linter runs only in CI, or one provider is linted and another is not. + FAIL: workflow files exist and nothing lints them. + NOT_APPLICABLE: never. + UNKNOWN when: no workflow file exists (`CI-01` reports that), or the only provider has no + known offline linter (Jenkins). +Evidence to cite The workflow files, and for a `PARTIAL` the CI line that runs the linter, with `note` + naming the linter. +Absence proof A FAIL cites every workflow file of the unlinted provider; the scan of the local surface + and the workflows for every linter in the provider's list is the search. +Remediation `add-ci-parity` +Pitfalls The scan matches tool names in text. A comment that names `actionlint` counts, and a + linter started from a script the scan does not read is missed. Read the hook or target + before trusting a `PASS`, and look for an indirect route before trusting a `FAIL`. diff --git a/audit/engine/audit/__init__.py b/audit/engine/audit/__init__.py index b68dc2ce..3ffc8709 100644 --- a/audit/engine/audit/__init__.py +++ b/audit/engine/audit/__init__.py @@ -24,7 +24,7 @@ #: Bumped whenever a rule pack, a check, or the instruction-variant table changes. #: Recorded in every report's provenance block so a report can be attributed to an #: exact catalogue revision (docs: Scan Engine & Tiering, § 7 Determinism). -__ruleset_revision__ = "2026-11-06.1" +__ruleset_revision__ = "2026-11-09.1" #: Highest framework phase this build evaluates. Phase 1 only in v1. __phase_ceiling__ = 1 diff --git a/audit/engine/audit/rules/checks/ci.py b/audit/engine/audit/rules/checks/ci.py index 92f61916..d5c4eebe 100644 --- a/audit/engine/audit/rules/checks/ci.py +++ b/audit/engine/audit/rules/checks/ci.py @@ -6,7 +6,7 @@ Anchor: `docs/framework/phased-approach.md` § Key Deliverables. The normative sentence is the one-command-surface note in the Phase 1 activities block. -Implemented here: `CI-01` … `CI-03`. +Implemented here: `CI-01` … `CI-03` and `CI-05` (`CI-04` is an agent-run check with no engine id). The pack verifies that a repository's CI is *the same command surface* an agent runs locally, not a second, secret one. Workflow files are read through `audit.scan`; the local verbs they are compared @@ -28,6 +28,15 @@ Framework-verb invocations are `CI-02`'s subject and are not repeated here; package-manager plumbing (`npm ci`, `npm publish`) and shell automation (`gh`, `echo`) are not project tools and are ignored. No findings ⇒ `PASS`. +* **CI-05** — the workflow files are linted, per CI provider, by a tool that knows that provider: + actionlint, zizmor or `check-jsonschema --builtin-schema vendor.github-workflows` for GitHub; + `check-jsonschema` with the provider's built-in schema (or the matching pre-commit hook id) for + GitLab, Azure, CircleCI and Buildkite; `glab ci lint` and `circleci config validate` count too. + A linter named in a local surface (a runner manifest, a build file, a hook config, a script) ⇒ + local; named only in a workflow file (a `run:` step or a `uses:` action) ⇒ CI only. Every + lintable provider linted locally ⇒ `PASS`; at least one linted only in CI, or one linted and + another not ⇒ `PARTIAL`; workflows exist and nothing lints them ⇒ `FAIL`. No workflow file + (`CI-01`'s subject), or only providers with no known offline linter (Jenkins) ⇒ `UNKNOWN`. Deliberate limits, recorded rather than hidden. **YAML is not parsed.** This module is a minimal, documented text scan: it recognises `run:` / `script:` / `command:` scalar and block-scalar steps, @@ -35,7 +44,10 @@ steps, matrix expansion and full YAML semantics are out of scope, so a command hidden inside an expression is not seen. **Verb mapping is textual**, reusing the runner-indirection idea from `CMD-01`; an ecosystem outside that set is not mapped. Neither limit produces a `PASS` by guessing: -an unmapped command is simply not counted as evidence. +an unmapped command is simply not counted as evidence. **CI-05 matches tool names in text**: a +local file that names `actionlint` in a comment counts, and a linter started through an indirection +the scan cannot see (a script it does not read) is missed. Documentation files, lockfiles and +dependency-update configs (Dependabot, Renovate) never count: they name a tool without running it. """ from __future__ import annotations @@ -48,6 +60,7 @@ from audit.rules.checks._common import resolve_verbs, runner_commands from audit.stack import CI_PROVIDERS, VERBS from audit.rules.payloads import Payload +from audit.scan import Kind #: The read-only verbs a CI pipeline may be expected to exercise (everything but a write verb). READ_ONLY_VERBS: tuple[str, ...] = tuple(v for v in VERBS if v not in MUTATING_VERBS) @@ -196,6 +209,67 @@ ) +#: CI-05: the linters that understand each provider's workflow files. A pattern is matched against +#: one line of text; pre-commit hook ids from check-jsonschema are listed so a hook config counts. +_WORKFLOW_LINTERS: dict[str, tuple[tuple[str, re.Pattern[str]], ...]] = { + provider: tuple((name, re.compile(pattern)) for name, pattern in linters) + for provider, linters in ( + ( + "github", + ( + ("actionlint", r"\bactionlint\b"), + ("zizmor", r"\bzizmor\b"), + ("check-jsonschema", r"vendor\.github-workflows|\bcheck-github-workflows\b"), + ), + ), + ( + "gitlab", + ( + ("check-jsonschema", r"vendor\.gitlab-ci|\bcheck-gitlab-ci\b"), + ("glab ci lint", r"\bglab\s+ci\s+lint\b"), + ), + ), + ("azure", (("check-jsonschema", r"vendor\.azure-pipelines|\bcheck-azure-pipelines\b"),)), + ( + "circleci", + ( + ("check-jsonschema", r"vendor\.circle-ci|\bcheck-circle-ci\b"), + ("circleci config validate", r"\bcircleci\s+config\s+validate\b"), + ), + ), + ("buildkite", (("check-jsonschema", r"vendor\.buildkite|\bcheck-buildkite"),)), + ) +} + +#: CI-05: file kinds that make up the local surface (runner manifests, build files, hook configs, +#: scripts, extension-less files such as `.husky/pre-commit` and `justfile`). Docs and sources never count. +_LOCAL_SURFACE_KINDS: tuple[Kind, ...] = (Kind.MANIFEST, Kind.BUILD, Kind.CONFIG, Kind.SCRIPT, Kind.OTHER) + +#: CI-05: files that name a tool without running it — lockfiles and dependency-update configs. Never local evidence. +_NOT_A_SURFACE: frozenset[str] = frozenset( + { + "package-lock.json", + "npm-shrinkwrap.json", + "yarn.lock", + "pnpm-lock.yaml", + "bun.lock", + "uv.lock", + "poetry.lock", + "Pipfile.lock", + "pdm.lock", + "Cargo.lock", + "go.sum", + "Gemfile.lock", + "composer.lock", + "dependabot.yml", + "dependabot.yaml", + "renovate.json", + ".renovaterc", + ".renovaterc.json", + } +) + + def _outcome( spec, verdict: Verdict, summary: str = "", findings: list[Finding] | None = None, data: Payload | None = None ) -> CheckOutcome: @@ -217,7 +291,15 @@ def _unknown(spec, reason: str) -> CheckOutcome: return _outcome(spec, Verdict.UNKNOWN, reason) -def _finding(spec, cannot: str, because: str, verdict: Verdict, evidence: list[Evidence], remediation: str) -> Finding: +def _finding( + spec, + cannot: str, + because: str, + verdict: Verdict, + evidence: list[Evidence], + remediation: str, + path: str | None = None, +) -> Finding: return Finding( check=spec.id, severity=spec.severity, @@ -226,6 +308,7 @@ def _finding(spec, cannot: str, because: str, verdict: Verdict, evidence: list[E statement=statement(cannot, because), evidence=evidence, remediation=remediation, + path=path, ) @@ -453,8 +536,111 @@ def check_ci03(*, spec, target, inventory, stack, components, session) -> CheckO return _outcome(spec, Verdict.FAIL, f"{len(findings)} CI-only step(s)", findings) +# =========================================================================== +# CI-05 — workflow files are linted +# =========================================================================== + + +def _local_linter_hits(inventory, provider: str, workflow_rels: set[str]) -> list[tuple[str, int, str]]: + """``(rel, line, linter)`` for each local-surface line that names one of the provider's linters.""" + hits: set[tuple[str, int, str]] = set() + for name, pattern in _WORKFLOW_LINTERS[provider]: + for rel, line, _text in inventory.grep(pattern, kinds=_LOCAL_SURFACE_KINDS): + if rel not in workflow_rels and rel.rsplit("/", 1)[-1] not in _NOT_A_SURFACE: + hits.add((rel, line, name)) + return sorted(hits) + + +def _ci_linter_hits(inventory, provider: str, workflow_rels: set[str]) -> list[tuple[str, int, str]]: + """``(rel, line, linter)`` for each workflow line (a `run:` step or a `uses:` action) naming a linter.""" + hits: set[tuple[str, int, str]] = set() + for rel in sorted(workflow_rels): + for number, text in enumerate((inventory.read(rel) or "").splitlines(), start=1): + for name, pattern in _WORKFLOW_LINTERS[provider]: + if pattern.search(text): + hits.add((rel, number, name)) + return sorted(hits) + + +def check_ci05(*, spec, target, inventory, stack, components, session) -> CheckOutcome: + workflows = _workflows(inventory) + if not workflows: + return _unknown(spec, "no workflow file, so there is nothing to lint (CI-01 reports the absence)") + + workflow_rels = {rel for _provider, rel in workflows} + by_provider: dict[str, list[str]] = {} + for provider, rel in workflows: + by_provider.setdefault(provider, []).append(rel) + lintable = [provider for provider in by_provider if provider in _WORKFLOW_LINTERS] + if not lintable: + names = ", ".join(sorted(by_provider)) + return _unknown(spec, f"no offline workflow linter is known for {names}") + + local: dict[str, list[tuple[str, int, str]]] = {} + ci_only: dict[str, list[tuple[str, int, str]]] = {} + for provider in lintable: + local[provider] = _local_linter_hits(inventory, provider, workflow_rels) + ci_only[provider] = _ci_linter_hits(inventory, provider, workflow_rels) + + def seen(hits: list[tuple[str, int, str]]) -> str: + return ", ".join(sorted({f"{name} ({rel})" for rel, _line, name in hits})) + + detail = "; ".join( + f"{provider}: local {seen(local[provider]) or 'none'}, CI {seen(ci_only[provider]) or 'none'}" + for provider in lintable + ) + unlinted = [provider for provider in lintable if not local[provider] and not ci_only[provider]] + if len(unlinted) == len(lintable): + return _outcome( + spec, + Verdict.FAIL, + detail, + [ + _finding( + spec, + "check a workflow change before it is pushed", + f"no linter for the {provider} workflow files runs locally or in CI", + Verdict.FAIL, + [Evidence(rel) for rel in by_provider[provider]], + "Add a workflow linter to the local command surface or a hook and run it in CI " + "(GitHub: actionlint and zizmor; other providers: check-jsonschema with the " + "provider's built-in schema).", + path=by_provider[provider][0], + ) + for provider in unlinted + ], + ) + + not_local = [provider for provider in lintable if not local[provider]] + if not not_local: + return _outcome(spec, Verdict.PASS, detail) + + findings = [] + for provider in not_local: + in_ci = bool(ci_only[provider]) + findings.append( + _finding( + spec, + "check a workflow change before it is pushed", + ( + f"the {provider} workflow linter runs only in CI" + if in_ci + else f"no linter for the {provider} workflow files runs locally or in CI" + ), + Verdict.PARTIAL, + [Evidence(rel, line=line, note=name) for rel, line, name in ci_only[provider]] + or [Evidence(rel) for rel in by_provider[provider]], + "Expose the workflow linter on the local command surface or in a hook, so an agent " + "can run the same check before it pushes.", + path=by_provider[provider][0], + ) + ) + return _outcome(spec, Verdict.PARTIAL, detail, findings) + + IMPLEMENTATIONS = { "CI-01": check_ci01, "CI-02": check_ci02, "CI-03": check_ci03, + "CI-05": check_ci05, } diff --git a/audit/engine/audit/rules/registry.py b/audit/engine/audit/rules/registry.py index d596c8c2..4ae3b7c9 100644 --- a/audit/engine/audit/rules/registry.py +++ b/audit/engine/audit/rules/registry.py @@ -174,6 +174,7 @@ def _c( _c("CI-01", "Pipeline present, triggered on PRs", "B", _S.BLOCKER), _c("CI-02", "Local↔CI parity", "B", _S.BLOCKER), _c("CI-03", "CI-only steps flagged", "B", _S.DEGRADER), + _c("CI-05", "Workflow files are linted", "B", _S.DEGRADER), # -- BASE ----------------------------------------------------------- _c("BASE-01", "Baseline artifact exists", "A", _S.DEGRADER), _c("BASE-02", "Current violation counts measured", "B", _S.DEGRADER), diff --git a/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md b/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md index b40284de..c35f88f8 100644 --- a/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md +++ b/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md @@ -25,23 +25,24 @@ | `DOC` Documentation contract | 4 | 3 | 1 | — | | `CMD` Command surface | 4 | — | 4 | — | | `TOOL` Tooling | 6 | 3 | 3 | — | -| `CI` CI | 3 | — | 3 | — | +| `CI` CI | 4 | — | 4 | — | | `BASE` Baselines & ratchet | 4 | 2 | 2 | — | | `CON` Constraints | 4 | — | 3 | — | | `EXEC` Execution determinism | 7 | 2 | 5 | — | | `TST` Tests | 5 | 2 | 2 | 1 | | `NAV` Navigation | 3 | 1 | 2 | — | | `SEC` Security hygiene | 3 | — | 3 | — | -| **Total catalogued** | **56** | **17** | **36** | **2** | +| **Total catalogued** | **57** | **17** | **37** | **2** | Plus **13 informational checks** (`NAV-04`, `NAV-05`, `HYG-01`–`HYG-11`), reported and never scored. -Of the 56, `CON-04` is an emitter rather than a check, leaving **55 scoreable checks** in v1. (`CON-01` carried weight 0 +Of the 57, `CON-04` is an emitter rather than a check, leaving **56 scoreable checks** in v1. (`CON-01` carried weight 0 until ruleset `2026-10-11.1`, because the engine did not read the artifact that the spec defines; it reads the prohibitions section now.) > [!NOTE] -> The tier columns sum to 55, not 56: `CON-04` is an emitter with no tier. The variant research added `AGT-10`. +> The tier columns sum to 56, not 57: `CON-04` is an emitter with no tier. The variant research added `AGT-10`; +> v0.3.3 added `CI-05`. > Earlier revisions of this table carried per-group tier splits that disagreed with their own rows — `--list-checks` > is the executable truth, and a mismatch between it and this page is a defect in this page. @@ -127,6 +128,7 @@ suspicious verdict. | `CI-01` | Pipeline present, triggered on PRs | B | BLOCKER | A workflow file exists and declares a pull-request trigger | Implemented | | `CI-02` | Local↔CI parity | B | BLOCKER | Set comparison of CI-invoked commands against the resolved verb set from `CMD-01` | Implemented | | `CI-03` | CI-only steps flagged | B | DEGRADER | CI steps with no local equivalent are listed as findings | Implemented | +| `CI-05` | Workflow files are linted | B | DEGRADER | A linter that knows the CI provider is named on the local surface (`PASS`) or only in CI (`PARTIAL`) | Implemented | --- diff --git a/audit/engine/docs/architecture/user-overview/03-why-these-checks.md b/audit/engine/docs/architecture/user-overview/03-why-these-checks.md index 77a7ea60..6714bc3c 100644 --- a/audit/engine/docs/architecture/user-overview/03-why-these-checks.md +++ b/audit/engine/docs/architecture/user-overview/03-why-these-checks.md @@ -93,6 +93,7 @@ One command surface, three consumers. This family is the core of Phase 1 and the | `CI-01` Pipeline present, triggered on PRs | Verification must be automatic and independent of the author | "I ran the tests" is unverifiable | | `CI-02` Local↔CI parity | The gate an agent *can* run must be the gate that *decides* | Agent optimises against a weaker local gate; rework at review | | `CI-03` CI-only steps flagged | A step with no local equivalent is unreachable by an agent | Agent cannot reproduce the failure it is being failed on | +| `CI-05` Workflow files are linted | An agent cannot run a workflow locally, so a linter is its only check before a push | A broken expression or an injection risk is found only after the push, by CI or by an attacker | --- diff --git a/audit/engine/docs/reference/changelog.md b/audit/engine/docs/reference/changelog.md index d7c99cb7..78fe5d4a 100644 --- a/audit/engine/docs/reference/changelog.md +++ b/audit/engine/docs/reference/changelog.md @@ -13,6 +13,7 @@ the record of how; git has the rest. | Revision | Date | Scoreable checks | What it is | |---|---|---|---| +| `2026-11-09.1` | 2026-11-09 | 56 | v0.3.3 C1: new check `CI-05` (workflow files are linted, DEGRADER). It reads the linters each CI provider supports (actionlint, zizmor, check-jsonschema schemas, `glab ci lint`, `circleci config validate`): local ⇒ `PASS`, CI only ⇒ `PARTIAL`, none ⇒ `FAIL`, no workflow or Jenkins only ⇒ `UNKNOWN`. Every repository with workflow files and no linter gains a `CI-05` finding | | `2026-11-06.1` | 2026-11-06 | 55 | MR06.4: `TST-04` accepts `python3 -m unittest -k ` as a single-test form; `AGT-08` takes a line of the root `AGENTS.md` that says "read by" or "supported harnesses" and names a known harness as the answer to the reach gap (#26) | | `2026-11-05.4` | 2026-11-05 | 55 | MR06.3: `NAV-02` entry point is a guard line, `DOC-04` ignores the ADR template, `--out` without a suffix, `--no-user-config` and `provenance.user_config` | | `2026-11-05.3` | 2026-11-05 | 55 | MR06.2 ([ADR-0009](../architecture/adrs/0009-nested-ignore-files-and-negation.md)): `DOC-03`, `CON-03`, `SEC-01` and `SEC-03` read nested `.gitignore` files and `!` negation | diff --git a/audit/engine/tests/fixtures/README.md b/audit/engine/tests/fixtures/README.md index c14354cd..4ef6515d 100644 --- a/audit/engine/tests/fixtures/README.md +++ b/audit/engine/tests/fixtures/README.md @@ -92,7 +92,9 @@ fixtures/ ci-01-no-pr-trigger/ workflow without a pull-request trigger -> CI-01 PARTIAL ci-02-parity-break/ CI runs a framework verb with no local script -> CI-02 FAIL ci-03-ci-only-step/ CI runs `npm run deploy` with no local script -> CI-03 FAIL - ci-ok-gh/ a PR-triggered GitHub workflow calling the local verbs + ci-05-no-workflow-lint/ workflows exist and no linter checks them -> CI-05 FAIL + ci-05-ci-only-lint/ actionlint runs only as a CI action -> CI-05 PARTIAL + ci-ok-gh/ a PR-triggered GitHub workflow calling the local verbs, actionlint local tst-01-no-suite/ test files but no resolvable test verb -> TST-01 PARTIAL tst-03-skip-markers/ a suite with skip/todo markers -> TST-03 PASS (non-zero census) tst-04-no-fastpath/ docs show the full suite only -> TST-04 FAIL diff --git a/audit/engine/tests/fixtures/ci-01-no-pr-trigger/package.json b/audit/engine/tests/fixtures/ci-01-no-pr-trigger/package.json index d6468e3c..5f48cedb 100644 --- a/audit/engine/tests/fixtures/ci-01-no-pr-trigger/package.json +++ b/audit/engine/tests/fixtures/ci-01-no-pr-trigger/package.json @@ -9,6 +9,7 @@ "typecheck": "tsc --noEmit", "test": "vitest run", "check": "npm run format:check && npm run typecheck && npm test", - "security": "npm audit --audit-level=high" + "security": "npm audit --audit-level=high", + "lint:workflows": "actionlint" } } diff --git a/audit/engine/tests/fixtures/ci-02-parity-break/package.json b/audit/engine/tests/fixtures/ci-02-parity-break/package.json index 2125f60a..4d45ec14 100644 --- a/audit/engine/tests/fixtures/ci-02-parity-break/package.json +++ b/audit/engine/tests/fixtures/ci-02-parity-break/package.json @@ -8,6 +8,7 @@ "lint": "eslint .", "typecheck": "tsc --noEmit", "test": "vitest run", - "check": "npm run format:check && npm run typecheck && npm test" + "check": "npm run format:check && npm run typecheck && npm test", + "lint:workflows": "actionlint" } } diff --git a/audit/engine/tests/fixtures/ci-03-ci-only-step/package.json b/audit/engine/tests/fixtures/ci-03-ci-only-step/package.json index 62441606..a10021d7 100644 --- a/audit/engine/tests/fixtures/ci-03-ci-only-step/package.json +++ b/audit/engine/tests/fixtures/ci-03-ci-only-step/package.json @@ -9,6 +9,7 @@ "typecheck": "tsc --noEmit", "test": "vitest run", "check": "npm run format:check && npm run typecheck && npm test", - "security": "npm audit --audit-level=high" + "security": "npm audit --audit-level=high", + "lint:workflows": "actionlint" } } diff --git a/audit/engine/tests/fixtures/ci-05-ci-only-lint/.github/workflows/ci.yml b/audit/engine/tests/fixtures/ci-05-ci-only-lint/.github/workflows/ci.yml new file mode 100644 index 00000000..6a74555e --- /dev/null +++ b/audit/engine/tests/fixtures/ci-05-ci-only-lint/.github/workflows/ci.yml @@ -0,0 +1,15 @@ +name: CI + +on: + pull_request: + branches: [main] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: raven-actions/actionlint@v2 + - run: npm ci + - run: npm run check + - run: npm run security diff --git a/audit/engine/tests/fixtures/ci-05-ci-only-lint/package.json b/audit/engine/tests/fixtures/ci-05-ci-only-lint/package.json new file mode 100644 index 00000000..814da486 --- /dev/null +++ b/audit/engine/tests/fixtures/ci-05-ci-only-lint/package.json @@ -0,0 +1,14 @@ +{ + "name": "ci-05-ci-only-lint", + "version": "0.1.0", + "private": true, + "scripts": { + "format": "prettier --write .", + "format:check": "prettier --check .", + "lint": "eslint .", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "check": "npm run format:check && npm run typecheck && npm test", + "security": "npm audit --audit-level=high" + } +} diff --git a/audit/engine/tests/fixtures/ci-05-no-workflow-lint/.github/workflows/ci.yml b/audit/engine/tests/fixtures/ci-05-no-workflow-lint/.github/workflows/ci.yml new file mode 100644 index 00000000..6cc3fc7e --- /dev/null +++ b/audit/engine/tests/fixtures/ci-05-no-workflow-lint/.github/workflows/ci.yml @@ -0,0 +1,14 @@ +name: CI + +on: + pull_request: + branches: [main] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: npm ci + - run: npm run check + - run: npm run security diff --git a/audit/engine/tests/fixtures/ci-05-no-workflow-lint/package.json b/audit/engine/tests/fixtures/ci-05-no-workflow-lint/package.json new file mode 100644 index 00000000..59e0f89a --- /dev/null +++ b/audit/engine/tests/fixtures/ci-05-no-workflow-lint/package.json @@ -0,0 +1,14 @@ +{ + "name": "ci-05-no-workflow-lint", + "version": "0.1.0", + "private": true, + "scripts": { + "format": "prettier --write .", + "format:check": "prettier --check .", + "lint": "eslint .", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "check": "npm run format:check && npm run typecheck && npm test", + "security": "npm audit --audit-level=high" + } +} diff --git a/audit/engine/tests/fixtures/ci-ok-gh/package.json b/audit/engine/tests/fixtures/ci-ok-gh/package.json index 749c7159..c51d5339 100644 --- a/audit/engine/tests/fixtures/ci-ok-gh/package.json +++ b/audit/engine/tests/fixtures/ci-ok-gh/package.json @@ -9,6 +9,7 @@ "typecheck": "tsc --noEmit", "test": "vitest run", "check": "npm run format:check && npm run typecheck && npm test", - "security": "npm audit --audit-level=high" + "security": "npm audit --audit-level=high", + "lint:workflows": "actionlint" } } diff --git a/audit/engine/tests/test_ci_pack.py b/audit/engine/tests/test_ci_pack.py index 2bdc4c0d..aaa8fd94 100644 --- a/audit/engine/tests/test_ci_pack.py +++ b/audit/engine/tests/test_ci_pack.py @@ -4,8 +4,9 @@ """CI checks against their one-defect fixtures. Each fixture is a small node project plus a GitHub workflow that isolates one CI defect; -`ci-ok-gh` passes all three. The inline cases cover the `UNKNOWN` floor (no local verbs), a -non-GitHub PR trigger, and the minimal YAML scan reading a block-scalar step. +`ci-ok-gh` passes every engine CI check. The inline cases cover the `UNKNOWN` floor (no local verbs), a +non-GitHub PR trigger, the minimal YAML scan reading a block-scalar step, and `CI-05`'s provider and +local-surface rules. """ from __future__ import annotations @@ -20,13 +21,15 @@ import support # noqa: E402 -CI_CHECKS = ("CI-01", "CI-02", "CI-03") +CI_CHECKS = ("CI-01", "CI-02", "CI-03", "CI-05") #: fixture directory -> (check id under test, expected verdict) FIXTURES: dict[str, tuple[str, str]] = { "ci-01-no-pr-trigger": ("CI-01", "PARTIAL"), "ci-02-parity-break": ("CI-02", "FAIL"), "ci-03-ci-only-step": ("CI-03", "FAIL"), + "ci-05-no-workflow-lint": ("CI-05", "FAIL"), + "ci-05-ci-only-lint": ("CI-05", "PARTIAL"), } @@ -42,7 +45,7 @@ def test_each_fixture_fails_exactly_its_check(self): expected_failing = [check] if expected == "FAIL" else [] self.assertEqual(failing, expected_failing, f"{name}: unexpected CI FAILs {failing}") - def test_positive_fixture_passes_all_three(self): + def test_positive_fixture_passes_every_ci_check(self): report = support.run_audit(support.fixture("ci-ok-gh")) verdicts = support.verdicts(report) for check in CI_CHECKS: @@ -110,5 +113,87 @@ def test_npm_stage_publish_is_a_builtin_not_ci_only(self): self.assertEqual(support.verdicts(report)["CI-03"], "PASS") +_GH_WORKFLOW = "on:\n pull_request:\njobs:\n b:\n steps:\n - run: npm run check\n" +_PACKAGE = '{"scripts": {"check": "true"}}' + + +class Ci05InlineTests(unittest.TestCase): + def _ci05(self, files: dict[str, str]) -> str: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for rel, text in files.items(): + support.write(root, rel, text) + return support.verdicts(support.run_audit(root))["CI-05"] + + def test_no_workflow_is_unknown(self): + self.assertEqual(self._ci05({"package.json": _PACKAGE}), "UNKNOWN") + + def test_jenkins_only_is_unknown(self): + self.assertEqual(self._ci05({"Jenkinsfile": "pipeline { stages { } }\n", "package.json": _PACKAGE}), "UNKNOWN") + + def test_make_target_running_zizmor_is_a_pass(self): + files = { + ".github/workflows/ci.yml": _GH_WORKFLOW, + "Makefile": "check:\n\ttrue\nworkflow-lint:\n\tuvx zizmor --offline .github\n", + } + self.assertEqual(self._ci05(files), "PASS") + + def test_pre_commit_hook_id_is_a_local_linter(self): + files = { + ".github/workflows/ci.yml": _GH_WORKFLOW, + "package.json": _PACKAGE, + ".pre-commit-config.yaml": "repos:\n - repo: https://github.com/python-jsonschema/check-jsonschema\n" + " rev: 0.39.0\n hooks:\n - id: check-github-workflows\n", + } + self.assertEqual(self._ci05(files), "PASS") + + def test_husky_hook_is_a_local_linter(self): + files = { + ".github/workflows/ci.yml": _GH_WORKFLOW, + "package.json": _PACKAGE, + ".husky/pre-commit": "actionlint\n", + } + self.assertEqual(self._ci05(files), "PASS") + + def test_gitlab_schema_check_is_a_pass(self): + files = { + ".gitlab-ci.yml": "test:\n script:\n - npm run check\n", + "Makefile": "lint-ci:\n\tcheck-jsonschema --builtin-schema vendor.gitlab-ci .gitlab-ci.yml\n", + } + self.assertEqual(self._ci05(files), "PASS") + + def test_a_github_linter_does_not_lint_gitlab(self): + files = { + ".gitlab-ci.yml": "test:\n script:\n - npm run check\n", + "Makefile": "lint-ci:\n\tactionlint\n", + } + self.assertEqual(self._ci05(files), "FAIL") + + def test_a_linter_named_only_in_docs_does_not_count(self): + files = { + ".github/workflows/ci.yml": _GH_WORKFLOW, + "package.json": _PACKAGE, + "CONTRIBUTING.md": "Run actionlint before you push.\n", + } + self.assertEqual(self._ci05(files), "FAIL") + + def test_a_lockfile_or_dependabot_naming_the_linter_does_not_count(self): + files = { + ".github/workflows/ci.yml": _GH_WORKFLOW, + "package.json": _PACKAGE, + "uv.lock": '[[package]]\nname = "zizmor"\n', + ".github/dependabot.yml": "updates:\n - package-ecosystem: pip # zizmor, actionlint\n", + } + self.assertEqual(self._ci05(files), "FAIL") + + def test_one_provider_linted_and_one_not_is_partial(self): + files = { + ".github/workflows/ci.yml": _GH_WORKFLOW, + ".gitlab-ci.yml": "test:\n script:\n - npm run check\n", + "Makefile": "lint-ci:\n\tactionlint\n", + } + self.assertEqual(self._ci05(files), "PARTIAL") + + if __name__ == "__main__": unittest.main() diff --git a/audit/engine/tests/test_fixture_matrix.py b/audit/engine/tests/test_fixture_matrix.py index 0187247f..9f49af25 100644 --- a/audit/engine/tests/test_fixture_matrix.py +++ b/audit/engine/tests/test_fixture_matrix.py @@ -112,7 +112,9 @@ "ci-01-no-pr-trigger": {"CI-01": "PARTIAL"}, "ci-02-parity-break": {"CI-02": "FAIL"}, "ci-03-ci-only-step": {"CI-03": "FAIL"}, - "ci-ok-gh": {"CI-01": "PASS", "CI-02": "PASS", "CI-03": "PASS"}, + "ci-05-no-workflow-lint": {"CI-05": "FAIL"}, + "ci-05-ci-only-lint": {"CI-05": "PARTIAL"}, + "ci-ok-gh": {"CI-01": "PASS", "CI-02": "PASS", "CI-03": "PASS", "CI-05": "PASS"}, # -- BASE --------------------------------------------------------------------------------- "base-01-no-baseline": {"BASE-01": "FAIL"}, "base-02-no-count": {"BASE-02": "PARTIAL"}, diff --git a/audit/engine/tests/test_hyg_pack.py b/audit/engine/tests/test_hyg_pack.py index 275fe890..4e6f520a 100644 --- a/audit/engine/tests/test_hyg_pack.py +++ b/audit/engine/tests/test_hyg_pack.py @@ -90,8 +90,8 @@ def test_report_lists_hyg_checks_but_excludes_them_from_the_scoreable_count(self reported = {c["id"] for c in report["checks"]} for check in HYG_IDS: self.assertIn(check, reported) - # 69 catalogued, 14 informational or emitter-only ⇒ 55 scoreable (CON-01 is scored since ruleset 2026-10-11.1). - self.assertEqual(report["summary"]["checks_scoreable"], 55) + # 70 catalogued, 14 informational or emitter-only ⇒ 56 scoreable (CON-01 is scored since ruleset 2026-10-11.1). + self.assertEqual(report["summary"]["checks_scoreable"], 56) class ScoreIsolationTests(unittest.TestCase): diff --git a/audit/engine/tests/test_registry_status.py b/audit/engine/tests/test_registry_status.py index 60a1548a..8b7b1f5c 100644 --- a/audit/engine/tests/test_registry_status.py +++ b/audit/engine/tests/test_registry_status.py @@ -63,9 +63,9 @@ class RegistryTests(unittest.TestCase): def test_every_catalogued_check_is_implemented_and_none_is_blocked(self): self.assertEqual({s.status for s in REGISTRY}, {"implemented"}) - def test_there_are_fifty_five_scoreable_checks_and_con_01_is_one_of_them(self): - self.assertEqual(len(scoreable()), 55) - self.assertEqual(len(REGISTRY), 69) + def test_there_are_fifty_six_scoreable_checks_and_con_01_is_one_of_them(self): + self.assertEqual(len(scoreable()), 56) + self.assertEqual(len(REGISTRY), 70) con_01 = REGISTRY_BY_ID["CON-01"] self.assertEqual( (con_01.scored, con_01.status, con_01.severity, con_01.tier), (True, "implemented", Severity.BLOCKER, "B") diff --git a/audit/engine/tests/test_smoke.py b/audit/engine/tests/test_smoke.py index 7400a801..596dabd6 100644 --- a/audit/engine/tests/test_smoke.py +++ b/audit/engine/tests/test_smoke.py @@ -39,8 +39,8 @@ def test_ids_are_unique(self): def test_catalogue_size_matches_the_documented_counts(self): from audit.rules.registry import informational, scoreable - self.assertEqual(len(REGISTRY), 69, "catalogue drifted from the documented 69 rows") - self.assertEqual(len(scoreable()), 55, "scoreable count drifted from the documented 55") + self.assertEqual(len(REGISTRY), 70, "catalogue drifted from the documented 70 rows") + self.assertEqual(len(scoreable()), 56, "scoreable count drifted from the documented 56") self.assertEqual(len(informational()), 14, "informational count drifted from 14") def test_every_spec_has_a_tier(self): diff --git a/audit/schema/examples/example-service.audit-report.20261010T094207Z.json b/audit/schema/examples/example-service.audit-report.20261010T094207Z.json index 9c3f2bec..c8586205 100644 --- a/audit/schema/examples/example-service.audit-report.20261010T094207Z.json +++ b/audit/schema/examples/example-service.audit-report.20261010T094207Z.json @@ -7,7 +7,7 @@ "version": "0.1.0" }, "framework_revision": "cafecafecafecafecafecafecafecafecafecafe", - "catalogue_version": "0.2.0", + "catalogue_version": "0.3.0", "engine": { "name": "python-agentic-audit", "version": "0.1.0", diff --git a/audit/schema/examples/example-service.audit-report.20261010T094207Z.md b/audit/schema/examples/example-service.audit-report.20261010T094207Z.md index b5260378..f46cb165 100644 --- a/audit/schema/examples/example-service.audit-report.20261010T094207Z.md +++ b/audit/schema/examples/example-service.audit-report.20261010T094207Z.md @@ -12,7 +12,7 @@ | Status | final | | Started | `2026-10-10T09:00:00+00:00` | | Finalised | `2026-10-10T09:42:07+00:00` | -| Recipe | `brownfield-audit` 0.1.0, catalogue `0.2.0` | +| Recipe | `brownfield-audit` 0.1.0, catalogue `0.3.0` | | Engine | `python-agentic-audit` 0.1.0, ruleset `2026-09-28.13`, ruleset hash `a1b2a1b2a1b2a1b2`, scanned `2026-10-10T09:01:30+00:00` | | Framework revision | `cafecafecafe` | | Agent (self-reported) | `claude-code` with `claude-sonnet-5-5` | diff --git a/audit/schema/examples/minimal.json b/audit/schema/examples/minimal.json index e6051b1c..cd13f4bb 100644 --- a/audit/schema/examples/minimal.json +++ b/audit/schema/examples/minimal.json @@ -7,7 +7,7 @@ "version": "0.1.0" }, "framework_revision": null, - "catalogue_version": "0.2.0", + "catalogue_version": "0.3.0", "agent": { "harness": "claude-code", "model": "claude-sonnet-5-5" diff --git a/audit/schema/examples/minimal.md b/audit/schema/examples/minimal.md index 60e6b9cf..9da4c43d 100644 --- a/audit/schema/examples/minimal.md +++ b/audit/schema/examples/minimal.md @@ -15,7 +15,7 @@ | Status | draft | | Started | `2026-10-10T09:00:00+00:00` | | Finalised | — | -| Recipe | `brownfield-audit` 0.1.0, catalogue `0.2.0` | +| Recipe | `brownfield-audit` 0.1.0, catalogue `0.3.0` | | Engine | No engine report is part of this report. | | Framework revision | — | | Agent (self-reported) | `claude-code` with `claude-sonnet-5-5` | diff --git a/audit/schema/examples/readiness-state.example-service.in-progress.json b/audit/schema/examples/readiness-state.example-service.in-progress.json index aa8106ee..3565b232 100644 --- a/audit/schema/examples/readiness-state.example-service.in-progress.json +++ b/audit/schema/examples/readiness-state.example-service.in-progress.json @@ -13,8 +13,8 @@ { "path": "docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json", "git_sha": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", - "catalogue_version": "0.2.0", - "sha256": "6deeb580fb116df5b17cb6fd148858b62426ba8cf1139be1a10684542a61d867" + "catalogue_version": "0.3.0", + "sha256": "443653e7d98e395b4b20e2f2a0f0020db50ce11ee7bdc1c63331ac61572ad1d9" } ], "preferences": [ diff --git a/audit/schema/examples/readiness-state.example-service.in-progress.md b/audit/schema/examples/readiness-state.example-service.in-progress.md index 4069f2d8..3d3a8af7 100644 --- a/audit/schema/examples/readiness-state.example-service.in-progress.md +++ b/audit/schema/examples/readiness-state.example-service.in-progress.md @@ -10,7 +10,7 @@ A skipped finding is final until the owner unskips it, and every preference bind |---|---| | Default branch | `main` | | Recipe | brownfield-implement 0.2.0-dev, framework `cafecafecafecafecafecafecafecafecafecafe` | -| Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdeadbeefdeadbeefdeadbeefdeadbeef`, catalogue 0.2.0 | +| Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdeadbeefdeadbeefdeadbeefdeadbeef`, catalogue 0.3.0 | ## Summary diff --git a/audit/schema/examples/readiness-state.example-service.json b/audit/schema/examples/readiness-state.example-service.json index 95a152da..467aa5e9 100644 --- a/audit/schema/examples/readiness-state.example-service.json +++ b/audit/schema/examples/readiness-state.example-service.json @@ -13,8 +13,8 @@ { "path": "docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json", "git_sha": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", - "catalogue_version": "0.2.0", - "sha256": "6deeb580fb116df5b17cb6fd148858b62426ba8cf1139be1a10684542a61d867" + "catalogue_version": "0.3.0", + "sha256": "443653e7d98e395b4b20e2f2a0f0020db50ce11ee7bdc1c63331ac61572ad1d9" } ], "preferences": [], diff --git a/audit/schema/examples/readiness-state.example-service.md b/audit/schema/examples/readiness-state.example-service.md index 11fd43d5..aaf9e71d 100644 --- a/audit/schema/examples/readiness-state.example-service.md +++ b/audit/schema/examples/readiness-state.example-service.md @@ -10,7 +10,7 @@ A skipped finding is final until the owner unskips it, and every preference bind |---|---| | Default branch | `main` | | Recipe | brownfield-implement 0.2.0-dev, framework `cafecafecafecafecafecafecafecafecafecafe` | -| Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdeadbeefdeadbeefdeadbeefdeadbeef`, catalogue 0.2.0 | +| Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdeadbeefdeadbeefdeadbeefdeadbeef`, catalogue 0.3.0 | ## Summary diff --git a/audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json b/audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json index d7e0d664..120cbeb8 100644 --- a/audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json +++ b/audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json @@ -7,7 +7,7 @@ "version": "0.1.0" }, "framework_revision": "7eedf23c99c39f94cad435e8a52eb28642308a92", - "catalogue_version": "0.2.0", + "catalogue_version": "0.3.0", "engine": { "name": "python-agentic-audit", "version": "0.1.0", @@ -1528,6 +1528,18 @@ "summary": "github offers branch protection as code, none found", "evidence": [], "searches": [] + }, + { + "id": "CI-05", + "title": "Workflow files are linted", + "phase": 1, + "severity": "DEGRADER", + "scored": true, + "method": "automated", + "verdict": "UNKNOWN", + "summary": "The engine did not run this check: its ruleset is older than the catalogue. Scan again with a newer engine.", + "evidence": [], + "searches": [] } ], "findings": [ @@ -2111,15 +2123,15 @@ } ], "summary": { - "phase1_score": 0.7131, - "scored_checks": 55, + "phase1_score": 0.7016, + "scored_checks": 56, "blockers_remaining": { "total": 6, "fail": 2, "partial": 2, "unknown": 2 }, - "unknown_scored_checks": 4, + "unknown_scored_checks": 5, "unconfirmed_agent_findings": 0 }, "unattested": [ diff --git a/audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json b/audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json index 022ad83b..48d31fe2 100644 --- a/audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json +++ b/audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json @@ -7,7 +7,7 @@ "version": "0.1.0" }, "framework_revision": "7eedf23c99c39f94cad435e8a52eb28642308a92", - "catalogue_version": "0.2.0", + "catalogue_version": "0.3.0", "engine": { "name": "python-agentic-audit", "version": "0.1.0", @@ -1754,6 +1754,18 @@ "summary": "github offers branch protection as code, none found", "evidence": [], "searches": [] + }, + { + "id": "CI-05", + "title": "Workflow files are linted", + "phase": 1, + "severity": "DEGRADER", + "scored": true, + "method": "automated", + "verdict": "UNKNOWN", + "summary": "The engine did not run this check: its ruleset is older than the catalogue. Scan again with a newer engine.", + "evidence": [], + "searches": [] } ], "findings": [ @@ -2686,15 +2698,15 @@ } ], "summary": { - "phase1_score": 0.6352, - "scored_checks": 55, + "phase1_score": 0.625, + "scored_checks": 56, "blockers_remaining": { "total": 6, "fail": 2, "partial": 2, "unknown": 2 }, - "unknown_scored_checks": 4, + "unknown_scored_checks": 5, "unconfirmed_agent_findings": 0 }, "unattested": [ diff --git a/audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json b/audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json index 5250e9e4..95cf2e1b 100644 --- a/audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json +++ b/audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json @@ -7,7 +7,7 @@ "version": "0.1.0" }, "framework_revision": "7eedf23c99c39f94cad435e8a52eb28642308a92", - "catalogue_version": "0.2.0", + "catalogue_version": "0.3.0", "engine": { "name": "python-agentic-audit", "version": "0.1.0", @@ -1506,6 +1506,18 @@ "summary": "github offers branch protection as code, none found", "evidence": [], "searches": [] + }, + { + "id": "CI-05", + "title": "Workflow files are linted", + "phase": 1, + "severity": "DEGRADER", + "scored": true, + "method": "automated", + "verdict": "UNKNOWN", + "summary": "The engine did not run this check: its ruleset is older than the catalogue. Scan again with a newer engine.", + "evidence": [], + "searches": [] } ], "findings": [ @@ -2281,15 +2293,15 @@ } ], "summary": { - "phase1_score": 0.4139, - "scored_checks": 55, + "phase1_score": 0.4073, + "scored_checks": 56, "blockers_remaining": { "total": 9, "fail": 5, "partial": 1, "unknown": 3 }, - "unknown_scored_checks": 6, + "unknown_scored_checks": 7, "unconfirmed_agent_findings": 0 }, "unattested": [ diff --git a/audit/tools/tests/_engine_reports.py b/audit/tools/tests/_engine_reports.py index 83cb585c..1371f666 100644 --- a/audit/tools/tests/_engine_reports.py +++ b/audit/tools/tests/_engine_reports.py @@ -12,6 +12,7 @@ import io import json +import re from pathlib import Path from typing import Any, Mapping, Sequence @@ -22,6 +23,14 @@ CATALOGUE: dict[str, Mapping[str, Any]] = {entry["id"]: entry for entry in vr.load_catalogue()["checks"]} +#: The engine's current ruleset revision, read from its source. A hand-built report claims it, so it stands for a +#: report of today's engine: no catalogue check is newer than it (`since_ruleset`), and none is filled in as UNKNOWN. +RULESET_REVISION: str = re.search( + r'^__ruleset_revision__ = "([^"]+)"', + (Path(launcher.__file__).resolve().parents[1] / "engine" / "audit" / "__init__.py").read_text(encoding="utf-8"), + re.MULTILINE, +).group(1) # type: ignore[union-attr] + def scan(target: Path | str, out_dir: Path | str, *extra: str) -> Path: """Run the real engine through the launcher on `target`, and return the path of the report it wrote.""" @@ -39,7 +48,7 @@ def provenance(target: Path | str, **changes: Any) -> dict[str, Any]: document: dict[str, Any] = { "tool": "python-agentic-audit", "version": "0.1.0", - "ruleset_revision": "2026-09-28.13", + "ruleset_revision": RULESET_REVISION, "ruleset_hash": "0123456789abcdef", "scanned_at": "2026-10-10T06:57:38+00:00", "target": str(target), diff --git a/audit/tools/tests/test_import_engine.py b/audit/tools/tests/test_import_engine.py index 3ba41f06..747bd789 100644 --- a/audit/tools/tests/test_import_engine.py +++ b/audit/tools/tests/test_import_engine.py @@ -76,7 +76,7 @@ def test_the_draft_records_the_engine_in_the_order_of_the_schema(self) -> None: { "name": "python-agentic-audit", "version": "0.1.0", - "ruleset_revision": "2026-09-28.13", + "ruleset_revision": er.RULESET_REVISION, "ruleset_hash": "0123456789abcdef", "scanned_at": "2026-10-10T06:57:38+00:00", }, @@ -475,7 +475,7 @@ def test_the_envelope_counts_what_came_in(self) -> None: self.assertEqual(data["by_verdict"], {"FAIL": 2, "NOT_APPLICABLE": 1, "PARTIAL": 1, "PASS": 1, "UNKNOWN": 1}) self.assertEqual(list(data["by_verdict"]), sorted(data["by_verdict"])) self.assertEqual((data["findings_imported"], data["findings"], data["findings_pending"]), (3, 3, 3)) - self.assertEqual((data["checks_recorded"], data["checks_expected"]), (6, 71)) + self.assertEqual((data["checks_recorded"], data["checks_expected"]), (6, 72)) self.assertEqual(data["evidence_not_carried"], 1) self.assertEqual( envelope["warnings"], @@ -726,28 +726,37 @@ def test_a_check_missing_from_the_engines_report_is_still_missing(self) -> None: #: What the adapter says on importing each raw report: the engine's score and the report's differ because the report -#: counts CON-01, which the engine's ruleset (before `2026-10-11.1`) had blocked. +#: counts CON-01, which the engine's ruleset (before `2026-10-11.1`) had blocked, and CI-05, which it did not have. BLOCKED_CON_01 = ( "CON-01: the engine's ruleset did not evaluate it, so it is recorded as UNKNOWN and is scored as the catalogue " "scores it" ) +#: The raw reports come from ruleset `2026-09-28.13`, older than `CI-05` (`since_ruleset` in the catalogue), so the +#: adapter records `CI-05` as UNKNOWN and says so. It counts in the report's score and not in the engine's. +NEWER_CI_05 = ( + "CI-05: the engine's ruleset 2026-09-28.13 is older than the check (2026-11-09.1), so it is recorded as " + "UNKNOWN; scan again with a newer engine" +) PAIRS = { "agentic-tdd": [ - "the engine's phase-1 score is 0.7311 and the report's is 0.7131; the report's follows " + "the engine's phase-1 score is 0.7311 and the report's is 0.7016; the report's follows " "spec/readiness-summary.md and is the one kept", "IDX-02: the evidence path '/home/kc/.cache/codebase-memory-mcp' is not carried: " "the path is absolute, so it is outside the repository", BLOCKED_CON_01, + NEWER_CI_05, ], "digital-assistant-sdk": [ - "the engine's phase-1 score is 0.6513 and the report's is 0.6352; the report's follows " + "the engine's phase-1 score is 0.6513 and the report's is 0.625; the report's follows " "spec/readiness-summary.md and is the one kept", BLOCKED_CON_01, + NEWER_CI_05, ], "digital-assistant-server": [ - "the engine's phase-1 score is 0.4244 and the report's is 0.4139; the report's follows " + "the engine's phase-1 score is 0.4244 and the report's is 0.4073; the report's follows " "spec/readiness-summary.md and is the one kept", BLOCKED_CON_01, + NEWER_CI_05, ], } @@ -824,15 +833,20 @@ def test_the_adapters_own_warnings_are_the_ones_expected(self) -> None: adapter_warnings = [w for w in envelope["warnings"] if not w.startswith("/")] self.assertEqual(adapter_warnings, expected) - def test_every_engine_verdict_and_finding_comes_through_and_the_score_is_the_engines_but_for_con_01(self) -> None: + def test_every_engine_verdict_and_finding_comes_through_and_the_score_is_the_engines_but_for_con_01_and_ci_05( + self, + ) -> None: for name in PAIRS: with self.subTest(pair=name): draft, _, _, engine, _ = self.run_pair(name) document = read_json(draft) self.assertEqual( - {c["id"]: (c["verdict"], c["engine_verdict"]) for c in document["checks"]}, + {c["id"]: (c["verdict"], c["engine_verdict"]) for c in document["checks"] if "engine_verdict" in c}, {c["id"]: (c["verdict"], c["verdict"]) for c in engine["checks"]}, ) + # CI-05 is newer than the engine's ruleset: recorded as UNKNOWN, with no verdict of the engine's. + ci_05 = next(c for c in document["checks"] if c["id"] == "CI-05") + self.assertEqual((ci_05["verdict"], "engine_verdict" in ci_05), ("UNKNOWN", False)) self.assertEqual([f["check"] for f in document["findings"]], [f["check"] for f in engine["findings"]]) # The engine's ruleset had blocked CON-01 and left it out of its score; the report counts it, as the # catalogue does. With it left out again, the report's score is the engine's. @@ -842,7 +856,7 @@ def test_every_engine_verdict_and_finding_comes_through_and_the_score_is_the_eng (True, "UNKNOWN", "The engine did not evaluate this check: its ruleset had blocked it."), ) self.assertLess(document["summary"]["phase1_score"], engine["summary"]["phase1_score"]) - con_01["scored"] = False + con_01["scored"] = ci_05["scored"] = False self.assertEqual(_summary.compute_summary(document)["phase1_score"], engine["summary"]["phase1_score"]) self.assertEqual({c["method"] for c in document["checks"]}, {"automated"}) self.assertEqual({f["disposition"] for f in document["findings"]}, {"pending"}) @@ -921,7 +935,7 @@ def test_the_whole_audit_goes_on_from_the_imported_draft_to_a_final_report(self) code, envelope, _ = self.finalise(draft) self.assertEqual(code, 0, envelope) final = read_json(envelope["data"]["report"]) - self.assertEqual((final["status"], len(final["checks"]), vr.validate_document(final).errors), ("final", 71, [])) + self.assertEqual((final["status"], len(final["checks"]), vr.validate_document(final).errors), ("final", 72, [])) self.assertTrue(Path(envelope["data"]["markdown"]).read_text(encoding="utf-8")) diff --git a/audit/tools/tests/test_import_fixture_matrix.py b/audit/tools/tests/test_import_fixture_matrix.py index aa840418..67c6083c 100644 --- a/audit/tools/tests/test_import_fixture_matrix.py +++ b/audit/tools/tests/test_import_fixture_matrix.py @@ -106,7 +106,7 @@ def test_every_fixture_imports_to_a_valid_draft_that_has_the_verdict_the_matrix_ document, envelope = self.imported(name) self.assertEqual(vr.validate_document(document).errors, []) self.assertTrue(render_report.render(document)) - self.assertEqual(len(document["checks"]), 69) + self.assertEqual(len(document["checks"]), 70) checks = {c["id"]: c for c in document["checks"]} for check_id, expected in self.matrix.MATRIX[name].items(): self.assertEqual(checks[check_id]["verdict"], expected, f"{name}: {check_id}") @@ -115,7 +115,7 @@ def test_every_fixture_imports_to_a_valid_draft_that_has_the_verdict_the_matrix_ if expected != "PASS": defects += 1 self.assertNotEqual(checks[check_id]["verdict"], "PASS") - self.assertEqual(envelope["data"]["checks_imported"], 69) + self.assertEqual(envelope["data"]["checks_imported"], 70) self.assertGreater(defects, 40) diff --git a/audit/tools/tests/test_render_report.py b/audit/tools/tests/test_render_report.py index 0184a385..d1b98642 100644 --- a/audit/tools/tests/test_render_report.py +++ b/audit/tools/tests/test_render_report.py @@ -508,7 +508,7 @@ def test_the_worked_example_says_what_was_audited_with_what_and_when(self) -> No self.assertEqual(values["Status"], "final") self.assertEqual(values["Started"], "`2026-10-10T09:00:00+00:00`") self.assertEqual(values["Finalised"], "`2026-10-10T09:42:07+00:00`") - self.assertEqual(values["Recipe"], "`brownfield-audit` 0.1.0, catalogue `0.2.0`") + self.assertEqual(values["Recipe"], "`brownfield-audit` 0.1.0, catalogue `0.3.0`") self.assertEqual( values["Engine"], "`python-agentic-audit` 0.1.0, ruleset `2026-09-28.13`, ruleset hash `a1b2a1b2a1b2a1b2`, " @@ -1499,7 +1499,7 @@ def test_every_check_of_every_example_has_a_row_in_report_order(self) -> None: order.setdefault(check["id"].split("-")[0], len(order)) expected = sorted(report["checks"], key=lambda c: order[c["id"].split("-")[0]]) self.assertEqual(found, [f"`{c['id']}`" for c in expected]) - self.assertEqual(len(rows_of(section(self.render(load(drafts()[0])), "Appendix — all checks"))), 69) + self.assertEqual(len(rows_of(section(self.render(load(drafts()[0])), "Appendix — all checks"))), 70) def test_a_row_gives_the_severity_method_scoring_both_verdicts_and_the_summary(self) -> None: text = section(self.render(service()), "Appendix — all checks") @@ -1879,7 +1879,7 @@ def test_every_realistic_draft_renders_with_every_check_and_every_finding(self) with self.subTest(path=path.name): report = load(path) markdown = self.render(report) - self.assertEqual(len(rows_of(section(markdown, "Appendix — all checks"))), 69) + self.assertEqual(len(rows_of(section(markdown, "Appendix — all checks"))), 70) for finding in report["findings"]: self.assertEqual(markdown.count(f"{BR}finding `{finding['id']}`"), 1) self.assertIn("> [!NOTE]\n> **Draft.**", markdown) diff --git a/audit/tools/tests/test_report.py b/audit/tools/tests/test_report.py index 348001e9..3aba239f 100644 --- a/audit/tools/tests/test_report.py +++ b/audit/tools/tests/test_report.py @@ -104,7 +104,8 @@ def drafts(self) -> list[str]: return sorted(p.name for p in self.out.iterdir()) if self.out.is_dir() else [] def engine_draft(self) -> Path: - """A copy of the engine's real report on agentic-tdd, converted: 69 engine checks, 24 pending findings. + """A copy of the engine's real report on agentic-tdd, converted: 70 engine checks (CI-05 newer, so UNKNOWN), 24 + pending findings. The copy points at the temporary target, with no commit, so that nothing depends on this machine. """ @@ -565,7 +566,7 @@ def test_it_records_an_agent_check_and_fills_in_what_the_catalogue_knows(self) - "action": "added", "verdict": "PASS", "checks_recorded": 1, - "checks_expected": 71, + "checks_expected": 72, }, ) self.assertEqual(stderr, "report: AGT-11 added as PASS\n") @@ -917,7 +918,7 @@ def test_only_phase_1_checks_count_toward_what_a_draft_is_expected_to_hold(self) def test_an_agent_only_check_is_added_beside_the_engines(self) -> None: code, envelope, _ = self.add_check(self.draft, agent_check()) self.assertEqual(code, 0) - self.assertEqual(envelope["data"]["checks_recorded"], 70) + self.assertEqual(envelope["data"]["checks_recorded"], 71) self.assertEqual(self.check("AGT-11")["method"], "agent") @@ -1548,20 +1549,20 @@ def test_a_new_draft_has_every_family_remaining(self) -> None: self.assertEqual( (data["status"], data["eval_mode"], data["confirmation_policy"]), ("draft", False, "interactive") ) - self.assertEqual((data["checks_recorded"], data["checks_expected"]), (0, 71)) + self.assertEqual((data["checks_recorded"], data["checks_expected"]), (0, 72)) self.assertEqual(data["families_done"], "") self.assertEqual( list(data["families_remaining"]), ["IDX", "AGT", "DOC", "CMD", "TOOL", "CI", "BASE", "CON", "EXEC", "TST", "NAV", "SEC", "HYG"], ) - self.assertEqual(data["families_remaining"]["CI"], "CI-01 CI-02 CI-03 CI-04") + self.assertEqual(data["families_remaining"]["CI"], "CI-01 CI-02 CI-03 CI-04 CI-05") self.assertEqual((data["findings"], data["findings_pending"], data["blockers_pending"]), (0, 0, 0)) - self.assertEqual((data["ready_to_finalise"], data["blocking"]), (False, "71 of 71 checks are not recorded")) + self.assertEqual((data["ready_to_finalise"], data["blocking"]), (False, "72 of 72 checks are not recorded")) self.assertEqual(data["pending"], []) self.assertNotIn("pending_omitted", data) self.assertFalse(envelope["truncated"]) self.assertEqual( - stderr, "report: 0 of 71 checks recorded; not ready to finalise: 71 of 71 checks are not recorded\n" + stderr, "report: 0 of 72 checks recorded; not ready to finalise: 72 of 72 checks are not recorded\n" ) def test_recording_a_check_takes_it_off_the_remaining_list(self) -> None: @@ -1569,17 +1570,17 @@ def test_recording_a_check_takes_it_off_the_remaining_list(self) -> None: self.add_check(draft, agent_check(id="CI-04", verdict="FAIL", evidence=[])) data = self.data(draft) self.assertEqual(data["checks_recorded"], 1) - self.assertEqual(data["families_remaining"]["CI"], "CI-01 CI-02 CI-03") + self.assertEqual(data["families_remaining"]["CI"], "CI-01 CI-02 CI-03 CI-05") self.assertNotIn("CI-04", " ".join(data["families_remaining"].values())) def test_an_engine_draft_lists_what_is_left_and_the_pending_findings_blockers_first(self) -> None: draft = self.engine_draft() data = self.data(draft) - self.assertEqual((data["checks_recorded"], data["checks_expected"]), (69, 71)) + self.assertEqual((data["checks_recorded"], data["checks_expected"]), (70, 72)) self.assertEqual(data["families_remaining"], {"AGT": "AGT-11", "CI": "CI-04"}) self.assertEqual(data["families_done"], "IDX DOC CMD TOOL BASE CON EXEC TST NAV SEC HYG") self.assertEqual((data["findings"], data["findings_pending"], data["blockers_pending"]), (24, 24, 4)) - self.assertEqual(data["blocking"], "2 of 71 checks are not recorded; 24 findings are pending") + self.assertEqual(data["blocking"], "2 of 72 checks are not recorded; 24 findings are pending") findings = read_json(draft)["findings"] blockers = [f for f in findings if f["severity"] == "BLOCKER"] self.assertEqual(data["pending"][: len(blockers)], [f"{f['id']} BLOCKER {f['key']}" for f in blockers]) @@ -1603,10 +1604,10 @@ def test_what_was_left_out_of_the_list_is_said_in_the_envelope(self) -> None: self.assertEqual(code, 0) self.assertIs(envelope["truncated"], True) - def test_a_report_of_70_checks_prints_40_lines_or_fewer(self) -> None: + def test_a_report_of_71_checks_prints_40_lines_or_fewer(self) -> None: draft = self.engine_draft() self.add_check(draft, agent_check()) - self.assertEqual(len(read_json(draft)["checks"]), 70) + self.assertEqual(len(read_json(draft)["checks"]), 71) self.assertLessEqual(self.stdout_lines("status", str(draft)), 40) def test_no_draft_can_make_it_print_more_than_40_lines(self) -> None: @@ -1626,11 +1627,11 @@ def test_no_draft_can_make_it_print_more_than_40_lines(self) -> None: def test_it_is_ready_when_every_check_is_recorded_and_every_finding_decided(self) -> None: draft = self.complete_engine_draft() data = self.data(draft) - self.assertEqual((data["checks_recorded"], data["checks_expected"]), (71, 71)) + self.assertEqual((data["checks_recorded"], data["checks_expected"]), (72, 72)) self.assertEqual((data["families_remaining"], data["findings_pending"], data["blockers_pending"]), ({}, 0, 0)) self.assertEqual((data["ready_to_finalise"], data["blocking"]), (True, "")) self.assertEqual(data["pending"], []) - self.assertEqual(self.status(draft)[2], "report: 71 of 71 checks recorded; ready to finalise\n") + self.assertEqual(self.status(draft)[2], "report: 72 of 72 checks recorded; ready to finalise\n") def test_one_pending_finding_is_enough_to_hold_it_back(self) -> None: draft = self.complete_engine_draft() @@ -1734,7 +1735,7 @@ def test_it_writes_the_final_report_and_its_markdown_as_new_files_and_leaves_the self.assertEqual(envelope["data"]["draft"], str(self.draft)) self.assertEqual(envelope["data"]["status"], "final") self.assertEqual(envelope["data"]["finalised_at"], "2026-10-10T11:45:30+00:00") - self.assertEqual((envelope["data"]["checks"], envelope["data"]["findings"]), (71, 24)) + self.assertEqual((envelope["data"]["checks"], envelope["data"]["findings"]), (72, 24)) self.assertEqual(envelope["data"]["summary"], read_json(final)["summary"]) self.assertRegex( stderr, diff --git a/audit/tools/tests/test_summary.py b/audit/tools/tests/test_summary.py index 68942901..8f3ed9b4 100644 --- a/audit/tools/tests/test_summary.py +++ b/audit/tools/tests/test_summary.py @@ -203,11 +203,13 @@ def test_the_score_and_the_scored_count_equal_the_engines_own(self) -> None: self.assertEqual(len(drafts), 1, raw_path.name) raw = json.loads(raw_path.read_text(encoding="utf-8")) draft = json.loads(drafts[0].read_text(encoding="utf-8")) - # A check the engine's ruleset blocked did not run: the draft scores it as the catalogue does, the engine's - # own number does not, so the engine's number is what the draft's is with that check left unscored. + # A check the engine's ruleset blocked did not run, and a check newer than its ruleset (CI-05) is not in + # its report: the draft scores both as the catalogue does, the engine's own number does not, so the + # engine's number is what the draft's is with those checks left unscored. blocked = {check["id"] for check in raw["checks"] if check["status"] == "blocked"} + reported = {check["id"] for check in raw["checks"]} | set(raw["not_applicable"]) for check in draft["checks"]: - if check["id"] in blocked: + if check["id"] in blocked or check["id"] not in reported: check["scored"] = False ours = _summary.compute_summary(draft) with self.subTest(report=raw_path.name, stamp=stamp): diff --git a/audit/tools/tests/test_validate_report.py b/audit/tools/tests/test_validate_report.py index d00babe7..0c0c75a8 100644 --- a/audit/tools/tests/test_validate_report.py +++ b/audit/tools/tests/test_validate_report.py @@ -227,7 +227,7 @@ def test_a_playbook_the_catalogue_names_needs_no_file_and_a_null_playbook_is_fin self.assertEqual(vr.validate_document(report).errors, []) def test_a_catalogue_of_another_major_or_minor_version_is_a_warning_only(self) -> None: - for version, warned in (("0.2.0", False), ("0.2.7", False), ("0.1.0", True), ("0.3.0", True), ("1.2.0", True)): + for version, warned in (("0.3.0", False), ("0.3.7", False), ("0.2.8", True), ("0.4.0", True), ("1.3.0", True)): with self.subTest(version=version): report = base() report["provenance"]["catalogue_version"] = version diff --git a/dev/eval/expected/fixtures/ci-01-no-pr-trigger.json b/dev/eval/expected/fixtures/ci-01-no-pr-trigger.json index 3c983bb5..e5714456 100644 --- a/dev/eval/expected/fixtures/ci-01-no-pr-trigger.json +++ b/dev/eval/expected/fixtures/ci-01-no-pr-trigger.json @@ -1,8 +1,8 @@ { "expected_version": "0.1", "repo_label": "ci-01-no-pr-trigger", - "repo_sha": "92a9a410912a4aae807d101fcfb2b5f9c90949ef", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-01-no-pr-trigger' (brownfield-recipes/deterministic-audit/tests/fixtures/ci-01-no-pr-trigger), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "repo_sha": "478e463754690702307d993254f646e25f81ba4f", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-01-no-pr-trigger' (audit/engine/tests/fixtures/ci-01-no-pr-trigger), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CI-01", diff --git a/dev/eval/expected/fixtures/ci-02-parity-break.json b/dev/eval/expected/fixtures/ci-02-parity-break.json index 6100bc43..e6afe83b 100644 --- a/dev/eval/expected/fixtures/ci-02-parity-break.json +++ b/dev/eval/expected/fixtures/ci-02-parity-break.json @@ -1,8 +1,8 @@ { "expected_version": "0.1", "repo_label": "ci-02-parity-break", - "repo_sha": "a981839b5321dbd576486e245493b6c2ec59661d", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-02-parity-break' (brownfield-recipes/deterministic-audit/tests/fixtures/ci-02-parity-break), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "repo_sha": "9b64c9236d97edf457bf81a1ddfb9a26caf43d8a", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-02-parity-break' (audit/engine/tests/fixtures/ci-02-parity-break), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CI-02", diff --git a/dev/eval/expected/fixtures/ci-03-ci-only-step.json b/dev/eval/expected/fixtures/ci-03-ci-only-step.json index 9441e60a..2f8b9691 100644 --- a/dev/eval/expected/fixtures/ci-03-ci-only-step.json +++ b/dev/eval/expected/fixtures/ci-03-ci-only-step.json @@ -1,8 +1,8 @@ { "expected_version": "0.1", "repo_label": "ci-03-ci-only-step", - "repo_sha": "b8aeddc9d70a7bd4dc7a2e81d3010e88991d861b", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-03-ci-only-step' (brownfield-recipes/deterministic-audit/tests/fixtures/ci-03-ci-only-step), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "repo_sha": "be0baaaf44f34d1e694b1899d6d72df3d2e70dee", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-03-ci-only-step' (audit/engine/tests/fixtures/ci-03-ci-only-step), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CI-03", diff --git a/dev/eval/expected/fixtures/ci-05-ci-only-lint.json b/dev/eval/expected/fixtures/ci-05-ci-only-lint.json new file mode 100644 index 00000000..a6e6b6bd --- /dev/null +++ b/dev/eval/expected/fixtures/ci-05-ci-only-lint.json @@ -0,0 +1,14 @@ +{ + "expected_version": "0.1", + "repo_label": "ci-05-ci-only-lint", + "repo_sha": "1e83b2a77847db141437468c274eec9a55857711", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-05-ci-only-lint' (audit/engine/tests/fixtures/ci-05-ci-only-lint), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "checks": [ + { + "id": "CI-05", + "acceptable_verdicts": [ + "PARTIAL" + ] + } + ] +} diff --git a/dev/eval/expected/fixtures/ci-05-no-workflow-lint.json b/dev/eval/expected/fixtures/ci-05-no-workflow-lint.json new file mode 100644 index 00000000..24e69d94 --- /dev/null +++ b/dev/eval/expected/fixtures/ci-05-no-workflow-lint.json @@ -0,0 +1,14 @@ +{ + "expected_version": "0.1", + "repo_label": "ci-05-no-workflow-lint", + "repo_sha": "a3361285b063a4b4771f2c2ecbc8235dd618be60", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-05-no-workflow-lint' (audit/engine/tests/fixtures/ci-05-no-workflow-lint), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "checks": [ + { + "id": "CI-05", + "acceptable_verdicts": [ + "FAIL" + ] + } + ] +} diff --git a/dev/eval/expected/fixtures/ci-ok-gh.json b/dev/eval/expected/fixtures/ci-ok-gh.json index b36ef5d1..93fe94f0 100644 --- a/dev/eval/expected/fixtures/ci-ok-gh.json +++ b/dev/eval/expected/fixtures/ci-ok-gh.json @@ -1,8 +1,8 @@ { "expected_version": "0.1", "repo_label": "ci-ok-gh", - "repo_sha": "3ff548c12cb0f3f308e840f2f28ed9904edafb86", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-ok-gh' (brownfield-recipes/deterministic-audit/tests/fixtures/ci-ok-gh), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "repo_sha": "c507095f93c759de05601eca66a5f1ee4ad56685", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-ok-gh' (audit/engine/tests/fixtures/ci-ok-gh), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CI-01", @@ -21,6 +21,12 @@ "acceptable_verdicts": [ "PASS" ] + }, + { + "id": "CI-05", + "acceptable_verdicts": [ + "PASS" + ] } ] } diff --git a/docs/explanation/contributor-deep-dive/03-check-catalogue-and-rubrics.md b/docs/explanation/contributor-deep-dive/03-check-catalogue-and-rubrics.md index 8df23b35..dc31b30f 100644 --- a/docs/explanation/contributor-deep-dive/03-check-catalogue-and-rubrics.md +++ b/docs/explanation/contributor-deep-dive/03-check-catalogue-and-rubrics.md @@ -37,7 +37,7 @@ checks, all informational and unscored). The catalogue itself, Where the entries come from: - **Engine checks** are seeded from the engine's own list, `python3 -m audit --list-checks --format json` ([`cli.py`](../../../audit/engine/audit/cli.py)). After seeding the file is edited by hand. The engine's ID is the check's ID, and `engine_id` repeats it. -- **Agent-only checks** are the ones no deterministic rule can settle. Today there are two: `AGT-11` (instructions are specific and not inferable, for [REQ-SH-015](../../../spec/readiness-requirements.md#req-sh-015--instructions-are-specific-and-not-inferable)) and `CI-04` (CI enforces the same gate as the local check, for [REQ-SH-035](../../../spec/readiness-requirements.md#req-sh-035--ci-enforces-the-gate)). Each took the next free number in its family when it was added. An engine check added later takes the next free number after it, and the test fails only if an engine check takes an agent check's ID. +- **Agent-only checks** are the ones no deterministic rule can settle. Today there are two: `AGT-11` (instructions are specific and not inferable, for [REQ-SH-015](../../../spec/readiness-requirements.md#req-sh-015--instructions-are-specific-and-not-inferable)) and `CI-04` (CI enforces the same gate as the local check, for [REQ-SH-035](../../../spec/readiness-requirements.md#req-sh-035--ci-enforces-the-gate)). Each took the next free number in its family when it was added. An engine check added later takes the next free number after it (`CI-05` follows the agent-only `CI-04`), and the test fails only if an engine check takes an agent check's ID. - **`status`** is the engine's own status for an engine check. An agent-only check is `planned` until its rubric exists. A check that is planned or blocked is reported `UNKNOWN`. `scored` is also the engine's: `CON-01` was `blocked` and unscored until catalogue 0.2.0, when the engine began to read the prohibitions section. ## 2. Methods diff --git a/docs/reference/how-the-checks-map-to-the-framework.md b/docs/reference/how-the-checks-map-to-the-framework.md index 01f5bb81..56b09dad 100644 --- a/docs/reference/how-the-checks-map-to-the-framework.md +++ b/docs/reference/how-the-checks-map-to-the-framework.md @@ -31,7 +31,7 @@ To go the other way, start from a framework heading, find the requirements whose ## Every check, generated -All 71 catalogue checks, each with its requirement(s) and the framework heading(s) the requirement's +All 72 catalogue checks, each with its requirement(s) and the framework heading(s) the requirement's `Source` field links. Generated 2026-10-19 from `catalogue.json`'s `req_ids` and `readiness-requirements.md`'s `Source` fields — the same two steps as the worked example above, run once per check rather than by hand. 14 checks are informational or an emitter and trace to no requirement @@ -58,6 +58,7 @@ per check rather than by hand. 14 checks are informational or an emitter and tra | `CI-02` | Local↔CI parity | BLOCKER | yes | [REQ-SH-034](../../spec/readiness-requirements.md#req-sh-034--ci-runs-the-verbs) | [greenfield/README.md § 1.4 The standardized command surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs) | | `CI-03` | CI-only steps flagged | DEGRADER | yes | [REQ-SH-034](../../spec/readiness-requirements.md#req-sh-034--ci-runs-the-verbs) | [greenfield/README.md § 1.4 The standardized command surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs) | | `CI-04` | CI enforces the same gate as the local check | DEGRADER | yes | [REQ-SH-035](../../spec/readiness-requirements.md#req-sh-035--ci-enforces-the-gate) | [greenfield/README.md § 1.4 The standardized command surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs) | +| `CI-05` | Workflow files are linted | DEGRADER | yes | [REQ-SH-057](../../spec/readiness-requirements.md#req-sh-057--workflow-files-linted) | [greenfield/README.md § 4.1 Workflows](../../greenfield/README.md#41-workflows) | | `CMD-01` | Seven verbs resolvable on one runner | BLOCKER | yes | [REQ-SH-021](../../spec/readiness-requirements.md#req-sh-021--verbs-defined-on-one-runner) | [greenfield/README.md § 1.4 The standardized command surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs) | | `CMD-02` | check composes the read-only verbs | BLOCKER | yes | [REQ-SH-022](../../spec/readiness-requirements.md#req-sh-022--check-composes-the-read-only-verbs) | [greenfield/README.md § 1.4 The standardized command surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs) | | `CMD-03` | No divergent second definition | DEGRADER | yes | [REQ-SH-023](../../spec/readiness-requirements.md#req-sh-023--no-second-definition-of-the-gate) | [greenfield/README.md § 1.4 The standardized command surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs) | @@ -111,7 +112,7 @@ per check rather than by hand. 14 checks are informational or an emitter and tra | `TST-04` | Fast path documented | DEGRADER | yes | [REQ-SH-048](../../spec/readiness-requirements.md#req-sh-048--fast-path-documented) | [Phased-Approach.md § Phase 1: Agentic Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-) | | `TST-05` | Coverage configuration present | COSMETIC | yes | [REQ-SH-049](../../spec/readiness-requirements.md#req-sh-049--coverage-reporting) | [greenfield/README.md § 3.1 vitest.config.ts](../../greenfield/README.md#31-vitestconfigts--with-coverage) | -71 checks total. +72 checks total. ## Where the relation is held diff --git a/greenfield/README.md b/greenfield/README.md index 5dafe5c6..4fbb3cae 100644 --- a/greenfield/README.md +++ b/greenfield/README.md @@ -121,7 +121,7 @@ git checkout -b dev # dev is the working branch from day one "files": ["dist"], // what npm publishes — nothing else leaves the repo "publishConfig": { "access": "public" }, "scripts": { - // the seven verbs from §1.4, plus clean/build/build:full/dev + // the seven verbs from §1.4, plus clean/build/build:full/dev and lint:workflows (actionlint) "check": "npm run format:check && npm run typecheck && npm test", "prepublishOnly": "npm run check && npm run build:full", "prepare": "husky" @@ -333,7 +333,10 @@ npx husky init # NOT needed: it overwrites .husky/pre-commit with `npm test` ## Phase 4 — CI/CD & Release Management ### 4.1 Workflows -- **`.github/workflows/ci.yml`**: `npm run check` → coverage thresholds → `npm run build:full` → `npm run security`. Every step is a script you can run locally. (See [template](templates/github/workflows/ci.yml)) +- **`.github/workflows/ci.yml`**: `npm run check` → workflow lint → coverage thresholds → `npm run build:full` → `npm run security`. Every step is a script you can run locally. (See [template](templates/github/workflows/ci.yml)) +- **Workflow lint**: the workflow files are linted locally and in CI by a linter that knows the CI provider. + For GitHub Actions that is actionlint (`npm run lint:workflows`; install the binary once per machine). An agent + that edits a workflow cannot run it locally, so the linter is its only check before a push. - **`.github/workflows/enforce-dev-base.yml`**: Enforce Dev branch PR source for Main. (See [template](templates/github/workflows/enforce-dev-base.yml)) - **`.github/workflows/release-and-sync.yml`**: Release Please, npm publish via OIDC provenance, automated back-merge to dev. (See [template](templates/github/workflows/release-and-sync.yml)) - **`.github/dependabot.yml`**: Weekly automated dependency updates. (See [template](templates/github/dependabot.yml)) diff --git a/greenfield/templates/github/workflows/ci.yml b/greenfield/templates/github/workflows/ci.yml index 65f71dfa..6be3df96 100644 --- a/greenfield/templates/github/workflows/ci.yml +++ b/greenfield/templates/github/workflows/ci.yml @@ -61,6 +61,11 @@ jobs: - name: Check (format, lint, typecheck, tests) run: npm run check + # The same linter as `npm run lint:workflows`; the action installs the binary. + - name: Lint workflow files + if: ${{ runner.os == 'Linux' }} + uses: raven-actions/actionlint@v2 + # Coverage thresholds are enabled with the first real test (see vitest.config.ts). # - name: Verify Coverage Thresholds # run: npm run test:coverage diff --git a/greenfield/templates/package.json b/greenfield/templates/package.json index 56974064..14155af6 100644 --- a/greenfield/templates/package.json +++ b/greenfield/templates/package.json @@ -35,6 +35,7 @@ "test:coverage": "vitest run --coverage", "check": "npm run format:check && npm run typecheck && npm test", "security": "npm audit --audit-level=high", + "lint:workflows": "actionlint", "check:fix": "biome check --fix .", "build": "tsc", "build:full": "npm run clean && npm run build", diff --git a/implement/playbooks/README.md b/implement/playbooks/README.md index e60e2973..24374bc3 100644 --- a/implement/playbooks/README.md +++ b/implement/playbooks/README.md @@ -61,7 +61,7 @@ in IR04.1, `untrack-generated` in IR05.1, `establish-docs-contract` in IR09.1, | [`pin-execution-environment`](pin-execution-environment.md) | T2 Execution determinism | EXEC-01, EXEC-02 | yes | Drafted (IR05.1); applied and corrected in IR05.2, IR05.3 (T2 closed) | | [`untrack-generated`](untrack-generated.md) | T2 Execution determinism | EXEC-05, EXEC-07 | yes | Drafted (IR05.1); applied and corrected in IR05.2, IR05.3 (T2 closed) | | [`env-inventory-and-secrets`](env-inventory-and-secrets.md) | T3 Secrets and env | EXEC-04, SEC-01, SEC-02, SEC-03 | yes | Applied and corrected (IR06.1-IR06.3) | -| [`add-ci-parity`](add-ci-parity.md) | T4 CI parity, hooks, commits | CI-01, CI-02, CI-03, CI-04 | yes | Drafted (IR07.1); applied and corrected in IR07.2, IR07.3 (T4 closed) | +| [`add-ci-parity`](add-ci-parity.md) | T4 CI parity, hooks, commits | CI-01, CI-02, CI-03, CI-04, CI-05 | yes | Drafted (IR07.1); applied and corrected in IR07.2, IR07.3 (T4 closed) | | [`add-hooks-and-commit-rules`](add-hooks-and-commit-rules.md) | T4 CI parity, hooks, commits | TOOL-05, TOOL-06 | yes | Drafted (IR07.1); applied and corrected in IR07.2, IR07.3 (T4 closed) | | [`add-baselines-and-ratchet`](add-baselines-and-ratchet.md) | T5 Tests and baselines | BASE-01, BASE-02, BASE-03, BASE-04, TST-02, TST-03, TST-04, TST-05 | yes | Drafted (IR08.1); applied in IR08.2, IR08.3 | | [`establish-docs-contract`](establish-docs-contract.md) | T6 Docs and navigation | DOC-01, DOC-02, DOC-03, DOC-04, NAV-01, NAV-02, NAV-03, NAV-04, NAV-05 | yes | Drafted (IR09.1); applied in IR09.2, IR09.3 | diff --git a/implement/tools/progress.py b/implement/tools/progress.py index d96340e7..078b78a5 100644 --- a/implement/tools/progress.py +++ b/implement/tools/progress.py @@ -77,7 +77,7 @@ def _span(prefix: str, first: int, last: int, *, skip: Sequence[int] = ()) -> tu ("T1", "Command surface and quality tools", _span("CMD", 1, 4) + _span("TOOL", 1, 4)), ("T2", "Execution determinism", _span("EXEC", 1, 7, skip=(4,))), ("T3", "Secrets and env", _span("SEC", 1, 3) + ("EXEC-04",)), - ("T4", "CI parity, hooks, commits", _span("CI", 1, 4) + ("TOOL-05", "TOOL-06")), + ("T4", "CI parity, hooks, commits", _span("CI", 1, 5) + ("TOOL-05", "TOOL-06")), ("T5", "Tests and baselines", _span("TST", 1, 5) + _span("BASE", 1, 4)), ("T6", "Docs and navigation", _span("DOC", 1, 4) + _span("NAV", 1, 5)), ("T7", "AGENTS.md and governance", _span("AGT", 1, 11) + _span("CON", 1, 4)), diff --git a/spec/readiness-requirements.md b/spec/readiness-requirements.md index 63f01584..d5e8fc0e 100644 --- a/spec/readiness-requirements.md +++ b/spec/readiness-requirements.md @@ -305,6 +305,16 @@ On every pull request the CI pipeline MUST run the verbs that make up `check` an - **Source:** [greenfield/README.md § 1.4 The standardized command surface](../greenfield/README.md#14-the-standardized-command-surface--seven-verbs) - **Checks:** CI-04 +### REQ-SH-057 — Workflow files linted + +The repository's CI workflow files MUST be checked by a linter that understands the CI provider. The linter MUST +be runnable locally, through the command surface or a hook, as well as in CI. + +- **Phase:** 1 +- **Verification:** automated +- **Source:** [greenfield/README.md § 4.1 Workflows](../greenfield/README.md#41-workflows) +- **Checks:** CI-05 + ## Baselines and the ratchet ### REQ-BF-001 — Quality baseline diff --git a/spec/traceability.md b/spec/traceability.md index 1bdf9c70..ca2161c2 100644 --- a/spec/traceability.md +++ b/spec/traceability.md @@ -6,7 +6,7 @@ Check IDs and severities are those of the [check catalogue](../audit/catalogue.j ## Checks to requirements -71 checks: 69 of the audit engine and 2 that only an agent runs. 57 scored checks trace to at least one requirement. The other 14 are informational or an emitter and trace to none. +72 checks: 70 of the audit engine and 2 that only an agent runs. 58 scored checks trace to at least one requirement. The other 14 are informational or an emitter and trace to none. | Check | Title | Severity | Scored | Requirements | |---|---|---|---|---| @@ -42,6 +42,7 @@ Check IDs and severities are those of the [check catalogue](../audit/catalogue.j | `CI-02` | Local↔CI parity | BLOCKER | yes | [REQ-SH-034](readiness-requirements.md#req-sh-034--ci-runs-the-verbs) | | `CI-03` | CI-only steps flagged | DEGRADER | yes | [REQ-SH-034](readiness-requirements.md#req-sh-034--ci-runs-the-verbs) | | `CI-04` | CI enforces the same gate as the local check | DEGRADER | yes (agent) | [REQ-SH-035](readiness-requirements.md#req-sh-035--ci-enforces-the-gate) | +| `CI-05` | Workflow files are linted | DEGRADER | yes | [REQ-SH-057](readiness-requirements.md#req-sh-057--workflow-files-linted) | | `BASE-01` | Baseline artifact exists | DEGRADER | yes | [REQ-BF-001](readiness-requirements.md#req-bf-001--quality-baseline) | | `BASE-02` | Current violation counts measured | DEGRADER | yes | [REQ-BF-002](readiness-requirements.md#req-bf-002--violation-count-recorded) | | `BASE-03` | No-new-violations mechanism | DEGRADER | yes | [REQ-BF-003](readiness-requirements.md#req-bf-003--no-new-violations) | @@ -84,7 +85,7 @@ Check IDs and severities are those of the [check catalogue](../audit/catalogue.j ## Requirements without a check -5 of the 60 requirements have no check. All five are attested, and they are the engine's unattested list: the report shows them as unattested and they earn no credit. +5 of the 61 requirements have no check. All five are attested, and they are the engine's unattested list: the report shows them as unattested and they earn no credit. | Requirement | Title | Verification | Why | |---|---|---|---| @@ -96,7 +97,7 @@ Check IDs and severities are those of the [check catalogue](../audit/catalogue.j ## Derived requirements -20 of the 60 requirements go beyond what the framework documents say. They follow from the Phase 1 objective, "Make the repository executable, discoverable, and safe for agent interaction", or they come from the audit engine's rule packs. Each is still tied to the nearest framework heading. These are the requirements to review first. +20 of the 61 requirements go beyond what the framework documents say. They follow from the Phase 1 objective, "Make the repository executable, discoverable, and safe for agent interaction", or they come from the audit engine's rule packs. Each is still tied to the nearest framework heading. These are the requirements to review first. | Requirement | Title | What the framework does not say | |---|---|---| From 22ba4a6283cea03fd2a1bbff57eea75737e21235 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 11:47:26 +0530 Subject: [PATCH 012/111] fix(dev): name more harnesses and move the first-run trigger to the top A live test showed two gaps. opencode read AGENTS.md but skipped the offer, which sat in a section further down; the rule now opens AGENTS.md and applies before any edit. Antigravity followed the offer but was not a known harness, so it picked `gemini` (the wrong file) and then probed its own environment. dev/mcp_config.py now knows `antigravity` (prints `agy mcp add`, checked with agy 1.5.1 under a scratch HOME) and `other` (prints the plain entry). A new `detect` reads CLAUDECODE, ANTIGRAVITY_AGENT and OPENCODE. A harness started inside another inherits the outer one's variable, so more than one match means "ask the person"; the how-to no longer says CLAUDECODE=1 confirms Claude Code. Roadmap-Step: B2 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 11 ++-- dev/mcp_config.py | 58 +++++++++++++++++-- dev/tests/test_mcp_config_tool.py | 49 +++++++++++++++- .../adrs/0006-first-run-agent-tool-offer.md | 11 ++-- docs/how-to/set-up-agent-tools.md | 18 ++++-- docs/reference/dev-tooling.md | 5 +- 6 files changed, 132 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f7d3edeb..be001e10 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,6 +6,11 @@ coding agents, plus tools that check it. `audit/` holds the audit recipe, its to rules: `audit/AGENTS.md`, `implement/AGENTS.md` and `audit/engine/AGENTS.md` (the strictest). Read the file for a folder before you change it. +**Before you edit any file in this repository**, run `python3 dev/mcp_config.py status --harness `. +If it prints `not asked`, first make the one-time offer in `docs/how-to/set-up-agent-tools.md` ("For agents"). +`python3 dev/mcp_config.py detect` helps you name your harness. Skip this for an audit or implement run on another +repository. Never install a server or write a harness config without the person's consent. + ## Users: audit or implement a repository - When asked to audit a repository, follow `audit/SKILL.md` from this repository's root. @@ -13,12 +18,6 @@ folder before you change it. The rest of this file is for changes to this repository. -## First session: offer the agent tools - -Before your first change in a clone, run `python3 dev/mcp_config.py status --harness `. If it prints -`not asked`, make the one-time offer in `docs/how-to/set-up-agent-tools.md` ("For agents"). Ask once. Never install a -server or write a harness config without the person's consent. - ## Commands The framework's seven verbs run through the root `Makefile`. CI runs the same commands. diff --git a/dev/mcp_config.py b/dev/mcp_config.py index 697206db..57cc4e08 100644 --- a/dev/mcp_config.py +++ b/dev/mcp_config.py @@ -3,6 +3,7 @@ """Translate this repository's ``.mcp.json`` into each harness's own MCP config, and record the first-run answer. + mcp_config.py detect which harness the environment names, if exactly one mcp_config.py show --harness [--scope project|user] print where the entry goes and the entry itself mcp_config.py write --harness [--scope project|user] merge the entry into that file (needs consent) mcp_config.py check is each server's command on PATH; how to install it @@ -10,7 +11,8 @@ mcp_config.py record --harness --answer accepted|declined [--scope project|user] ``.mcp.json`` is the one source (ADR 0006). Claude Code and VS Code read it as it is, so ``write`` has nothing to -do for them. For the other harnesses, ``write`` merges the entry and never replaces an entry that differs: it +do for them. Antigravity keeps servers only at user level, through ``agy mcp add``; the tool prints that command. +``--harness other`` prints the plain entry for a harness this tool does not know. For the other harnesses, ``write`` merges the entry and never replaces an entry that differs: it stops and prints both. The answer goes into ``.agentic-readiness/setup.json``, which git ignores. Nothing here installs a server or runs one, except ``check``, which runs `` --version`` for a command on ``PATH``. @@ -24,23 +26,33 @@ import datetime import json import re +import os +import shlex import shutil import subprocess import sys import tomllib from dataclasses import dataclass from pathlib import Path -from typing import Any, Sequence +from typing import Any, Mapping, Sequence REPO = Path(__file__).resolve().parents[1] SOURCE = ".mcp.json" RECORD = Path(".agentic-readiness") / "setup.json" -HARNESSES = ("claude-code", "vscode", "cursor", "gemini", "opencode", "codex") +HARNESSES = ("claude-code", "vscode", "cursor", "gemini", "opencode", "codex", "antigravity", "other") SCOPES = ("project", "user") ANSWERS = ("accepted", "declined") BARE_KEY = re.compile(r"^[A-Za-z0-9_-]+$") +#: The environment variable each harness sets for the commands its agent runs (checked 2026-11-09). A harness +#: started from inside another one inherits the outer one's variable too, so more than one match means "ask". +ENV_MARKERS: dict[str, str] = { + "claude-code": "CLAUDECODE", + "antigravity": "ANTIGRAVITY_AGENT", + "opencode": "OPENCODE", +} + #: Install routes from each server's own documentation. ``--skip-config`` stops the upstream installer from #: writing agent configs for every client it finds; this tool writes the one entry the person agreed to. INSTALL_HINTS: dict[str, tuple[str, ...]] = { @@ -90,7 +102,11 @@ def target(harness: str, scope: str, root: Path, home: Path) -> Target: if harness == "claude-code": if scope == "project": return Target(None, "Claude Code reads .mcp.json. On the next start it asks you to approve each server.") - return Target(None, "Run: claude mcp add --scope user -- [args]. Claude Code keeps it.") + return Target(None, "Claude Code keeps user-level servers itself. Run:") + if harness == "antigravity": + return Target(None, "Antigravity keeps MCP servers at user level, for every repository. Run:") + if harness == "other": + return Target(None, "Add this entry to your harness's MCP config by hand. It is in the .mcp.json format:") if harness == "vscode": if scope == "project": return Target(None, "VS Code reads .mcp.json. It starts the servers once you trust the folder.") @@ -129,6 +145,25 @@ def translate(harness: str, entry: dict[str, Any]) -> dict[str, Any]: return result +def native_commands(harness: str, scope: str, servers: dict[str, dict[str, Any]]) -> list[str]: + """What the person runs or pastes for a harness that keeps its config itself; empty when there is nothing.""" + if harness == "other": + return json.dumps({"mcpServers": servers}, indent=2).splitlines() + if harness == "antigravity" or (harness == "claude-code" and scope == "user"): + prefix = ["agy", "mcp", "add"] if harness == "antigravity" else ["claude", "mcp", "add", "--scope", "user"] + lines = [] + for name, entry in servers.items(): + env = [part for key, value in entry.get("env", {}).items() for part in ("--env", f"{key}={value}")] + lines.append(shlex.join([*prefix, *env, name, "--", entry["command"], *entry.get("args", [])])) + return lines + return [] + + +def detect(environ: Mapping[str, str]) -> list[str]: + """The harnesses whose marker variable is set, in :data:`ENV_MARKERS` order.""" + return [harness for harness, variable in ENV_MARKERS.items() if environ.get(variable)] + + def _json_path(harness: str) -> tuple[str, ...]: """The keys that lead to the server map in a harness's JSON config.""" return ("mcp", "servers") if harness == "opencode" else ("mcpServers",) @@ -275,10 +310,13 @@ def main(argv: Sequence[str] | None = None, *, root: Path = REPO, home: Path | N if action == "record": command.add_argument("--answer", choices=ANSWERS, required=True) sub.add_parser("check") + sub.add_parser("detect") args = parser.parse_args(argv) home = home if home is not None else Path.home() try: + if args.action == "detect": + return _print_detect(detect(os.environ)) if args.action == "status": entry = read_record(root).get(args.harness) print(f"asked: {entry['answer']} on {entry['date']}" if entry else "not asked") @@ -293,6 +331,8 @@ def main(argv: Sequence[str] | None = None, *, root: Path = REPO, home: Path | N where = target(args.harness, args.scope, root, home) if where.path is None: print(where.native) + for line in native_commands(args.harness, args.scope, servers): + print(f" {line}" if args.harness != "other" else line) return 0 if args.action == "show": print(f"# {where.path}") @@ -309,6 +349,16 @@ def main(argv: Sequence[str] | None = None, *, root: Path = REPO, home: Path | N return 2 +def _print_detect(found: list[str]) -> int: + if len(found) == 1: + print(f"{found[0]} (confirm it with what you know about yourself)") + elif found: + print(f"ambiguous: {', '.join(found)}; one harness may run inside another. Ask the person.") + else: + print("unknown: no harness variable set. Ask the person.") + return 0 + + def _print_check(servers: dict[str, dict[str, Any]]) -> int: missing = False for name, version, note in check_commands(servers): diff --git a/dev/tests/test_mcp_config_tool.py b/dev/tests/test_mcp_config_tool.py index 04570050..e3ea44d8 100644 --- a/dev/tests/test_mcp_config_tool.py +++ b/dev/tests/test_mcp_config_tool.py @@ -82,7 +82,7 @@ def test_codex_quotes_a_key_that_is_not_bare(self) -> None: class Targets(Workspace): def test_native_harnesses_have_nothing_to_write(self) -> None: - for harness in ("claude-code", "vscode"): + for harness in ("claude-code", "vscode", "antigravity", "other"): for scope in ("project", "user"): with self.subTest(harness=harness, scope=scope): self.assertIsNone(mcp_config.target(harness, scope, self.root, self.home).path) @@ -216,6 +216,53 @@ def test_present_command_reports_its_version(self) -> None: self.assertIn("codebase-memory-mcp 9.9.9", out) +class HarnessesThatKeepTheirOwnConfig(Workspace): + servers = {"codebase-memory-mcp": SERVER, "x": WITH_ARGS} + + def test_antigravity_prints_agy_mcp_add_and_writes_nothing(self) -> None: + code, out, _ = self.run_tool("write", "--harness", "antigravity") + self.assertEqual(code, 0) + self.assertIn("agy mcp add codebase-memory-mcp -- codebase-memory-mcp\n", out) + self.assertIn("agy mcp add --env MODE=fast x -- tool --stdio\n", out) + self.assertEqual(sorted(p.name for p in self.root.iterdir()), [".mcp.json"]) + + def test_claude_code_user_scope_prints_claude_mcp_add(self) -> None: + out = self.run_tool("show", "--harness", "claude-code", "--scope", "user")[1] + self.assertIn("claude mcp add --scope user codebase-memory-mcp -- codebase-memory-mcp\n", out) + + def test_other_prints_the_plain_entry(self) -> None: + out = self.run_tool("show", "--harness", "other")[1] + entry = json.loads(out[out.index("{") :]) + self.assertEqual(entry, {"mcpServers": self.servers}) + + def test_other_can_be_recorded(self) -> None: + self.run_tool("record", "--harness", "other", "--answer", "declined") + self.assertTrue(self.run_tool("status", "--harness", "other")[1].startswith("asked: declined")) + + +class Detect(Workspace): + def detect_with(self, environ: dict[str, str]) -> str: + with mock.patch.dict(os.environ, environ, clear=True): + return self.run_tool("detect")[1] + + def test_one_marker_names_the_harness(self) -> None: + for variable, harness in ( + ("CLAUDECODE", "claude-code"), + ("ANTIGRAVITY_AGENT", "antigravity"), + ("OPENCODE", "opencode"), + ): + with self.subTest(variable=variable): + self.assertTrue(self.detect_with({variable: "1"}).startswith(f"{harness} ")) + + def test_nested_harnesses_are_ambiguous(self) -> None: + out = self.detect_with({"CLAUDECODE": "1", "ANTIGRAVITY_AGENT": "1"}) + self.assertTrue(out.startswith("ambiguous: claude-code, antigravity")) + self.assertIn("Ask the person", out) + + def test_no_marker_is_unknown(self) -> None: + self.assertTrue(self.detect_with({"AI_AGENT": "something"}).startswith("unknown")) + + class SourceErrors(Workspace): def test_missing_source_is_a_usage_error(self) -> None: (self.root / ".mcp.json").unlink() diff --git a/docs/architecture/adrs/0006-first-run-agent-tool-offer.md b/docs/architecture/adrs/0006-first-run-agent-tool-offer.md index 02986a94..6612355d 100644 --- a/docs/architecture/adrs/0006-first-run-agent-tool-offer.md +++ b/docs/architecture/adrs/0006-first-run-agent-tool-offer.md @@ -21,6 +21,7 @@ Six harnesses are in use. Each reads a different project file, and they act diff * Cursor (`.cursor/mcp.json`), Gemini CLI (`.gemini/settings.json`), opencode (`opencode.json`) and Codex CLI (`.codex/config.toml`) do not read `.mcp.json`. * opencode connects a project server with no prompt. +* Antigravity keeps its servers at user level only (`agy mcp add`, checked with `agy` 1.5.1). An MCP server is a program that runs on the person's machine. Many people clone this repository only to audit their own code; they do not need the server here. The owner decided on one committed file, not one per harness. @@ -34,18 +35,20 @@ their own code; they do not need the server here. The owner decided on one commi one-place pin rule (ADR 0003, `dev/tests/test_tool_pins.py`) needs no exception. 2. **Approval stays with each person.** The committed `.claude/settings.json` never sets `enableAllProjectMcpServers` or `enabledMcpjsonServers`. `dev/tests/test_mcp_config.py` checks this. -3. **A one-time offer, from `AGENTS.md`.** Every harness reads `AGENTS.md`. Before the first change in a clone, +3. **A one-time offer, from the top of `AGENTS.md`.** Every harness reads `AGENTS.md`. Before it edits a file, the agent runs `python3 dev/mcp_config.py status --harness `. If it prints `not asked`, the agent makes the offer in [Set up the agent tools](../../how-to/set-up-agent-tools.md). It makes no offer for an audit or an implement run on another repository. -4. **The agent names its harness.** It uses its own knowledge, confirmed by an environment variable where one is - known (`CLAUDECODE=1`). If it is not sure, it asks. The tool takes `--harness`; it does not guess. +4. **The agent names its harness.** It uses its own knowledge, then `dev/mcp_config.py detect`. That reads the + variable each harness sets (`CLAUDECODE`, `ANTIGRAVITY_AGENT`, `OPENCODE`). A nested harness inherits the outer + one's variable, so more than one match means "ask the person". An unlisted harness uses `--harness other`. 5. **The answer is recorded per harness, outside git.** `.agentic-readiness/setup.json` is git-ignored. A declined offer is recorded too, so the agent asks once per harness and clone. 6. **Project scope by default.** [`dev/mcp_config.py`](../../../dev/mcp_config.py) translates `.mcp.json` into each harness's format and merges it into the project file, or the user file with `--scope user`. Git ignores the project files it writes. It never replaces a different entry with the same name. For Claude Code and VS Code it - writes nothing and explains the approval. + writes nothing and explains the approval. For Antigravity it prints the `agy mcp add` command; for `other`, the + plain entry. 7. **Nothing is installed by the tool or the agent.** `check` runs ` --version` for a command on `PATH`. For a missing command it prints the upstream install routes. They carry `--skip-config`, so the installer writes no config into other harnesses. The person runs the installer. diff --git a/docs/how-to/set-up-agent-tools.md b/docs/how-to/set-up-agent-tools.md index 2a9366a1..771b5d1f 100644 --- a/docs/how-to/set-up-agent-tools.md +++ b/docs/how-to/set-up-agent-tools.md @@ -18,7 +18,7 @@ This page uses these terms: | Term | Meaning | | --- | --- | -| Harness | The program that runs the agent: Claude Code, VS Code with Copilot, Cursor, Gemini CLI, opencode or Codex CLI | +| Harness | The program that runs the agent: Claude Code, VS Code with Copilot, Cursor, Gemini CLI, opencode, Codex CLI or Antigravity | | MCP server | A program that the harness starts to give the agent extra tools | | Project scope | The setting applies to this clone only | | User scope | The setting applies to every repository you open with that harness | @@ -67,6 +67,14 @@ To skip the tool, go to step 5. Claude Code and VS Code read `.mcp.json` directly. For them, there is nothing to write. +Antigravity keeps its servers at user level, for every repository. The tool prints the command to run: + +```bash +python3 dev/mcp_config.py show --harness antigravity +``` + +For a harness not listed here, `--harness other` prints the plain entry to paste into its MCP config. + For another harness, look at the entry first, then write it: ```bash @@ -89,6 +97,7 @@ different entry with the same name, the tool stops and changes nothing. | Gemini CLI | Trust the folder when Gemini CLI asks | | opencode | Nothing. opencode connects the server with no prompt | | Codex CLI | Trust the project. Codex loads `.codex/config.toml` only for a trusted project | +| Antigravity | Run the `agy mcp add` command from step 3. `agy mcp list` shows the server | ### Step 5: Record your answer @@ -119,8 +128,9 @@ connected (for example in Claude Code's `/mcp` list). `AGENTS.md` sends you here when you start a change to this repository. Do not make the offer when the person only wants to audit or implement another repository. -1. Name your harness. Use your own knowledge first. `CLAUDECODE=1` in the environment confirms Claude Code. If - you are not sure, ask the person. +1. Name your harness. Use your own knowledge first, then run `python3 dev/mcp_config.py detect`. It reads the + variable each harness sets. A harness started inside another also has the outer one's variable, so + `ambiguous` or `unknown` means: ask the person. For a harness without a name in the tool, use `other`. 2. Run `python3 dev/mcp_config.py status --harness `. If it prints `asked: ...`, stop here. 3. Run `python3 dev/mcp_config.py check`. 4. Explain the server in two sentences: what it gives you and what it needs. Then ask one question: set it up now, @@ -129,7 +139,7 @@ wants to audit or implement another repository. 6. If the person says yes: - When the server is not installed, show the install routes from step 2. Never run an installer yourself. - Run `show`, then `write` for the person's harness and scope. For Claude Code and VS Code, explain the - approval instead (step 4). + approval instead (step 4). For Antigravity and `other`, give the person the printed command or entry. - Run `check` again, and tell the person to restart the harness. - Record `--answer accepted`. 7. Ask once. Make the offer again only when the person asks for it. diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index b7d5df6a..89c6a174 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -98,6 +98,7 @@ How each harness treats a project MCP file (checked 2026-11-09 in each harness's | Gemini CLI | `.gemini/settings.json` (`mcpServers`) | Starts it after you trust the folder | | opencode | `opencode.json` (`mcp`) | Connects it with no prompt | | Codex CLI | `.codex/config.toml` (`[mcp_servers]`) | Loads it only for a trusted project | +| Antigravity | Its own user-level list (`agy mcp add`) | Starts it in every repository | For a harness that does not read `.mcp.json`, `dev/mcp_config.py` translates the entry into that harness's format: @@ -105,11 +106,13 @@ For a harness that does not read `.mcp.json`, `dev/mcp_config.py` translates the | --- | --- | | `python3 dev/mcp_config.py show --harness [--scope project\|user]` | Prints the target file and the entry. Writes nothing | | `python3 dev/mcp_config.py write --harness [--scope project\|user]` | Adds the entry to the file. Stops if a different entry has the same name | +| `python3 dev/mcp_config.py detect` | Names the harness from the variable it sets (`CLAUDECODE`, `ANTIGRAVITY_AGENT`, `OPENCODE`), or says to ask | | `python3 dev/mcp_config.py check` | Is each server's command on `PATH`? Prints its version, or the install routes | | `python3 dev/mcp_config.py status --harness ` | Has the person answered the setup offer for this harness? | | `python3 dev/mcp_config.py record --harness --answer accepted\|declined` | Records the answer in `.agentic-readiness/setup.json` | -The names are `claude-code`, `vscode`, `cursor`, `gemini`, `opencode` and `codex`. The tool installs nothing. If +The names are `claude-code`, `vscode`, `cursor`, `gemini`, `opencode`, `codex`, `antigravity` and `other`. The tool +installs nothing. If the server is not installed, it fails to start and the agent reads files instead. ## Settings on GitHub From 24a56755133bec151f580234be174b60f042a0aa Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 11:57:53 +0530 Subject: [PATCH 013/111] docs(implement): add the workflow-lint step for CI-05 add-ci-parity now resolves CI-05. It keeps a workflow linter that already runs; otherwise it picks the linter by CI provider (actionlint plus zizmor for GitHub, check-jsonschema's built-in schemas for the others, nothing for Jenkins) and the local route by stack (a Husky-called npm script, pre-commit hooks, a Make or Gradle target, lefthook). The same command runs in CI as a blocking step. Each stack profile's CI idiom names its route. Roadmap-Step: C1.3 Co-Authored-By: Claude Opus 5.5 --- audit/stacks/java.md | 2 +- audit/stacks/python.md | 1 + audit/stacks/typescript-node.md | 3 ++- implement/playbooks/add-ci-parity.md | 31 ++++++++++++++++++++++++++-- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/audit/stacks/java.md b/audit/stacks/java.md index 3fd3ccd6..b14d0960 100644 --- a/audit/stacks/java.md +++ b/audit/stacks/java.md @@ -65,7 +65,7 @@ can carry workflow files that never invoke a build verb (see Pitfalls), so prese build gate. The idiomatic job is `actions/checkout`, `actions/setup-java` with `distribution` and `java-version` pinned, Gradle's own dependency cache (`actions/setup-java`'s `cache: gradle` input, or `gradle/actions/setup-gradle`), then `./gradlew check` (never a bare `gradle check`, which bypasses the -pinned wrapper version). +pinned wrapper version). Workflow lint (`CI-05`): actionlint from a Gradle task, `.githooks/` or lefthook. ## Baseline / ratchet diff --git a/audit/stacks/python.md b/audit/stacks/python.md index 77249bcf..2f1866cd 100644 --- a/audit/stacks/python.md +++ b/audit/stacks/python.md @@ -48,6 +48,7 @@ actually acts on, not just which file exists. The matching lockfile (`poetry.loc GitHub Actions is the common provider: `actions/checkout`, `actions/setup-python` with `python-version` pinned to `requires-python`/`.python-version`, the matching manager's own cache (`setup-python`'s `cache: pip`, or `astral-sh/setup-uv`/`cache: poetry` for the others), then the project's `check` target. +Workflow lint (`CI-05`): actionlint or zizmor as pre-commit hooks (or `uvx zizmor` in a target), run in CI too. ## Baseline / ratchet diff --git a/audit/stacks/typescript-node.md b/audit/stacks/typescript-node.md index 201b2eaa..54256dc4 100644 --- a/audit/stacks/typescript-node.md +++ b/audit/stacks/typescript-node.md @@ -60,7 +60,8 @@ GitHub Actions is the provider seen in all three repositories: `actions/checkout `actions/setup-node` with `node-version` pinned to the same major as `engines.node` and `cache: npm` (or the matching `pnpm`/`yarn` cache key), then `npm ci`, then `npm run check` (or the repository's own `check` script) as the one gate job calls. A matrix across `ubuntu-latest`/`macos-latest`/ -`windows-latest` was seen in one repository (§Verified) and is a strong signal, not a requirement. +`windows-latest` was seen in one repository (§Verified) and is a strong signal, not a requirement. Workflow +lint (`CI-05`): a `lint:workflows` script running actionlint, called from the Husky pre-commit hook and in CI. ## Baseline / ratchet diff --git a/implement/playbooks/add-ci-parity.md b/implement/playbooks/add-ci-parity.md index 53e0264b..70302d13 100644 --- a/implement/playbooks/add-ci-parity.md +++ b/implement/playbooks/add-ci-parity.md @@ -17,6 +17,7 @@ recorded, and a committed record shows which CI job the platform requires before | CI-02 | [06-ci.md](../../audit/checks/06-ci.md#ci-02--localci-parity) | CI runs the local verbs, directly or through a wrapper script that calls them | | CI-03 | [06-ci.md](../../audit/checks/06-ci.md#ci-03--ci-only-steps-flagged) | Every CI step is a local command; a step that exists only in CI is flagged in its name and in the CI doc | | CI-04 | [06-ci.md](../../audit/checks/06-ci.md#ci-04--ci-enforces-the-same-gate-as-the-local-check) | No gating step swallows its exit code; non-gating steps are visibly report-only; the required-check wiring is recorded with evidence | +| CI-05 | [06-ci.md](../../audit/checks/06-ci.md#ci-05--workflow-files-are-linted) | A workflow linter that knows the CI provider runs from the local surface and in CI | ## Preconditions @@ -27,7 +28,7 @@ recorded, and a committed record shows which CI job the platform requires before call does not exist, stop and report; do not invent a CI-only command. - Preferences read, when present: `ci.edits_authorised` (a `no` or a scoped value limits which files may change; honour the exact scope), `ci.image_scan_debt`, `scanner.choice`, `format.mass_reformat`, `layout.command_runner`, - `lint.mode`, `security.verb`. + `lint.mode`, `security.verb`, `ci.workflow_linter`, `hooks.manager`. - **Drift** ([Stage 3](../SKILL.md#stage-3-apply), step 2): a step already fixed or a wrapper already calling the verb makes a finding stale; re-read the workflow on the current integration branch. @@ -53,6 +54,25 @@ recorded, and a committed record shows which CI job the platform requires before mechanism (classic rule or ruleset), required checks, the up-to-date setting, bypass actors, and whether the plan allows protection at all. **A session never guesses these.** If they are not supplied, stop for that part. 5. **Names to preserve:** the merge-gate job name and the docs-only skip (path filter or change-detection job). +6. **Workflow linter (CI-05).** Detect first and keep a linter that already runs. Look in Makefile targets, + `package.json` scripts, hooks (`.husky/`, `.pre-commit-config.yaml`, `lefthook.yml`) and CI steps. If there is + none, choose the linter by CI provider and the route by stack (the stack profile's Tooling defaults). + Currency checked 2026-11-09 (GitHub releases API, PyPI); re-check when applied. + + | Provider | Recommend | Version, date | Notes | + |---|---|---|---| + | GitHub Actions | actionlint, plus zizmor for security | actionlint v1.7.12, 2026-03-30; zizmor 1.32.0, 2026-11-02 | actionlint checks syntax, expressions and action inputs; zizmor finds script injection and over-broad permissions. One is enough for CI-05; offer both | + | GitLab, Azure, Bitbucket, CircleCI, Buildkite | check-jsonschema with the provider's built-in schema | 0.39.0, 2026-10-26 | Offline. `glab ci lint` and `circleci config validate` also count but need the platform CLI | + | Jenkins | none offline | — | CI-05 reads `UNKNOWN`; say so, add nothing | + + | Stack | Local route | + |---|---| + | Node with Husky | a `lint:workflows` script, called from `.husky/pre-commit` when workflow files are staged | + | Python with pre-commit | the `actionlint` hook (`rhysd/actionlint`), `zizmorcore/zizmor-pre-commit`, or check-jsonschema's `check-github-workflows` / `check-gitlab-ci` hooks | + | Make, Gradle, lefthook or mixed | a `workflow-lint` target or lefthook command that runs the linter on the workflow folder | + + Each developer installs the linter once. Pin its version where the repository pins its other dev tools, never + in the workflow alone. Record the choice as the preference `ci.workflow_linter`. ## Steps @@ -82,7 +102,11 @@ recorded, and a committed record shows which CI job the platform requires before required check name, up-to-date setting, bypass actors, the date the owner confirmed, and the exact read-only command that re-checks it. If the platform cannot enforce a required check (plan limitation), the page says so and lists the mitigations; the check is then recorded `partial`, never `done`. -7. **Lint the workflow** with the repository's workflow linter before pushing. Edit workflows only in a PR. +7. **Workflow lint (CI-05).** Add the chosen linter to the local surface (the route from Inputs 6). Run the same + command in CI as a blocking step, beside the gate. Prefer calling the local command over a third-party + `uses:` action; an action needs the owner's approval. Fix what the linter reports in the workflows you edit. + Record older findings as debt, as for scans. Lint every workflow you changed before pushing. Edit workflows + only in a PR. 8. **Commits:** wrapper, workflow, record. ### GitHub Actions and Gradle @@ -135,6 +159,7 @@ Lessons from the multi-app Node application (IR07.3): | CI-02 | Read each CI step and wrapper; run the wrapper locally | Every verb CI runs is the local verb with the same arguments; local run is green | | CI-03 | `grep -n "continue-on-error\|\|\| true" .github/workflows/*.yml scripts/ci/*.sh` plus the step list | Every CI-only step is named so and listed in the CI doc | | CI-04 | The same grep; the job summary of a CI run; the branch-protection record | No swallowed gating command; every non-blocking step says report-only and has recorded debt; the record names the required check, or says protection cannot be enforced | +| CI-05 | Run the local workflow-lint command; read the CI step | The linter runs locally and in CI and is clean on the edited workflows; the engine reads CI-05 `PASS` | The engine reads `run:` text only. After a correct wrapper-based fix a re-audit can still read CI-02 or CI-03 wrongly (known engine defects), and CI-04 needs the platform confirmation: say so in the pull request body; the @@ -171,4 +196,6 @@ python3 implement/tools/progress.py pref set ci.shape --phase --reason "" --by owner python3 implement/tools/progress.py pref set ci.edits_authorised \ --target --phase --reason "" --by owner +python3 implement/tools/progress.py pref set ci.workflow_linter \ + --target --phase --reason "" --by owner ``` From 98a13e1b84271c38f8b41c79d67eefa5e0f1f45d Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 12:09:50 +0530 Subject: [PATCH 014/111] docs: link the agent tool setup from the contributor entry points CONTRIBUTING.md lists codebase-memory-mcp as an optional prerequisite, the contribute-a-change how-to adds it as an optional one-time setup step, and the README points contributors to the how-to. Roadmap-Step: B3 Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 1 + README.md | 3 ++- docs/how-to/contribute-a-change.md | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fe650af2..fe1277f6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,6 +32,7 @@ the Makefile. The rules an agent follows are in [`AGENTS.md`](AGENTS.md); they a | `make` | Every command below | Linux and macOS. On Windows use WSL | | Node 22 | `make docs-lint` | The pinned `markdownlint-cli2` runs through `npx`; not needed for `make check` | | Chrome, through Puppeteer | `make docs-mermaid` | Only when you add or change a Mermaid block; the pinned mermaid-cli fetches it on first use | +| `codebase-memory-mcp` (optional) | Faster code search for your coding agent | Listed in `.mcp.json`. Your agent offers to set it up once; see [Set up the agent tools](docs/how-to/set-up-agent-tools.md). No gate needs it | | The sibling scaffolder | One engine test, the positive fixture | Set `AUDIT_BOOTSTRAP=`. With `CI=true` and no scaffolder the test is skipped; locally it fails without it | `make setup` checks Python, `uv` and `git`, installs nothing, and names the first one that is missing. diff --git a/README.md b/README.md index 0e8986f2..7be6c535 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,8 @@ it pushes a branch only if you opt in. ## Contributing Start at [`CONTRIBUTING.md`](CONTRIBUTING.md): the repository map, the gates (`make check`, `make security`) and the -commit rules. Agents read [`AGENTS.md`](AGENTS.md). +commit rules. Agents read [`AGENTS.md`](AGENTS.md). To give your coding agent the code index here, see +[Set up the agent tools](docs/how-to/set-up-agent-tools.md). `dev` is the default branch and holds the work in progress; `main` holds the latest release. Start your branch from `dev` and open your pull request into `dev`. Nobody commits to `dev` or `main` directly, and only `dev` is promoted diff --git a/docs/how-to/contribute-a-change.md b/docs/how-to/contribute-a-change.md index 9100ba3e..22cf569f 100644 --- a/docs/how-to/contribute-a-change.md +++ b/docs/how-to/contribute-a-change.md @@ -79,6 +79,9 @@ Do these steps once, on each new machine. Get `lefthook` from [lefthook.dev](https://lefthook.dev). The hooks do not replace the gates in CI. +5. Optional: give your coding agent the code index. Your agent offers this once, before its first edit. To do it + yourself, follow [Set up the agent tools](set-up-agent-tools.md). No gate needs it. + --- ## Step-by-step instructions From 07d42bbf282b38b7e757c44219ca172e8eb4183f Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 12:21:40 +0530 Subject: [PATCH 015/111] feat(dev): document language servers beside the code index docs/framework/tool-ecosystem.md now compares a language server with the code index and lists each harness's route (checked 2026-11-09). The how-to adds an optional language-server step: the pyright-lsp plugin for Claude Code, opencode's built-in "lsp", and Serena for Gemini CLI, Codex and Antigravity. pyrightconfig.json gives pyright and basedpyright the mypy.ini import paths: basedpyright drops from 294 errors and 6,249 warnings to 71 errors (61 in the engine, which mypy does not gate yet). dev/tests/test_pyright_config.py keeps the paths in step. mypy strict stays the gate. EXEC-07 now fails (two root type configs). The owner accepted it as an exception, recorded in docs/reference/self-audit.md; the baseline takes the finding (score 0.8770). Serena stays out of .mcp.json: the harnesses that need it do not read that file, and its launch command differs per harness (ADR 0006). Roadmap-Step: B4 Co-Authored-By: Claude Opus 5.5 --- .audit-baseline.json | 63 ++++++++++++------- .secrets.baseline | 14 ++--- .../examples/self-audit/2026-11-09.json | 63 ++++++++++++------- dev/tests/test_pyright_config.py | 54 ++++++++++++++++ .../adrs/0006-first-run-agent-tool-offer.md | 5 +- docs/framework/tool-ecosystem.md | 24 +++++++ docs/how-to/set-up-agent-tools.md | 21 +++++++ docs/reference/dev-tooling.md | 4 ++ docs/reference/self-audit.md | 12 ++-- pyrightconfig.json | 16 +++++ 10 files changed, 221 insertions(+), 55 deletions(-) create mode 100644 dev/tests/test_pyright_config.py create mode 100644 pyrightconfig.json diff --git a/.audit-baseline.json b/.audit-baseline.json index b2c6f613..b07bad98 100644 --- a/.audit-baseline.json +++ b/.audit-baseline.json @@ -223,7 +223,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "20/28 harnesses reached; AGENTS.md:46 states the supported harnesses (Anthropic Claude Code, Cursor, Google Gemini CLI, OpenAI Codex CLI, opencode)", + "summary": "20/28 harnesses reached; AGENTS.md:51 states the supported harnesses (Anthropic Claude Code, Cursor, Google Gemini CLI, OpenAI Codex CLI, opencode)", "tier": "B", "title": "No competing instruction files, and the reach matrix is stated", "verdict": "PASS" @@ -262,7 +262,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "43 markdown file(s) under docs/", + "summary": "45 markdown file(s) under docs/", "tier": "A", "title": "docs/ exists", "verdict": "PASS" @@ -301,7 +301,7 @@ "scored": true, "severity": "COSMETIC", "status": "implemented", - "summary": "5 ADR(s) indexed in docs/architecture/adrs/README.md", + "summary": "6 ADR(s) indexed in docs/architecture/adrs/README.md", "tier": "A", "title": "ADR directory and index", "verdict": "PASS" @@ -790,10 +790,10 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "lint: ruff.toml; types: mypy.ini", + "summary": "competing configs: types (mypy.ini, pyrightconfig.json)", "tier": "B", "title": "No competing configs for one concern", - "verdict": "PASS" + "verdict": "FAIL" }, { "data": {}, @@ -803,7 +803,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "test verb on Makefile (cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests); 85 test file(s)", + "summary": "test verb on Makefile (cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests); 87 test file(s)", "tier": "A", "title": "Test verb exists, suite non-empty", "verdict": "PASS" @@ -836,7 +836,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "skip/xfail census across 85 test file(s): python 1; total 1", + "summary": "skip/xfail census across 87 test file(s): python 1; total 1", "tier": "B", "title": "Skip / xfail census", "verdict": "PASS" @@ -849,7 +849,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "fast path documented: AGENTS.md:36 `python3 -m unittest tests.test_cmd_pack -k verbs` (scope: README.md, AGENTS.md, CONTRIBUTING.md)", + "summary": "fast path documented: AGENTS.md:41 `python3 -m unittest tests.test_cmd_pack -k verbs` (scope: README.md, AGENTS.md, CONTRIBUTING.md)", "tier": "B", "title": "Fast path documented", "verdict": "PASS" @@ -921,10 +921,10 @@ }, { "data": { - "denominator": 1152, + "denominator": 1178, "kind": "ratio", "method": "python: module-level def/class not starting with '_'", - "numerator": 456, + "numerator": 470, "unit": "public symbols" }, "id": "NAV-05", @@ -933,7 +933,7 @@ "scored": false, "severity": "COSMETIC", "status": "implemented", - "summary": "456/1152 public symbols documented (40%); method: python: module-level def/class not starting with '_'", + "summary": "470/1178 public symbols documented (40%); method: python: module-level def/class not starting with '_'", "tier": "B", "title": "Public-symbol documentation coverage", "verdict": "PARTIAL" @@ -1368,6 +1368,27 @@ "statement": "An agent cannot state how many violations exist today because a counting surface exists but no numeric violation budget or count is committed.", "verdict": "PARTIAL" }, + { + "check": "EXEC-07", + "evidence": [ + { + "line": null, + "note": null, + "path": "mypy.ini" + }, + { + "line": null, + "note": null, + "path": "pyrightconfig.json" + } + ], + "id": "EXEC-07", + "phase": 1, + "remediation": "Keep one authoritative config per concern and document any deliberate delegation, or delete the redundant config.", + "severity": "DEGRADER", + "statement": "An agent cannot know which config wins for each concern because 1 concern(s) have more than one config: types (mypy.ini, pyrightconfig.json).", + "verdict": "FAIL" + }, { "check": "SEC-03", "evidence": [ @@ -1437,8 +1458,8 @@ } ], "inventory": { - "files_inspected": 336, - "files_set_aside": 460, + "files_inspected": 342, + "files_set_aside": 461, "files_skipped": 3, "set_aside": [ { @@ -1450,7 +1471,7 @@ "path": "audit/engine/tests/fixtures" }, { - "files": 32, + "files": 33, "path": "audit/schema/examples" }, { @@ -1517,15 +1538,15 @@ } ], "provenance": { - "framework_revision": "320fad8969a38d09331ece11634cc36e124db725", + "framework_revision": "98a13e1b84271c38f8b41c79d67eefa5e0f1f45d", "git_branch": "feat/v0.3.3-track-b-mcp", "git_dirty": true, - "git_sha": "320fad8969a38d09331ece11634cc36e124db725", + "git_sha": "98a13e1b84271c38f8b41c79d67eefa5e0f1f45d", "phase": 1, "probes_enabled": false, "ruleset_hash": "970e6792f98eeeb5", "ruleset_revision": "2026-11-06.1", - "scanned_at": "2026-11-09T05:04:17+00:00", + "scanned_at": "2026-11-09T06:47:54+00:00", "target": ".", "tool": "python-agentic-audit", "user_config": { @@ -1557,9 +1578,9 @@ "blockers": 1, "checks": { "attest": 0, - "fail": 0, + "fail": 1, "partial": 6, - "pass": 59, + "pass": 58, "unknown": 4 }, "checks_applicable": 69, @@ -1567,8 +1588,8 @@ "checks_scoreable": 55, "cosmetic": 0, "coverage_note": "55 of 55 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", - "degraders": 3, - "phase1_score": 0.8934 + "degraders": 4, + "phase1_score": 0.877 }, "unattested": [ "index-in-use", diff --git a/.secrets.baseline b/.secrets.baseline index 13fd8c9c..718199c7 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -131,16 +131,16 @@ { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", - "hashed_secret": "8614d586c51ed65bb882b415a3c861a2c92b157f", + "hashed_secret": "70e5aa2e9690505b775c6a71c8d9969d78fc0b4a", "is_verified": false, - "line_number": 1520 + "line_number": 1541 }, { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", "hashed_secret": "228ea6c3984c8d1226e0ffc33455f16dce652d01", "is_verified": false, - "line_number": 1526 + "line_number": 1547 } ], "audit/catalogue.json": [ @@ -451,16 +451,16 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/self-audit/2026-11-09.json", - "hashed_secret": "8614d586c51ed65bb882b415a3c861a2c92b157f", + "hashed_secret": "70e5aa2e9690505b775c6a71c8d9969d78fc0b4a", "is_verified": false, - "line_number": 1520 + "line_number": 1541 }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/self-audit/2026-11-09.json", "hashed_secret": "228ea6c3984c8d1226e0ffc33455f16dce652d01", "is_verified": false, - "line_number": 1526 + "line_number": 1547 } ], "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json": [ @@ -1465,5 +1465,5 @@ } ] }, - "generated_at": "2026-11-09T05:09:30Z" + "generated_at": "2026-11-09T06:48:08Z" } diff --git a/audit/schema/examples/self-audit/2026-11-09.json b/audit/schema/examples/self-audit/2026-11-09.json index b2c6f613..b07bad98 100644 --- a/audit/schema/examples/self-audit/2026-11-09.json +++ b/audit/schema/examples/self-audit/2026-11-09.json @@ -223,7 +223,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "20/28 harnesses reached; AGENTS.md:46 states the supported harnesses (Anthropic Claude Code, Cursor, Google Gemini CLI, OpenAI Codex CLI, opencode)", + "summary": "20/28 harnesses reached; AGENTS.md:51 states the supported harnesses (Anthropic Claude Code, Cursor, Google Gemini CLI, OpenAI Codex CLI, opencode)", "tier": "B", "title": "No competing instruction files, and the reach matrix is stated", "verdict": "PASS" @@ -262,7 +262,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "43 markdown file(s) under docs/", + "summary": "45 markdown file(s) under docs/", "tier": "A", "title": "docs/ exists", "verdict": "PASS" @@ -301,7 +301,7 @@ "scored": true, "severity": "COSMETIC", "status": "implemented", - "summary": "5 ADR(s) indexed in docs/architecture/adrs/README.md", + "summary": "6 ADR(s) indexed in docs/architecture/adrs/README.md", "tier": "A", "title": "ADR directory and index", "verdict": "PASS" @@ -790,10 +790,10 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "lint: ruff.toml; types: mypy.ini", + "summary": "competing configs: types (mypy.ini, pyrightconfig.json)", "tier": "B", "title": "No competing configs for one concern", - "verdict": "PASS" + "verdict": "FAIL" }, { "data": {}, @@ -803,7 +803,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "test verb on Makefile (cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests); 85 test file(s)", + "summary": "test verb on Makefile (cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests); 87 test file(s)", "tier": "A", "title": "Test verb exists, suite non-empty", "verdict": "PASS" @@ -836,7 +836,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "skip/xfail census across 85 test file(s): python 1; total 1", + "summary": "skip/xfail census across 87 test file(s): python 1; total 1", "tier": "B", "title": "Skip / xfail census", "verdict": "PASS" @@ -849,7 +849,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "fast path documented: AGENTS.md:36 `python3 -m unittest tests.test_cmd_pack -k verbs` (scope: README.md, AGENTS.md, CONTRIBUTING.md)", + "summary": "fast path documented: AGENTS.md:41 `python3 -m unittest tests.test_cmd_pack -k verbs` (scope: README.md, AGENTS.md, CONTRIBUTING.md)", "tier": "B", "title": "Fast path documented", "verdict": "PASS" @@ -921,10 +921,10 @@ }, { "data": { - "denominator": 1152, + "denominator": 1178, "kind": "ratio", "method": "python: module-level def/class not starting with '_'", - "numerator": 456, + "numerator": 470, "unit": "public symbols" }, "id": "NAV-05", @@ -933,7 +933,7 @@ "scored": false, "severity": "COSMETIC", "status": "implemented", - "summary": "456/1152 public symbols documented (40%); method: python: module-level def/class not starting with '_'", + "summary": "470/1178 public symbols documented (40%); method: python: module-level def/class not starting with '_'", "tier": "B", "title": "Public-symbol documentation coverage", "verdict": "PARTIAL" @@ -1368,6 +1368,27 @@ "statement": "An agent cannot state how many violations exist today because a counting surface exists but no numeric violation budget or count is committed.", "verdict": "PARTIAL" }, + { + "check": "EXEC-07", + "evidence": [ + { + "line": null, + "note": null, + "path": "mypy.ini" + }, + { + "line": null, + "note": null, + "path": "pyrightconfig.json" + } + ], + "id": "EXEC-07", + "phase": 1, + "remediation": "Keep one authoritative config per concern and document any deliberate delegation, or delete the redundant config.", + "severity": "DEGRADER", + "statement": "An agent cannot know which config wins for each concern because 1 concern(s) have more than one config: types (mypy.ini, pyrightconfig.json).", + "verdict": "FAIL" + }, { "check": "SEC-03", "evidence": [ @@ -1437,8 +1458,8 @@ } ], "inventory": { - "files_inspected": 336, - "files_set_aside": 460, + "files_inspected": 342, + "files_set_aside": 461, "files_skipped": 3, "set_aside": [ { @@ -1450,7 +1471,7 @@ "path": "audit/engine/tests/fixtures" }, { - "files": 32, + "files": 33, "path": "audit/schema/examples" }, { @@ -1517,15 +1538,15 @@ } ], "provenance": { - "framework_revision": "320fad8969a38d09331ece11634cc36e124db725", + "framework_revision": "98a13e1b84271c38f8b41c79d67eefa5e0f1f45d", "git_branch": "feat/v0.3.3-track-b-mcp", "git_dirty": true, - "git_sha": "320fad8969a38d09331ece11634cc36e124db725", + "git_sha": "98a13e1b84271c38f8b41c79d67eefa5e0f1f45d", "phase": 1, "probes_enabled": false, "ruleset_hash": "970e6792f98eeeb5", "ruleset_revision": "2026-11-06.1", - "scanned_at": "2026-11-09T05:04:17+00:00", + "scanned_at": "2026-11-09T06:47:54+00:00", "target": ".", "tool": "python-agentic-audit", "user_config": { @@ -1557,9 +1578,9 @@ "blockers": 1, "checks": { "attest": 0, - "fail": 0, + "fail": 1, "partial": 6, - "pass": 59, + "pass": 58, "unknown": 4 }, "checks_applicable": 69, @@ -1567,8 +1588,8 @@ "checks_scoreable": 55, "cosmetic": 0, "coverage_note": "55 of 55 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", - "degraders": 3, - "phase1_score": 0.8934 + "degraders": 4, + "phase1_score": 0.877 }, "unattested": [ "index-in-use", diff --git a/dev/tests/test_pyright_config.py b/dev/tests/test_pyright_config.py new file mode 100644 index 00000000..a902a8b9 --- /dev/null +++ b/dev/tests/test_pyright_config.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna + +"""``pyrightconfig.json`` lets a language server resolve the same imports as the ``mypy`` gate. + +A language server (pyright, basedpyright, or Serena on top of one) gives an agent definitions, references and +type errors after an edit. It is an aid only: ``make typecheck`` (mypy strict) stays the gate. The tools import +each other by module name across folders, so without the same search paths as ``mypy.ini`` the server reports +hundreds of unresolved imports and the agent chases them. +""" + +from __future__ import annotations + +import configparser +import json +import unittest + +import _dev_paths # noqa: F401 (puts dev/ on sys.path) +from _dev_paths import REPO + +PYRIGHT = REPO / "pyrightconfig.json" +MYPY = REPO / "mypy.ini" + + +def pyright() -> dict[str, object]: + data: dict[str, object] = json.loads(PYRIGHT.read_text(encoding="utf-8")) + return data + + +class PyrightFollowsMypy(unittest.TestCase): + def test_extra_paths_cover_the_mypy_path(self) -> None: + mypy = configparser.ConfigParser() + mypy.read(MYPY, encoding="utf-8") + mypy_paths = set(mypy["mypy"]["mypy_path"].split(":")) + extra = pyright()["extraPaths"] + assert isinstance(extra, list) + self.assertLessEqual(mypy_paths, set(extra)) + self.assertIn("audit/engine", extra, "the engine package `audit` resolves from audit/engine") + + def test_python_version_matches(self) -> None: + mypy = configparser.ConfigParser() + mypy.read(MYPY, encoding="utf-8") + self.assertEqual(pyright()["pythonVersion"], mypy["mypy"]["python_version"]) + + def test_every_extra_path_exists(self) -> None: + extra = pyright()["extraPaths"] + assert isinstance(extra, list) + for path in extra: + with self.subTest(path=path): + self.assertTrue((REPO / path).is_dir()) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/architecture/adrs/0006-first-run-agent-tool-offer.md b/docs/architecture/adrs/0006-first-run-agent-tool-offer.md index 6612355d..c0abdeb8 100644 --- a/docs/architecture/adrs/0006-first-run-agent-tool-offer.md +++ b/docs/architecture/adrs/0006-first-run-agent-tool-offer.md @@ -52,7 +52,9 @@ their own code; they do not need the server here. The owner decided on one commi 7. **Nothing is installed by the tool or the agent.** `check` runs ` --version` for a command on `PATH`. For a missing command it prints the upstream install routes. They carry `--skip-config`, so the installer writes no config into other harnesses. The person runs the installer. -8. **No gate depends on MCP.** Anything that must be right runs through `make`. +8. **No gate depends on MCP or a language server.** Anything that must be right runs through `make`. A + language server is an optional aid: `pyrightconfig.json` mirrors the `mypy.ini` import paths, and the offer + does not cover it. --- @@ -65,6 +67,7 @@ their own code; they do not need the server here. The owner decided on one commi | A dedicated skill for the offer | Not every harness reads skills. Every harness reads `AGENTS.md` | | Detect the harness from environment variables only | Only Claude Code's variable is confirmed. A wrong guess writes the wrong file | | Record the answer in a tracked file | One person's answer would apply to every clone | +| Serena in `.mcp.json` | Only Claude Code and VS Code read the file, and both have their own language-server route. The harnesses that need Serena do not read it, and its launch command differs per harness (`--context`) | | Pre-approve the server in `.claude/settings.json` | It would start a program on every contributor's machine without asking | | No offer: document the setup only | The setup is easy to miss, and agents then read whole files where a graph query would do | diff --git a/docs/framework/tool-ecosystem.md b/docs/framework/tool-ecosystem.md index 9f649e41..2a583c4c 100644 --- a/docs/framework/tool-ecosystem.md +++ b/docs/framework/tool-ecosystem.md @@ -114,6 +114,30 @@ graph TB --- +## Language Servers Beside the Code Index + +A language server (LSP) and the code index answer different questions. Use both where the harness allows it. + +| | Code index (`codebase-memory-mcp`) | Language server | +| --- | --- | --- | +| Gives the agent | The whole-repository graph: callers, dependencies, tests, across languages | Compiler-accurate definitions, references, types and renames for one language | +| After an edit | A re-index refreshes the graph | Type errors show at once | +| Needs from the repository | An index registered for the checkout (`IDX-01`) | The type-checker config, the import paths, and locked dependencies | + +A language server is an aid, never a gate. The repository's own type checker, run through its `typecheck` verb, +stays the source of truth. No requirement asks for a language server to be installed. + +How each harness reaches a language server (checked 2026-11-09 in each harness's own documentation): + +| Harness | Route | +| --- | --- | +| Claude Code | A code-intelligence plugin, for example `pyright-lsp`. The language-server binary must be on `PATH` | +| opencode | Built in, off by default: `"lsp": true` in `opencode.json`. The server, for example `pyright`, must be installed | +| Cursor, VS Code | The editor runs language servers for its extensions. Their docs do not say what the agent receives | +| Gemini CLI, Codex CLI, Antigravity | No built-in route documented. [Serena](https://github.com/oraios/serena), an MCP server on top of language servers, fills the gap | + +--- + ## Reference Implementation — `agentic-tdd` `agentic-tdd` is the live worked example of Phases 1–4, including the Phase 1 command diff --git a/docs/how-to/set-up-agent-tools.md b/docs/how-to/set-up-agent-tools.md index 771b5d1f..543060d7 100644 --- a/docs/how-to/set-up-agent-tools.md +++ b/docs/how-to/set-up-agent-tools.md @@ -123,6 +123,27 @@ connected (for example in Claude Code's `/mcp` list). - In Claude Code, run `claude mcp reset-project-choices` to be asked to approve again. - Delete `.agentic-readiness/setup.json` to get the offer again. +## Optional: a language server + +A language server gives the agent compiler-accurate definitions, references and type errors right after an edit. +It complements the code index; the comparison is in [Tool Ecosystem](../framework/tool-ecosystem.md#language-servers-beside-the-code-index). +No gate needs it: `make typecheck` (mypy strict) stays the gate. + +This repository commits `pyrightconfig.json`, so pyright and basedpyright resolve the same imports as mypy. +`dev/tests/test_pyright_config.py` keeps it in step with `mypy.ini`. Pyright is stricter than mypy in places, and +the engine is not yet under the mypy gate. So treat a pyright error that `make typecheck` does not report as a +hint, not a failure. + +| Harness | How to add it | +| --- | --- | +| Claude Code | `npm install -g pyright`, then `/plugin install pyright-lsp@claude-plugins-official` | +| opencode | `npm install -g pyright`, then set `"lsp": true` in your own `opencode.json` | +| Cursor, VS Code | Install the Python extension. Their docs do not say what the agent receives from it | +| Gemini CLI, Codex CLI, Antigravity | Install Serena: `uv tool install -p 3.13 serena-agent`, then `serena init`. Add `serena start-mcp-server --context --project-from-cwd` as an MCP server in your harness. The client names are in [Serena's client guide](https://oraios.github.io/serena/02-usage/030_clients.html) | + +Serena is not in `.mcp.json`. The harnesses that need it do not read that file, and its launch command differs +for each harness. + ## For agents: the first-run offer `AGENTS.md` sends you here when you start a change to this repository. Do not make the offer when the person only diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 89c6a174..af4ba0cf 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -111,6 +111,10 @@ For a harness that does not read `.mcp.json`, `dev/mcp_config.py` translates the | `python3 dev/mcp_config.py status --harness ` | Has the person answered the setup offer for this harness? | | `python3 dev/mcp_config.py record --harness --answer accepted\|declined` | Records the answer in `.agentic-readiness/setup.json` | +A language server is an optional aid beside the index. `pyrightconfig.json` gives pyright and basedpyright the same +import paths as `mypy.ini`, and `dev/tests/test_pyright_config.py` keeps the two in step. `make typecheck` stays the +gate. Routes per harness: [Set up the agent tools](../how-to/set-up-agent-tools.md#optional-a-language-server). + The names are `claude-code`, `vscode`, `cursor`, `gemini`, `opencode`, `codex`, `antigravity` and `other`. The tool installs nothing. If the server is not installed, it fails to start and the agent reads files instead. diff --git a/docs/reference/self-audit.md b/docs/reference/self-audit.md index e8eec815..af108c1e 100644 --- a/docs/reference/self-audit.md +++ b/docs/reference/self-audit.md @@ -16,12 +16,12 @@ score from falling. | Field | Value | | --- | --- | -| Phase-1 score | **0.8934** | +| Phase-1 score | **0.8770** | | Date | 2026-11-09 | | Engine | `python-agentic-audit` 0.3.2 | | Ruleset | `2026-11-06.1` (hash `970e6792f98eeeb5`) | | Applicable checks | 69 (55 scoreable) | -| Findings | 4: 1 BLOCKER, 3 DEGRADER | +| Findings | 5: 1 BLOCKER, 4 DEGRADER | | Scope | the whole repository except `artefacts/` and `dist/`; no user-level configuration read | `HYG-05`, `HYG-07`, `HYG-08` and `HYG-09` closed with `CONTRIBUTING.md`, the root changelog, `RELEASE_PROCESS.md` and @@ -33,9 +33,9 @@ The score was 0.6762 on 2026-11-05, before the repository adopted the Phase 1 re | Verdict | Checks | | --- | --: | -| PASS | 59 | +| PASS | 58 | | PARTIAL | 6 | -| FAIL | 0 | +| FAIL | 1 | | UNKNOWN | 4 | | ATTEST | 0 | | **Total** | **69** | @@ -53,12 +53,14 @@ Every check that is not `PASS`, with the reason it is open. | `SEC-01` | PARTIAL | BLOCKER | `/.env` is root-anchored on purpose: the engine's fixtures hold `.env` files as data and must stay committable (comment in `.gitignore`). The check reads a root-only rule as incomplete | | `SEC-03` | PARTIAL | DEGRADER | Same cause as `SEC-01` for the dotenv shapes; the key, certificate and keystore shapes are covered | | `BASE-02` | PARTIAL | DEGRADER | The T5 playbook was applied (D32): `.quality-baseline.json` holds the counts (ruff 0, mypy 0, Markdown lint outside the ratchet 735) and `make format-check`, `make typecheck` and `make docs-lint` fail when one goes up. The check still reads PARTIAL by design: it credits only a numeric budget flag (`--max-warnings N`), and none of these tools has one. Engine issue [#45](https://github.com/Nistapp/codebase-agentic-readiness-framework/issues/45) tracks it; the coverage floor (91) is unchanged and `BASE-04` passes | +| `EXEC-07` | FAIL | DEGRADER | **Owner exception** (2026-11-09): `mypy.ini` and `pyrightconfig.json` both configure types. `make typecheck` (mypy strict) is the gate; `pyrightconfig.json` only gives a language server the same import paths, so an agent gets definitions and type errors without hundreds of unresolved imports. `dev/tests/test_pyright_config.py` keeps the paths in step. The check cannot read that delegation ([Set up the agent tools](../how-to/set-up-agent-tools.md#optional-a-language-server)) | | `TOOL-04` | PARTIAL | DEGRADER | **Owner exception** (D33): the repository has no third-party dependencies, so there is nothing for a dependency audit to scan; `make security` is a secrets scan | | `NAV-05` | PARTIAL | COSMETIC | Informational, no work (D34): 40% of public symbols documented (456 of 1152) | | `IDX-02`, `IDX-03`, `TST-02`, `CON-04` | UNKNOWN | mixed | Need a probe, the user-level index store, or `--emit-baseline`; not run here | | `EXEC-02` | PARTIAL | BLOCKER | The two lockfiles of the pinned dev tools (`dev/tools/uv.lock`, `dev/tools/package-lock.json`) cannot be checked against their manifests without running the package managers. `dev/tests/test_tool_pins.py` checks that they agree; the engine does not read that test | -Recorded decisions that keep a finding open or exempt: `TOOL-04` is an owner exception (D33). The `IDX-01` exception +Recorded decisions that keep a finding open or exempt: `TOOL-04` (D33) and `EXEC-07` (2026-11-09) are owner +exceptions. The `IDX-01` exception (D21) ended on 2026-11-09: `.mcp.json` now registers `codebase-memory-mcp`, and each person still approves it. `AGT-08` stays as it is for v0.2, because a stated harness list closes the gap even when a named harness does not read `AGENTS.md` natively, and is revisited in v0.3 (D31); `NAV-05` is informational and unscored (D34). The engine has no diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 00000000..e9a80110 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,16 @@ +{ + "pythonVersion": "3.11", + "typeCheckingMode": "standard", + "include": ["audit", "implement", "dev"], + "exclude": ["artefacts", "dist", "**/fixtures", "**/testdata", "**/__pycache__", "dev/dist", "audit/engine/dist"], + "extraPaths": [ + "audit/engine", + "audit/tools", + "audit/tools/tests", + "implement/tools", + "implement/tools/tests", + "dev", + "dev/eval", + "dev/tests" + ] +} From 1a32a514fdf70b19c3586df91a2c88610e92d388 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 10:06:51 +0530 Subject: [PATCH 016/111] ci: add an advisory CodeQL workflow for Python and Actions Scan the Python code and the workflows with CodeQL on pull requests into dev and main, pushes to dev, and weekly. The engine's test fixtures and JavaScript/TypeScript (fixtures only) are excluded. No ruleset requires the workflow; branch-protection.md says how the maintainer handles alerts. Roadmap-Step: A1 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/codeql.yml | 49 +++++++++++++++++++++++++++++ docs/reference/branch-protection.md | 19 +++++++++++ docs/reference/dev-tooling.md | 1 + 3 files changed, 69 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..408a9ccf --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna +# +# CodeQL code scanning for the Python code and the workflows. Results go to the Security tab. +# Alerts are advisory: no ruleset requires this workflow (docs/reference/branch-protection.md, "Code scanning"). +# JavaScript and TypeScript are not scanned: the only such files are the engine's test fixtures, which hold +# deliberately flawed code. The Python fixtures are excluded for the same reason. +name: CodeQL + +on: + pull_request: + branches: [dev, main] + push: + branches: [dev] + schedule: + - cron: "17 4 * * 1" + +permissions: + contents: read + +concurrency: + group: codeql-${{ github.ref }} + cancel-in-progress: true + +jobs: + analyze: + name: analyze (${{ matrix.language }}) + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + language: [python, actions] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + languages: ${{ matrix.language }} + build-mode: none + config: | + paths-ignore: + - audit/engine/tests/fixtures + - uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + category: /language:${{ matrix.language }} diff --git a/docs/reference/branch-protection.md b/docs/reference/branch-protection.md index 3a37d78c..76d43191 100644 --- a/docs/reference/branch-protection.md +++ b/docs/reference/branch-protection.md @@ -99,6 +99,22 @@ A pull request into main must come from the dev branch of Nistapp/codebase-agent Pull requests into `dev` do not run this workflow. The `dev` ruleset does not require it. +## Code scanning + +[`.github/workflows/codeql.yml`](../../.github/workflows/codeql.yml) runs CodeQL on the Python code and on the +workflows. It runs on each pull request into `dev` or `main`, on each push to `dev`, and every Monday. It does not +scan the engine's test fixtures (`audit/engine/tests/fixtures/`), because they hold flawed code on purpose. It does +not scan JavaScript or TypeScript, because the only such files are those fixtures. + +Code scanning is advisory. No ruleset requires the CodeQL workflow, and an alert does not block a merge. The +maintainer handles an alert in the Security tab: + +1. Fix the code in a pull request into `dev`. The alert closes when the fix reaches `dev`. +2. Or dismiss the alert as "false positive" or "used in tests", with a one-line reason. + +A later change may add the ruleset rule "Require code scanning results" to `dev-protection.json`. That change needs +the maintainer. + ## Current state | Ruleset | Target | State on GitHub | @@ -111,6 +127,9 @@ On 2026-11-08 the default branch changed from `main` to `dev`. On 2026-11-07 these settings were also turned on: private vulnerability reporting, secret scanning, push protection and Dependabot security updates (see [`SECURITY.md`](../../SECURITY.md)). +On 2026-11-09 the CodeQL workflow was added. Code scanning default setup stays off: the workflow replaces it (see +[Code scanning](#code-scanning)). + On 2026-11-08 "Automatically delete head branches" was turned on. GitHub deletes the head branch of a pull request when the pull request is merged. It does not delete `dev` after a promotion pull request, because the `dev` ruleset blocks deletion. To check the setting, run this command: diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 9f4f0068..883201d6 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -27,6 +27,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | skills-ref | A skill bundle that breaks the Agent Skills specification | `make validate-skills` | Release only | `dev/tools/pyproject.toml` | | `dev/commit_lint.py` | A commit subject that is not Conventional Commits | `make commit-lint` | `commits` | This repository | | The audit engine | A readiness finding that `.audit-baseline.json` does not have | `make self-audit` | `self-audit` | This repository | +| CodeQL | Security flaws in the Python code and the workflows. Advisory: alerts go to the Security tab ([Code scanning](branch-protection.md#code-scanning)) | — | `codeql.yml`, outside `CI Gate` | `.github/workflows/codeql.yml` (action pinned by SHA) | | release-please | Computes the next version from the commit subjects, writes `CHANGELOG.md`, tags and creates the GitHub release (`release-please-config.json`) | — | `release-please.yml`, on each push to `main` | `.github/workflows/release-please.yml` (action pinned by SHA) | | uv | Runs the Python tools through `uvx` and `uv run` | — | All jobs | `dev/tools/pyproject.toml` | | Node | Runs the Node tools through `npx` | — | `docs` | `.nvmrc` | From d21d06140a952898527db0e1e1089c44f293a48e Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 10:24:25 +0530 Subject: [PATCH 017/111] build(dev): update uv to 0.13.5 for two security advisories uv 0.12.10 bundles a CA store that trusts a compromised emSign root (PYSEC-2026-4321, GHSA-hq85-5cpj-mr62) and has a Windows path traversal in wheel extraction (PYSEC-2026-4305, GHSA-2cv4-cqwr-gwf7). osv-scanner found both. 0.13's breaking changes (Python 3.15 as the default download, constraints-file strictness) do not touch this repository: CI installs Python explicitly and nothing here uses constraints files. Roadmap-Step: A2 Co-Authored-By: Claude Opus 5.5 --- dev/tools/pyproject.toml | 2 +- dev/tools/uv.lock | 42 ++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/dev/tools/pyproject.toml b/dev/tools/pyproject.toml index 405d3ada..f15377c0 100644 --- a/dev/tools/pyproject.toml +++ b/dev/tools/pyproject.toml @@ -22,6 +22,6 @@ tools = [ "mypy==2.4.0", "ruff==0.16.10", "skills-ref==0.1.1", - "uv==0.12.10", + "uv==0.13.5", "zizmor==1.32.0", ] diff --git a/dev/tools/uv.lock b/dev/tools/uv.lock index b850f772..fb0c86fc 100644 --- a/dev/tools/uv.lock +++ b/dev/tools/uv.lock @@ -297,7 +297,7 @@ tools = [ { name = "mypy", specifier = "==2.4.0" }, { name = "ruff", specifier = "==0.16.10" }, { name = "skills-ref", specifier = "==0.1.1" }, - { name = "uv", specifier = "==0.12.10" }, + { name = "uv", specifier = "==0.13.5" }, { name = "zizmor", specifier = "==1.32.0" }, ] @@ -1112,28 +1112,28 @@ wheels = [ [[package]] name = "uv" -version = "0.12.10" +version = "0.13.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/48/92/167ad4b0795530383a84560dc8bc8c0afab6b6fb10baba34b2a257fa6894/uv-0.12.10.tar.gz", hash = "sha256:27e8350faffb35d8ecaa5ed39236709809bb9a883f1d11d13c0964d6051b2384", size = 7144864, upload-time = "2026-09-04T23:14:21.301Z" } +sdist = { url = "https://files.pythonhosted.org/packages/de/f0/7565e80415f841c3500ed69d86516c7676e3b24470a218ab61b1f43071a2/uv-0.13.5.tar.gz", hash = "sha256:979409b8c9a78df140ffb19c53f511b7a73526da8d1d5e36558833055e371c6c", size = 7513309, upload-time = "2026-10-22T01:32:33.909Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/d1/b3752f00e67e5bf141adec0225c62a17c1cf3fb524db463c541c080f1237/uv-0.12.10-py3-none-linux_armv6l.whl", hash = "sha256:efbbc586d40014b9dd3bc37a2ad5803d12919458f0f9d16f122db5d91626f5ab", size = 22182923, upload-time = "2026-09-04T23:13:31.502Z" }, - { url = "https://files.pythonhosted.org/packages/ab/4d/ef846ec0a4dfa8d4f47b96a87a1e331143be5a045c28c8b4b0374946455f/uv-0.12.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7e3a70d9dff95481afccd7d41e7ec42fdc230d53487b3e7e633f9b35610401b5", size = 20494379, upload-time = "2026-09-04T23:13:34.339Z" }, - { url = "https://files.pythonhosted.org/packages/04/2e/5614e9a940fe1bf291342410b2cde9fe35a1430135ae7434ea8fd67033f8/uv-0.12.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5dc26c73826d2119292d49d71c5a9d5ba9dd97dd02459d816b8226f47f3dc6bd", size = 17238363, upload-time = "2026-09-04T23:13:36.905Z" }, - { url = "https://files.pythonhosted.org/packages/d1/03/e71936935b8ac26390b280e102b9db823e90adb1134f7e226538054fdece/uv-0.12.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:bd0afae6918795c6a61a649e64d735e339f3d62f8310235da6f19eb3f66f323b", size = 21590025, upload-time = "2026-09-04T23:13:39.588Z" }, - { url = "https://files.pythonhosted.org/packages/b2/c0/c6388a722fb9db8adf8ec8eca762547b5a06ccbd1176571dc584636af5f1/uv-0.12.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:c1fd0fcabc18dccaf541ca44a061c245045c937bec75c0491e94d59cd33ca97f", size = 21677222, upload-time = "2026-09-04T23:13:42.218Z" }, - { url = "https://files.pythonhosted.org/packages/9d/c6/96698aed29f25ff289b28b6312c72e260c56f7e1de2e3f690788bddea19e/uv-0.12.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:de4b06934bd025ceeeb48af5c6f9a82407a2fe2b5b35e456cfb3198ef5029de0", size = 21712683, upload-time = "2026-09-04T23:13:44.927Z" }, - { url = "https://files.pythonhosted.org/packages/df/98/5871a2bff93bd6118ae877b874a09202d39e2831af37f56e2070d5c4bc87/uv-0.12.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b610e8cf7c6d51dc119d352e699b674534d0de327b4d2a67eb5d71ec364b8970", size = 22331081, upload-time = "2026-09-04T23:13:47.833Z" }, - { url = "https://files.pythonhosted.org/packages/3d/b6/d690f34901bdc5094645cf75a00381cfb8f1f28ee87b7089af761094bb6f/uv-0.12.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f43c06b19137bb05877b0ec59c4932878db5d1aeb6f7b5c33259c22e0359c4fe", size = 23785643, upload-time = "2026-09-04T23:13:51.111Z" }, - { url = "https://files.pythonhosted.org/packages/90/da/168a8c69f60b9a2c90afb0b9211911ca42a81300c098d545da911bd4ef8f/uv-0.12.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0334577671a65b621faddfdf5f9e9dd4eb08eaa7231c4405c4a7f4f17d224d9f", size = 23337759, upload-time = "2026-09-04T23:13:54.163Z" }, - { url = "https://files.pythonhosted.org/packages/f0/41/f0e14ba1f881f7126152dcaf04119b3a2bd7566ba1cac1d22834bd134cbf/uv-0.12.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7d6248ad9f2d282fea795f248da8fa666ab382df50d8385bd98e01049440a0c", size = 19978237, upload-time = "2026-09-04T23:13:56.994Z" }, - { url = "https://files.pythonhosted.org/packages/7a/20/acebfb5fc01b29cce5e68b4824fa451d5fb1e569bb2579bcc7233583a894/uv-0.12.10-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:5c0bcf0598742eb417f1aa41ec82f0d89bd7b7f42d0a519bea495a9419d8d8ee", size = 19402684, upload-time = "2026-09-04T23:13:59.579Z" }, - { url = "https://files.pythonhosted.org/packages/49/ee/c40dc8b7b143a6f7843994b1b8a13aa8859d8404973704cd6bad7a26ac8d/uv-0.12.10-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:0ac914e9750bb1de2518d444cc1cff084f4eff3eefe9cdd5814b775443a16df1", size = 22450726, upload-time = "2026-09-04T23:14:02.316Z" }, - { url = "https://files.pythonhosted.org/packages/c0/ec/b5df49f571b58e694ca8c3ddadd89939036ac0fe7ddaadc7600225ea8585/uv-0.12.10-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:46ca83d33bf87a67d0c5aa5b64f5576fc5c8e700a0ecf4900d409f8a1a1c9810", size = 22571602, upload-time = "2026-09-04T23:14:05.088Z" }, - { url = "https://files.pythonhosted.org/packages/4a/3e/f0335142e8a4c6ddc5117dcc9b5fd55c83cdd8e20a7f3be9584282dbfb64/uv-0.12.10-py3-none-musllinux_1_1_i686.whl", hash = "sha256:c49cd1f6e5ded1bd844119da8b184d5c044ae51441c476af2382fdb8af99edb8", size = 21499938, upload-time = "2026-09-04T23:14:08.109Z" }, - { url = "https://files.pythonhosted.org/packages/98/3e/991f08fcab1706175ae30eda6ef483223511679c56fe1032b919ca3ea391/uv-0.12.10-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:a34e1be296e6e30fdbfee9b84a45d082ce7df4dd4e588ebcfc44249505c10f02", size = 22879435, upload-time = "2026-09-04T23:14:11.1Z" }, - { url = "https://files.pythonhosted.org/packages/c4/80/8db01aabf9937cbc228003392ee9fe542c4dad5c3c5b26f2e1f136d8e463/uv-0.12.10-py3-none-win32.whl", hash = "sha256:f7f2bc56fbd367b1066e8461a82e51e8886bbab52b1f8d1793af8ce52232348c", size = 19873629, upload-time = "2026-09-04T23:14:13.755Z" }, - { url = "https://files.pythonhosted.org/packages/a6/88/e980985d93283c546374b6b9dfa486a28091c4c056226711e31bb0f722d1/uv-0.12.10-py3-none-win_amd64.whl", hash = "sha256:2ad71395c31b5db20c56327f62ee0299f75833308d4413284aba87324d092afa", size = 18084435, upload-time = "2026-09-04T23:14:16.323Z" }, - { url = "https://files.pythonhosted.org/packages/84/10/fa542546044f783060d09a624f9964b595c0a0713cf2d89ade69c6667d8b/uv-0.12.10-py3-none-win_arm64.whl", hash = "sha256:4f097c6b7f63eceb3faf5102009f9a8cc22fd7d8a341858d2fd540f46a592f7a", size = 19598033, upload-time = "2026-09-04T23:14:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/24/ff/e30478710f53cf388c892e27e90662aa2acac7a7998bdb18023c595213c1/uv-0.13.5-py3-none-linux_armv6l.whl", hash = "sha256:8d092d07d2389909027c2ed4965ade1bd46bbcecceb26686ff40927684f8c457", size = 18325046, upload-time = "2026-10-22T01:31:37.115Z" }, + { url = "https://files.pythonhosted.org/packages/e3/2a/dcae794203261e49564bb9ff7c4b64a093f69ac73476cc8ce54c1deed3a1/uv-0.13.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:337a14829bb499e114989df4f908f134ff37f1fb6c33497541cfb05b28080e39", size = 17335724, upload-time = "2026-10-22T01:31:40.889Z" }, + { url = "https://files.pythonhosted.org/packages/d2/26/9ac45211123244ea68be852775093af487a64debc13621a2de239f8ebe73/uv-0.13.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a42c85fa8156e1e5e2cecdbc77bb818ea1dcf5f78b8e2a8f84c7fe4f16f0702e", size = 14470685, upload-time = "2026-10-22T01:31:43.897Z" }, + { url = "https://files.pythonhosted.org/packages/e8/2e/5bf9acf9d11fb790420e258a875e13bba36bc890858350642aca185efe0e/uv-0.13.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:2386c2dc9b59fc1d39b1c500e4319d71d3268232eb3b38e29cd6775d6269e8d4", size = 17784583, upload-time = "2026-10-22T01:31:46.95Z" }, + { url = "https://files.pythonhosted.org/packages/f3/70/b02fdcbae77346e3ec1310e7de9dde541252e709f6641db3fbd32cd4da79/uv-0.13.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:4f62680fe58c5a83fd539e38f1956c2b577b1f5939c9020aa4c7f0addda2f4ad", size = 17866408, upload-time = "2026-10-22T01:31:50.352Z" }, + { url = "https://files.pythonhosted.org/packages/fd/b9/6663095851e0943e05e47318d7600e567bc172a1e7a26b21b8e6608e8491/uv-0.13.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e03f5717bdaaf5ab4e2df361d2395e36b515946f56ef4bed88801aeb9959fb0", size = 17753574, upload-time = "2026-10-22T01:31:53.502Z" }, + { url = "https://files.pythonhosted.org/packages/e5/84/1bb89a94486b813bbeaa1b8b491e69a25fa864626711944127b186f903de/uv-0.13.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f867c5ec24c563afaae730a13e46cdf74a1a0b64c134286ff99762a96e2abf40", size = 18502087, upload-time = "2026-10-22T01:31:56.748Z" }, + { url = "https://files.pythonhosted.org/packages/a4/a1/b0b0c7247f3bb0f307eae8a28679ecd0d4b0e2757fcf6c87dea6f679a099/uv-0.13.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3fd5db55c20feb672ea457361aba29ab68b9ae69a14d98ad6c138b5f86dcca4", size = 19927049, upload-time = "2026-10-22T01:31:59.831Z" }, + { url = "https://files.pythonhosted.org/packages/b4/44/6bf11de78ce18869849a14178534b900dbe4e310270b2f60df6503c03137/uv-0.13.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e01481ae5212e45ee7c85fad5fc67f06b0ee0329a7c3c289dc529da884b03881", size = 19662743, upload-time = "2026-10-22T01:32:03.258Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ad/3c2a81d66d1839063662e5b6c0002ec31f8ee987dbc5295747627ca6160a/uv-0.13.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa8d26e08e34c29c0c4e820cf3181757086d547240b4f4d31ff1d8bd810ca946", size = 17211825, upload-time = "2026-10-22T01:32:06.393Z" }, + { url = "https://files.pythonhosted.org/packages/4c/92/d83d019c63c1ccc6ff17df2bfe76270412f9766e7e71aa4d4ccbf018683f/uv-0.13.5-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:91e58e304b63b6c91bac5561459ab0c3ea24f17bc01ea9359d69fed72142978f", size = 16654136, upload-time = "2026-10-22T01:32:10.086Z" }, + { url = "https://files.pythonhosted.org/packages/b8/5a/d42f9d93b322ad36647eb3bf516cc528847c240f630f6df213b6221c5262/uv-0.13.5-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:309a74d3c488bfaa3ba7bff759da73193d8781729c0a01274f4940fd32bb5712", size = 17921105, upload-time = "2026-10-22T01:32:13.174Z" }, + { url = "https://files.pythonhosted.org/packages/97/79/c5c2c39c78cf7a1dd6e041b3f7aa2cc4258d351d42dea06f6fee0bde5daa/uv-0.13.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:469cca66de1b17c75b9b9de806a64cffb57cba5e42b1e30d0bba6fa72a146f58", size = 18103266, upload-time = "2026-10-22T01:32:16.302Z" }, + { url = "https://files.pythonhosted.org/packages/01/2f/e5c70a05013077e5ecadb804418ddbab33f7279f96d052f79f565369b2a7/uv-0.13.5-py3-none-musllinux_1_1_i686.whl", hash = "sha256:2386e8dbfe110ff2f8ec8bd344dd18bf39eb9eb54e1edeeb87c3b85c34229cd3", size = 17842061, upload-time = "2026-10-22T01:32:19.208Z" }, + { url = "https://files.pythonhosted.org/packages/4f/b0/7dba2cb636d7f58e0d74e948ad1afc6b7923e84e82de025e9951a68055db/uv-0.13.5-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:d162669e3a7811c584b03690eaa11e796ad1d309aa8229d5dfe3cbcd68a90400", size = 18963145, upload-time = "2026-10-22T01:32:22.362Z" }, + { url = "https://files.pythonhosted.org/packages/00/7c/e38b312abaae1bd59ff5886126f48e9e223fbb9a8edf9e236ede976f67cb/uv-0.13.5-py3-none-win32.whl", hash = "sha256:f4e55a6e2a1089fe74ed50fbdf2df0f6f336ee148ad232b05932db5deee8007e", size = 16240919, upload-time = "2026-10-22T01:32:25.329Z" }, + { url = "https://files.pythonhosted.org/packages/32/ea/771744a784b16aa21d195cb5d5d5a2878d72fa00fc26f11ea3f9646994fe/uv-0.13.5-py3-none-win_amd64.whl", hash = "sha256:1fb29fc46ba12033e2ef95a95ac79d649e71bbb1358419bbbcbd19d86026ef2f", size = 15358650, upload-time = "2026-10-22T01:32:28.22Z" }, + { url = "https://files.pythonhosted.org/packages/52/67/1f31243f75ece410f2db1afdfc9a9f9722fdee2385e4a347c3ca1dee6d13/uv-0.13.5-py3-none-win_arm64.whl", hash = "sha256:0548460d46c8667488b1f089e936f29f0edfb1cc753f9152ffbd8bc84a46aa20", size = 15533668, upload-time = "2026-10-22T01:32:31.335Z" }, ] [[package]] From a43265966ad5ed76032c00fecd906c19c87e5120 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 10:27:50 +0530 Subject: [PATCH 018/111] build(dev): update mermaid-cli to 12.2.0 for a katex advisory mermaid-cli 12.0.0 pulls katex 0.16.x, affected by GHSA-238p-pmpm-9mq7 (prototype pollution bypasses trust restrictions; fixed in 0.18.2). 12.2.0 depends on katex ^0.18.10. Mermaid blocks still render. Roadmap-Step: A2 Co-Authored-By: Claude Opus 5.5 --- dev/tools/package-lock.json | 29 +++++++++++++++++++++++------ dev/tools/package.json | 2 +- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/dev/tools/package-lock.json b/dev/tools/package-lock.json index bbd577c6..adc75688 100644 --- a/dev/tools/package-lock.json +++ b/dev/tools/package-lock.json @@ -7,7 +7,7 @@ "name": "codebase-agentic-readiness-framework-dev-tools", "license": "AGPL-3.0-or-later", "devDependencies": { - "@mermaid-js/mermaid-cli": "12.0.0", + "@mermaid-js/mermaid-cli": "12.2.0", "markdownlint-cli2": "0.24.0" } }, @@ -279,9 +279,9 @@ } }, "node_modules/@mermaid-js/mermaid-cli": { - "version": "12.0.0", - "resolved": "https://registry.npmjs.org/@mermaid-js/mermaid-cli/-/mermaid-cli-12.0.0.tgz", - "integrity": "sha512-AA18uaJtWVlM5PX7RGBxrVrqTTesmEaWCmfV+l5RkD6A1CnpUAtPpL+wWhkQAZrDSfyAw5FZyj084MsnDIPY3g==", + "version": "12.2.0", + "resolved": "https://registry.npmjs.org/@mermaid-js/mermaid-cli/-/mermaid-cli-12.2.0.tgz", + "integrity": "sha512-oPfUxf4ly4MEf2wIgJh7DvtCz5mQFn9KFqCOZv5k2Vo7AtCAXO3NULy3W3UpNHDrP05Lbld7Gn6XeUgUWVpA2g==", "dev": true, "license": "MIT", "dependencies": { @@ -292,8 +292,8 @@ "chalk": "^6.0.0", "commander": "^15.0.0", "import-meta-resolve": "^4.1.0", - "katex": "^0.16.25", - "mermaid": "^12.0.0", + "katex": "^0.18.10", + "mermaid": "^12.2.0", "p-limit": "^7.0.0" }, "bin": { @@ -309,6 +309,23 @@ "puppeteer": "^25.0.0" } }, + "node_modules/@mermaid-js/mermaid-cli/node_modules/katex": { + "version": "0.18.10", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.18.10.tgz", + "integrity": "sha512-/B6p9eY9DX7aHBfpkHdpirDTZ5QH9xTwL9y837PWuzuv41O5jFdqNQwVQEQsimY0/iVNkwBY4+4hMfhQ7d4Dbw==", + "dev": true, + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "license": "MIT", + "dependencies": { + "commander": "^15.0.0" + }, + "bin": { + "katex": "cli.js" + } + }, "node_modules/@mermaid-js/mermaid-zenuml": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@mermaid-js/mermaid-zenuml/-/mermaid-zenuml-1.0.1.tgz", diff --git a/dev/tools/package.json b/dev/tools/package.json index b972bc37..8f8a7aef 100644 --- a/dev/tools/package.json +++ b/dev/tools/package.json @@ -4,7 +4,7 @@ "description": "The Node dev tools of this repository, as exact versions. Nothing installs this file: the Makefile and dev/mermaid_check.py read each version from here and run the tool through npx. It exists so that Dependabot (the npm entry for /dev/tools in .github/dependabot.yml) sees the tools and, through package-lock.json, their dependencies. After a change, run `npm install --package-lock-only --ignore-scripts` in dev/tools. dev/tests/test_tool_pins.py checks that every version is exact.", "license": "AGPL-3.0-or-later", "devDependencies": { - "@mermaid-js/mermaid-cli": "12.0.0", + "@mermaid-js/mermaid-cli": "12.2.0", "markdownlint-cli2": "0.24.0" } } From 345890bcbf2d89401a238dbbdb8e8a37b0917ff0 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 10:53:55 +0530 Subject: [PATCH 019/111] build(dev): audit the dev tools' dependencies with osv-scanner in make security make security now runs osv-scanner after the secrets scan, on every dev-tool lockfile (uv.lock, package-lock.json and osv-scanner's own go.mod). A known vulnerability fails the gate unless dev/tools/osv-scanner.toml accepts it with a reason and an ignoreUntil date at most six months ahead; test_tool_pins.py checks both. osv-scanner v2.7.0 is a Go `tool` requirement in dev/tools/osv/go.mod, so its version is written once and Dependabot's new gomod entry sees it (ADR 0003). Four indirect Go modules are raised past their advisories. CI's security job and the release gates install Go from go.mod. ADR 0006 records the decision; make setup checks for Go. TOOL-04 now passes. EXEC-01 reports a false FAIL because the engine reads the go directive only from a root go.mod; it is recorded as an owner exception in self-audit.md and the baseline, and an engine fix is planned. Roadmap-Step: A2 Co-Authored-By: Claude Opus 5.5 --- .audit-baseline.json | 93 ++- .github/dependabot.yml | 19 + .github/workflows/ci.yml | 6 + .github/workflows/release.yml | 6 + .secrets.baseline | 8 +- AGENTS.md | 10 +- CONTRIBUTING.md | 3 +- Makefile | 16 +- RELEASE_PROCESS.md | 6 +- audit/engine/AGENTS.md | 2 +- dev/tests/test_tool_pins.py | 83 ++- dev/tools/osv-scanner.toml | 23 + dev/tools/osv/go.mod | 206 ++++++ dev/tools/osv/go.sum | 670 ++++++++++++++++++ ...-dependabot-security-pull-requests-only.md | 7 +- ...rb-audits-dependencies-with-osv-scanner.md | 91 +++ docs/architecture/adrs/README.md | 1 + docs/reference/dev-tooling.md | 25 +- docs/reference/self-audit.md | 4 +- 19 files changed, 1209 insertions(+), 70 deletions(-) create mode 100644 dev/tools/osv-scanner.toml create mode 100644 dev/tools/osv/go.mod create mode 100644 dev/tools/osv/go.sum create mode 100644 docs/architecture/adrs/0006-security-verb-audits-dependencies-with-osv-scanner.md diff --git a/.audit-baseline.json b/.audit-baseline.json index cc9eac64..19e63241 100644 --- a/.audit-baseline.json +++ b/.audit-baseline.json @@ -22,7 +22,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "index store could not be located; searched: user-level locations (not read: --no-user-config), /home/kc/Projects/Nistapp-agentic-frameworks/codebase-agentic-readiness-framework/.codebase-memory/graph.db.zst", + "summary": "index store could not be located; searched: user-level locations (not read: --no-user-config), /home/kc/Projects/Nistapp-agentic-frameworks/car-framework-track-a/.codebase-memory/graph.db.zst", "tier": "A", "title": "An index exists for this checkout", "verdict": "UNKNOWN" @@ -135,7 +135,7 @@ "verb": "check" }, { - "command": "git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline", + "command": "git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline && go -C dev/tools/osv tool osv-scanner scan source --no-call-analysis=go --config=$(CURDIR)/dev/tools/osv-scanner.toml $(foreach f,$(DEP_LOCKFILES),--lockfile=$(CURDIR)/$(f))", "resolved": true, "runner": "Makefile", "verb": "security" @@ -161,7 +161,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "20 boundary rules reference real paths", + "summary": "19 boundary rules reference real paths", "tier": "B", "title": "Architectural boundaries declared", "verdict": "PASS" @@ -262,7 +262,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "39 markdown file(s) under docs/", + "summary": "44 markdown file(s) under docs/", "tier": "A", "title": "docs/ exists", "verdict": "PASS" @@ -301,7 +301,7 @@ "scored": true, "severity": "COSMETIC", "status": "implemented", - "summary": "3 ADR(s) indexed in docs/architecture/adrs/README.md", + "summary": "6 ADR(s) indexed in docs/architecture/adrs/README.md", "tier": "A", "title": "ADR directory and index", "verdict": "PASS" @@ -349,7 +349,7 @@ "verb": "check" }, { - "command": "git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline", + "command": "git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline && go -C dev/tools/osv tool osv-scanner scan source --no-call-analysis=go --config=$(CURDIR)/dev/tools/osv-scanner.toml $(foreach f,$(DEP_LOCKFILES),--lockfile=$(CURDIR)/$(f))", "resolved": true, "runner": "Makefile", "verb": "security" @@ -453,10 +453,10 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "security verb on Makefile runs no dependency audit (git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline)", + "summary": "security -> git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline && go -C dev/tools/osv tool osv-scanner scan source --no-call-analysis=go --config=$(CURDIR)/dev/tools/osv-scanner.toml $(foreach f,$(DEP_LOCKFILES),--lockfile=$(CURDIR)/$(f))", "tier": "A", "title": "Dependency / security scan verb", - "verdict": "PARTIAL" + "verdict": "PASS" }, { "data": {}, @@ -492,7 +492,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "PR-triggered: .github/workflows/ci.yml (github); .github/workflows/pr-source-branch.yml (github)", + "summary": "PR-triggered: .github/workflows/ci.yml (github); .github/workflows/codeql.yml (github); .github/workflows/pr-source-branch.yml (github)", "tier": "B", "title": "Pipeline present, triggered on PRs", "verdict": "PASS" @@ -585,6 +585,7 @@ ".python-version", "dev/tools/pyproject.toml", "dev/tools/package.json", + "dev/tools/osv/go.mod", "lefthook.yml", "dev", ".github/", @@ -625,6 +626,8 @@ "docs/architecture/adrs/README.md", ".env", "RELEASE_PROCESS.md", + "CHANGELOG.md", + ".release-please-manifest.json", ".github/rulesets/", "docs/reference/branch-protection.md", "audit/tools/", @@ -642,7 +645,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "the prohibitions section of AGENTS.md names paths that exist: Makefile, audit/engine/, AGENTS.md, .python-version, dev/tools/pyproject.toml, dev/tools/package.json, lefthook.yml, dev (+44 more)", + "summary": "the prohibitions section of AGENTS.md names paths that exist: Makefile, audit/engine/, AGENTS.md, .python-version, dev/tools/pyproject.toml, dev/tools/package.json, dev/tools/osv/go.mod, lefthook.yml (+45 more)", "tier": "B", "title": "Allow/deny artifact present", "verdict": "PASS" @@ -703,10 +706,10 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "pinned: node (.nvmrc); python (.python-version)", + "summary": "unpinned: go", "tier": "A", "title": "Toolchain pinned / wrapper committed", - "verdict": "PASS" + "verdict": "FAIL" }, { "data": {}, @@ -716,7 +719,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "consistency unverified: dev/tools/package-lock.json, dev/tools/uv.lock", + "summary": "consistency unverified: dev/tools/osv/go.sum, dev/tools/package-lock.json, dev/tools/uv.lock", "tier": "B", "title": "Lockfile committed and consistent", "verdict": "PARTIAL" @@ -801,7 +804,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "test verb on Makefile (cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests); 82 test file(s)", + "summary": "test verb on Makefile (cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests); 84 test file(s)", "tier": "A", "title": "Test verb exists, suite non-empty", "verdict": "PASS" @@ -834,7 +837,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "skip/xfail census across 82 test file(s): python 1; total 1", + "summary": "skip/xfail census across 84 test file(s): python 1; total 1", "tier": "B", "title": "Skip / xfail census", "verdict": "PASS" @@ -919,10 +922,10 @@ }, { "data": { - "denominator": 1138, + "denominator": 1152, "kind": "ratio", "method": "python: module-level def/class not starting with '_'", - "numerator": 453, + "numerator": 459, "unit": "public symbols" }, "id": "NAV-05", @@ -931,7 +934,7 @@ "scored": false, "severity": "COSMETIC", "status": "implemented", - "summary": "453/1138 public symbols documented (40%); method: python: module-level def/class not starting with '_'", + "summary": "459/1152 public symbols documented (40%); method: python: module-level def/class not starting with '_'", "tier": "B", "title": "Public-symbol documentation coverage", "verdict": "PARTIAL" @@ -1274,6 +1277,16 @@ } ], "findings": [ + { + "check": "EXEC-01", + "evidence": [], + "id": "EXEC-01:go", + "phase": 1, + "remediation": "Commit the pin appropriate to go \u2014 .tool-versions \u2014 so the agent's runtime is the repository's, not the host's.", + "severity": "BLOCKER", + "statement": "An agent cannot reproduce the go toolchain because the detected go project pins no toolchain version.", + "verdict": "FAIL" + }, { "check": "IDX-01", "evidence": [ @@ -1432,28 +1445,12 @@ "severity": "DEGRADER", "statement": "An agent cannot trust that secret shapes cannot be committed because ignore rules are incomplete (root-anchored only: dotenv, dotenv variant (the rule covers the root, not nested files such as `a/.env`)).", "verdict": "PARTIAL" - }, - { - "check": "TOOL-04", - "evidence": [ - { - "line": null, - "note": "git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline", - "path": "Makefile" - } - ], - "id": "TOOL-04", - "phase": 1, - "remediation": "Make the security verb run a dependency audit (npm audit, pip-audit, osv-scanner, cargo audit, govulncheck, snyk).", - "severity": "DEGRADER", - "statement": "An agent cannot know whether a dependency carries a known vulnerability because a security verb exists but it does not run a dependency-audit check.", - "verdict": "PARTIAL" } ], "inventory": { - "files_inspected": 325, + "files_inspected": 339, "files_set_aside": 460, - "files_skipped": 2250, + "files_skipped": 3, "set_aside": [ { "files": 3, @@ -1531,22 +1528,22 @@ } ], "provenance": { - "framework_revision": "bdcbe3d516552fb17e1606cd8c77ae86635f688d", - "git_branch": "chore/review-for-release", + "framework_revision": "b678cd339ce6114fd1ed6df18a68d6b5596a8557", + "git_branch": "chore/v0.3.3-track-a-tooling", "git_dirty": true, - "git_sha": "bdcbe3d516552fb17e1606cd8c77ae86635f688d", + "git_sha": "b678cd339ce6114fd1ed6df18a68d6b5596a8557", "phase": 1, "probes_enabled": false, "ruleset_hash": "970e6792f98eeeb5", "ruleset_revision": "2026-11-06.1", - "scanned_at": "2026-11-08T07:19:00+00:00", - "target": "/home/kc/Projects/Nistapp-agentic-frameworks/codebase-agentic-readiness-framework", + "scanned_at": "2026-11-09T05:17:36+00:00", + "target": "/home/kc/Projects/Nistapp-agentic-frameworks/car-framework-track-a", "tool": "python-agentic-audit", "user_config": { "paths_read": [], "read_allowed": false }, - "version": "0.1.0" + "version": "0.3.2" }, "schema_version": "2", "stack": { @@ -1554,6 +1551,7 @@ "github" ], "ecosystems": [ + "go", "node", "python" ], @@ -1562,17 +1560,18 @@ ], "notes": [], "package_managers": [ + "gomod", "npm", "uv" ], "test_frameworks": [] }, "summary": { - "blockers": 2, + "blockers": 3, "checks": { "attest": 0, - "fail": 1, - "partial": 6, + "fail": 2, + "partial": 5, "pass": 58, "unknown": 4 }, @@ -1581,8 +1580,8 @@ "checks_scoreable": 55, "cosmetic": 0, "coverage_note": "55 of 55 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", - "degraders": 3, - "phase1_score": 0.8689 + "degraders": 2, + "phase1_score": 0.8525 }, "unattested": [ "index-in-use", diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 38901d5a..5b1aa04e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -69,6 +69,25 @@ updates: commit-message: prefix: "build" + # osv-scanner, the dependency audit in `make security`, pinned as a Go `tool` in dev/tools/osv/go.mod with its + # dependencies. The Makefile runs it with `go -C dev/tools/osv tool osv-scanner`. + - package-ecosystem: gomod + directory: /dev/tools/osv + schedule: + interval: weekly + cooldown: + default-days: 7 + open-pull-requests-limit: 0 + ignore: + - dependency-name: "*" + update-types: ["version-update:semver-major"] + groups: + security-fixes: + applies-to: security-updates + patterns: ["*"] + commit-message: + prefix: "build" + # Data, not dependencies: the greenfield template and the engine's test fixtures. No pull request, ever. - package-ecosystem: npm directories: ["/greenfield/templates", "/audit/engine/tests/fixtures/**"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5957c221..9a182ff4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ # Copyright (C) 2026 K. C. Ramakrishna # # CI runs the same verbs a human or agent runs locally (see the Makefile): `make check` and `make security`. +# `make security` needs the network: its dependency audit (osv-scanner) queries osv.dev. # Python 3.11 is the supported minimum and 3.14 the newest release (Q8); Windows is supported through WSL only. # The `docs` job runs `make docs-lint` and the Mermaid check when a Markdown file changed; it needs Node, so it # stays out of `make check` and the matrix. It also annotates the prose of the changed files, which never fails. @@ -56,6 +57,11 @@ jobs: - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version-file: dev/tools/pyproject.toml + # Go builds osv-scanner, the dependency audit in `make security`, from dev/tools/osv/go.mod. + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: dev/tools/osv/go.mod + cache-dependency-path: dev/tools/osv/go.sum - run: make security # The engine's and the recipe tools' suites under coverage.py; fails below the floor in .coveragerc. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e2439b8d..293b9689 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,6 +73,12 @@ jobs: - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version-file: dev/tools/pyproject.toml + # Go builds osv-scanner for the dependency audit in `make security`. No cache in a release job: a cache + # written by another workflow must not reach the release files. + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: dev/tools/osv/go.mod + cache: false - name: Gates run: | make check diff --git a/.secrets.baseline b/.secrets.baseline index ae38561d..2d2cda55 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -131,16 +131,16 @@ { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", - "hashed_secret": "63215b3c1959cb7cfcaaad73ca8b8ebbbcf1c805", + "hashed_secret": "f7a50539d1173ad5105d5dc9fe93786e0e3fec89", "is_verified": false, - "line_number": 1534 + "line_number": 1531 }, { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", "hashed_secret": "228ea6c3984c8d1226e0ffc33455f16dce652d01", "is_verified": false, - "line_number": 1540 + "line_number": 1537 } ], "audit/engine/audit/rules/checks/con.py": [ @@ -1458,5 +1458,5 @@ } ] }, - "generated_at": "2026-11-09T05:58:50Z" + "generated_at": "2026-11-09T06:52:39Z" } diff --git a/AGENTS.md b/AGENTS.md index ba543d64..d98d38ac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ The rest of this file is for changes to this repository. The framework's seven verbs run through the root `Makefile`. CI runs the same commands. ```bash -make setup # check the prerequisites (Python 3.11+, uv, git); nothing is installed +make setup # check the prerequisites (Python 3.11+, uv, git, Go); nothing is installed make format # rewrite formatting in place make format:check # read-only: formatting and lint rules (alias: make format-check) make lint # read-only lint @@ -26,7 +26,7 @@ make typecheck # mypy, strict, on audit/tools, implement/tools and dev, and make test # the engine's suite, then the audit tools', the implement tools' and the dev checks' make check # THE gate: format:check, typecheck, test, the engine's --verify-rules, the schema # cross-check, recipe-lint, workflow-lint (actionlint, zizmor, check-jsonschema) -make security # secrets scan against .secrets.baseline +make security # secrets scan, then the dependency audit (osv-scanner; needs Go and the network) make commit-lint # Conventional Commits subjects on this branch (RANGE=origin/dev..HEAD); CI runs it on each PR ``` @@ -49,8 +49,10 @@ no vendor-specific instruction file exists beside it. - Run `make check` and `make security` before you propose a change. Do not run an ad-hoc substitute. - When you change Markdown, also run `make docs-lint` (needs Node; lints the ratchet path list in the `Makefile`) and validate every Mermaid block you add (`make docs-mermaid FILES=`). -- Python 3.11 or newer (pinned in `.python-version`), and `uv` for the pinned dev tools (ruff, mypy, detect-secrets). - Each tool version is written once, in `dev/tools/pyproject.toml` or `dev/tools/package.json` (ADR 0003); never +- Python 3.11 or newer (pinned in `.python-version`), `uv` for the pinned dev tools (ruff, mypy, detect-secrets), + and Go 1.27 or newer for osv-scanner in `make security`. + Each tool version is written once, in `dev/tools/pyproject.toml`, `dev/tools/package.json` or + `dev/tools/osv/go.mod` (ADR 0003); never write a version in the `Makefile`, a hook or a workflow. Nothing is installed into the project. - Local hooks are `lefthook.yml` (a stop on commits to `dev` and `main`, format check on staged Python, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fe650af2..db578023 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,6 +29,7 @@ the Makefile. The rules an agent follows are in [`AGENTS.md`](AGENTS.md); they a | Python 3.11 or newer | Everything | Pinned in `.python-version`; CI runs 3.11 and 3.14 | | `git` | Everything | | | [`uv`](https://docs.astral.sh/uv/) | `make check`, `make security`, `make coverage` | The dev tools (ruff, mypy, detect-secrets, coverage.py) run through `uvx` at the versions pinned in `dev/tools/`; nothing is installed into the project | +| [Go](https://go.dev/doc/install) 1.27 or newer | `make security` | Builds the pinned osv-scanner from `dev/tools/osv/go.mod` on first use. `make security` also needs the network | | `make` | Every command below | Linux and macOS. On Windows use WSL | | Node 22 | `make docs-lint` | The pinned `markdownlint-cli2` runs through `npx`; not needed for `make check` | | Chrome, through Puppeteer | `make docs-mermaid` | Only when you add or change a Mermaid block; the pinned mermaid-cli fetches it on first use | @@ -46,7 +47,7 @@ Run these before you propose a change. CI runs the same commands. | `make check` | The gate: formatting and lint, `mypy` strict, the engine's, the tools' and the dev checks' tests, the engine's rule check, the schema cross-check, the recipe linter and the workflow lint. It takes a couple of minutes | | `make workflow-lint` | Part of `make check`. actionlint, zizmor and check-jsonschema on `.github/`: workflow mistakes, workflow security, and `dependabot.yml` against its schema. Offline | | `make commit-lint` | Checks that each commit subject on your branch follows Conventional Commits (`RANGE=origin/dev..HEAD` by default). CI runs the same check on every pull request | -| `make security` | A secrets scan against `.secrets.baseline` | +| `make security` | A secrets scan against `.secrets.baseline`, then a dependency audit of the dev tools (osv-scanner) | | `make docs-lint` | Markdown lint over the ratchet path list in the `Makefile`. Run it when Markdown changed | | `make docs-prose FILES=` | The plain-language check from [`docs/COMMUNICATION_RULES.md`](docs/COMMUNICATION_RULES.md). Advisory: it prints warnings and notices and never fails. Fix the warnings on the lines you changed | | `make docs-mermaid FILES=` | Renders every Mermaid block in the files. Run it when you added or changed one | diff --git a/Makefile b/Makefile index c90f0be0..59c30dfb 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,8 @@ # # The framework's command surface for this repository: seven verbs, the same names a human, an agent # and CI run. The runtime code is standard-library Python; the dev tools below are pinned in dev/tools/ and run -# through uvx, so nothing is installed into the project. Needs make, git, python3 (3.11+) and uv. +# through uvx, so nothing is installed into the project. Needs make, git, python3 (3.11+) and uv; `make security` +# also needs Go (1.27+) and the network. # # make format rewrite formatting in place # make format:check read-only: formatting and lint rules, and the ruff count against .quality-baseline.json @@ -13,7 +14,8 @@ # make test the engine's unittest suite, then the recipe tools' (the catalogue checks), then the dev checks' # make check THE gate: format:check -> typecheck -> test, plus the engine's rule check, the schema # cross-check, the recipe linter and the workflow lint -# make security secrets scan against the committed baseline +# make security secrets scan against the committed baseline, then the dependency audit (osv-scanner) of +# the dev tools' lockfiles; needs Go and the network # make coverage (extra) both unittest suites under coverage.py; fails below the floor in .coveragerc # make self-audit (extra) the engine on this repository, read-only, against .audit-baseline.json; exit 2 on a regression # @@ -44,6 +46,8 @@ SKILLS_REF := uvx --from skills-ref==$(call pin,skills-ref) agentskills MARKDOWNLINT := npx --yes markdownlint-cli2@$(call npm_pin,markdownlint-cli2) # The compare step of the quality baseline (.quality-baseline.json): fails when a tool's full-tree count goes up. QUALITY_BASELINE := $(PYTHON) dev/quality_baseline.py check +# The lockfiles that `make security` audits: every dev tool, Python, Node and Go. +DEP_LOCKFILES := dev/tools/uv.lock dev/tools/package-lock.json dev/tools/osv/go.mod COVERAGE := uvx coverage@$(call pin,coverage) # Workflow and GitHub-config checks: actionlint (syntax, expressions, shellcheck on run: blocks), zizmor (workflow # and Dependabot security, offline) and check-jsonschema (the SchemaStore schemas it bundles, offline). @@ -91,7 +95,8 @@ setup: @$(PYTHON) -c 'import sys; sys.exit(0 if sys.version_info >= (3, 11) else "setup: Python 3.11 or newer is required (see .python-version)")' @command -v uv >/dev/null || { echo "setup: uv is required (https://docs.astral.sh/uv/)" >&2; exit 1; } @command -v git >/dev/null || { echo "setup: git is required" >&2; exit 1; } - @echo "setup: prerequisites present (python, uv, git); nothing to install" + @command -v go >/dev/null || { echo "setup: Go 1.27 or newer is required for make security (https://go.dev/doc/install)" >&2; exit 1; } + @echo "setup: prerequisites present (python, uv, git, go); nothing to install" format: $(RUFF) format audit implement dev @@ -174,8 +179,13 @@ check: format-check typecheck test verify-rules crosscheck recipe-lint workflow- # No third-party runtime dependencies exist to audit, so the gate is a secrets scan. The baseline # lists the deliberate key-shaped strings in the fixtures and tests; a new hit fails the scan. +# The dependency audit: osv-scanner, pinned in dev/tools/osv/go.mod (a Go `tool` directive, so Dependabot sees it), +# reads the dev tools' lockfiles and fails on a known vulnerability that dev/tools/osv-scanner.toml does not accept. +# --no-call-analysis=go: the module holds no Go code of its own, so govulncheck has nothing to analyse. +# The audit engine's TOOL-04 reads this recipe and needs the tool name written out, not hidden in a variable. security: git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline + go -C dev/tools/osv tool osv-scanner scan source --no-call-analysis=go --config=$(CURDIR)/dev/tools/osv-scanner.toml $(foreach f,$(DEP_LOCKFILES),--lockfile=$(CURDIR)/$(f)) # Run this after a deliberate change to a fixture or test that holds a key-shaped string, then review the diff. update-secrets-baseline: diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 7f011aba..812d9093 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -350,7 +350,8 @@ The fix needs a major version, or the alert is on the template or a fixture. Each dev tool version is written once, in `dev/tools/`. The `Makefile`, `lefthook.yml`, `dev/mermaid_check.py` and the workflows read it from there. `dev/tests/test_tool_pins.py` fails when a version is written anywhere else. -1. Change the version in `dev/tools/pyproject.toml` (Python tools) or `dev/tools/package.json` (Node tools). +1. Change the version in `dev/tools/pyproject.toml` (Python tools), `dev/tools/package.json` (Node tools) or + `dev/tools/osv/go.mod` (osv-scanner). 2. Make the lockfile again: ```bash @@ -358,6 +359,9 @@ the workflows read it from there. `dev/tests/test_tool_pins.py` fails when a ver (cd dev/tools && npm install --package-lock-only --ignore-scripts) ``` + For osv-scanner, or one of its Go dependencies, run + `go -C dev/tools/osv get @`, then `go -C dev/tools/osv mod tidy`. + 3. Run `make check`. If ruff or markdownlint changed a rule, fix the new findings in the same branch. 4. Commit the manifest and its lockfile together, for example `build(dev): update ruff to 0.17.0`. diff --git a/audit/engine/AGENTS.md b/audit/engine/AGENTS.md index ef85463c..d929931a 100644 --- a/audit/engine/AGENTS.md +++ b/audit/engine/AGENTS.md @@ -107,7 +107,7 @@ The framework's verbs are the contract. The runner is the **root `Makefile`** of | `typecheck` | `make typecheck` | mypy strict, on `audit/tools`, `implement/tools`, `dev` and this directory's `build.py`. The rest of the engine joins later through a ratchet. | | `test` | `make test` | `python3 -m unittest discover -s tests` in this directory, then the audit tools' suite in `audit/tools/tests`, the implement tools' in `implement/tools/tests`, then the dev checks' in `dev/tests`. | | `check` | `make check` | `format:check`, `typecheck`, `test`, `--verify-rules`, the schema cross-check (`crosscheck`) and `recipe-lint`. | -| `security` | `make security` | Secrets scan against the committed baseline. No dependency audit: there are no dependencies. | +| `security` | `make security` | Secrets scan against the committed baseline, then osv-scanner on the dev tools' lockfiles. The engine itself has no dependencies to audit. | The dev tools (ruff, mypy, detect-secrets) are pinned in `dev/tools/pyproject.toml` and run through `uvx`. They are never imported by the engine's `audit/` package or by `build.py`, so the standard-library-only rule stands. diff --git a/dev/tests/test_tool_pins.py b/dev/tests/test_tool_pins.py index 81fb8376..d5b1aa7f 100644 --- a/dev/tests/test_tool_pins.py +++ b/dev/tests/test_tool_pins.py @@ -1,15 +1,17 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # Copyright (C) 2026 K. C. Ramakrishna -"""The dev tool versions live in two manifests only: ``dev/tools/pyproject.toml`` and ``dev/tools/package.json``. +"""The dev tool versions live in three manifests only: ``dev/tools/pyproject.toml``, ``dev/tools/package.json`` and, +for osv-scanner, the Go module ``dev/tools/osv/go.mod``. -Dependabot reads those two files and their lockfiles (``.github/dependabot.yml``). A version that is also written +Dependabot reads those files and their lockfiles (``.github/dependabot.yml``). A version that is also written somewhere else would not move with a Dependabot pull request, so these tests fail when a pin is not exact, when a lockfile disagrees with its manifest, or when a file that runs a tool repeats its version. """ from __future__ import annotations +import datetime import json import re import shutil @@ -25,6 +27,13 @@ TOOL_LOCK = TOOLS / "uv.lock" NODE_PINS = TOOLS / "package.json" NODE_LOCK = TOOLS / "package-lock.json" +GO_MOD = TOOLS / "osv" / "go.mod" +GO_SUM = TOOLS / "osv" / "go.sum" +OSV_CONFIG = TOOLS / "osv-scanner.toml" +OSV_MODULE = "github.com/google/osv-scanner/v2" +EXACT_GO = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+$") +#: The longest an accepted vulnerability may stay accepted before someone looks again. +MAX_IGNORE_DAYS = 183 # The files that run a dev tool. Each one must read the version from a manifest, never write it. CONSUMERS = [ @@ -102,6 +111,67 @@ def test_package_lock_records_the_pins(self) -> None: self.assertEqual(lock["packages"][f"node_modules/{name}"]["version"], version) +def go_requirements() -> dict[str, str]: + """``module -> version`` for each ``require`` line of the osv-scanner tool module.""" + text = GO_MOD.read_text(encoding="utf-8") + block = re.search(r"^require \((.*?)^\)", text, re.S | re.M) + lines = block.group(1).splitlines() if block else [] + lines += [m.group(1) for m in re.finditer(r"^require (\S+ \S+)", text, re.M)] + requirements: dict[str, str] = {} + for line in lines: + fields = line.split("//")[0].split() + if len(fields) == 2: + requirements[fields[0]] = fields[1] + return requirements + + +class GoModulePinsOsvScanner(unittest.TestCase): + """osv-scanner is a Go program. Its version is a Go `tool` requirement, so Dependabot's gomod entry sees it.""" + + def test_go_mod_declares_osv_scanner_as_a_tool(self) -> None: + text = GO_MOD.read_text(encoding="utf-8") + self.assertRegex(text, rf"(?m)^tool {re.escape(OSV_MODULE)}/cmd/osv-scanner$") + + def test_osv_scanner_is_an_exact_release(self) -> None: + version = go_requirements().get(OSV_MODULE) + self.assertIsNotNone(version, f"{GO_MOD.name} does not require {OSV_MODULE}") + self.assertRegex(version or "", EXACT_GO, "osv-scanner must be a tagged release, not a pseudo-version") + + def test_go_sum_holds_the_pinned_osv_scanner(self) -> None: + version = go_requirements()[OSV_MODULE] + self.assertIn(f"{OSV_MODULE} {version} h1:", GO_SUM.read_text(encoding="utf-8")) + + +class AcceptedVulnerabilitiesAreReviewed(unittest.TestCase): + """Each entry of dev/tools/osv-scanner.toml says why, and expires within six months.""" + + def entries(self) -> list[dict[str, object]]: + data = tomllib.loads(OSV_CONFIG.read_text(encoding="utf-8")) + self.assertEqual(set(data) - {"IgnoredVulns"}, set(), "only IgnoredVulns belongs in the config") + entries: list[dict[str, object]] = data.get("IgnoredVulns", []) + return entries + + def test_each_entry_has_an_id_a_reason_and_a_date(self) -> None: + for entry in self.entries(): + with self.subTest(entry=entry.get("id")): + self.assertIsInstance(entry.get("id"), str) + self.assertGreaterEqual(len(str(entry.get("reason", "")).strip()), 40, "write a real reason") + self.assertIsInstance(entry.get("ignoreUntil"), datetime.date) + + def test_no_entry_is_accepted_for_more_than_six_months(self) -> None: + latest = datetime.date.today() + datetime.timedelta(days=MAX_IGNORE_DAYS) + for entry in self.entries(): + until = entry["ignoreUntil"] + assert isinstance(until, datetime.date) + day = until.date() if isinstance(until, datetime.datetime) else until + with self.subTest(entry=entry["id"]): + self.assertLessEqual(day, latest) + + def test_ids_are_unique(self) -> None: + ids = [entry["id"] for entry in self.entries()] + self.assertEqual(len(ids), len(set(ids))) + + class NoOtherFileRepeatsAVersion(unittest.TestCase): def test_consumers_do_not_write_a_pinned_version(self) -> None: pins = {**python_pins(), **node_pins()} @@ -131,6 +201,15 @@ def test_tool_commands_carry_the_manifest_versions(self) -> None: self.assertIn(f"coverage@{python['coverage']}", output) self.assertIn(f"markdownlint-cli2@{node['markdownlint-cli2']}", output) + def test_security_audits_every_dev_tool_lockfile(self) -> None: + output = self.dry_run("security") + self.assertIn("go -C dev/tools/osv tool osv-scanner scan source", output) + self.assertIn(f"--config={REPO}/dev/tools/osv-scanner.toml", output) + for lockfile in (TOOL_LOCK, NODE_LOCK, GO_MOD): + with self.subTest(lockfile=lockfile.name): + self.assertIn(f"--lockfile={lockfile}", output) + self.assertNotIn(go_requirements()[OSV_MODULE], output, "the Makefile must not write the osv-scanner version") + def test_a_missing_pin_stops_make(self) -> None: result = subprocess.run( ["make", "-n", "-s", "-C", str(REPO), "lint", "TOOL_PINS=/dev/null"], diff --git a/dev/tools/osv-scanner.toml b/dev/tools/osv-scanner.toml new file mode 100644 index 00000000..55314ee5 --- /dev/null +++ b/dev/tools/osv-scanner.toml @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna +# +# Known vulnerabilities that `make security` (osv-scanner) accepts for now. Every other one fails the gate. +# Each entry needs an `id`, an `ignoreUntil` date at most six months ahead and a `reason` that says why the +# vulnerability cannot reach this repository or why no fix exists yet. dev/tests/test_tool_pins.py checks this. +# When `ignoreUntil` passes, the scan fails again: fix it or renew the entry with a new reason. +# How to read the scan: docs/reference/dev-tooling.md, "Dependency audit". + +[[IgnoredVulns]] +id = "GHSA-gg6h-894w-4m9c" +ignoreUntil = 2027-02-09 +reason = "re2js 2.8.6: quadratic-time lookbehind matching. markdownlint 0.42.0 pins it exactly, and markdownlint-cli2 0.24.0 is the newest release. It only lints this repository's own Markdown, which a maintainer reviews in each pull request." + +[[IgnoredVulns]] +id = "GHSA-238p-pmpm-9mq7" +ignoreUntil = 2027-02-09 +reason = "katex 0.16.47: prototype pollution bypasses trust restrictions when KaTeX renders. Reached only through markdownlint 0.42.0 (micromark-extension-math), which parses Markdown and renders no HTML. No markdownlint release raises it yet." + +[[IgnoredVulns]] +id = "GO-2026-5932" +ignoreUntil = 2027-02-09 +reason = "golang.org/x/crypto: the advisory covers the deprecated openpgp package in every version; no tagged release fixes it. The osv-scanner binary does not link golang.org/x/crypto/openpgp (checked with go list -deps on 2026-11-09)." diff --git a/dev/tools/osv/go.mod b/dev/tools/osv/go.mod new file mode 100644 index 00000000..675849ce --- /dev/null +++ b/dev/tools/osv/go.mod @@ -0,0 +1,206 @@ +module github.com/Nistapp/codebase-agentic-readiness-framework/dev/tools/osv + +go 1.27.0 + +tool github.com/google/osv-scanner/v2/cmd/osv-scanner + +require ( + bitbucket.org/creachadair/stringset v0.0.14 // indirect + charm.land/glamour/v2 v2.0.1 // indirect + charm.land/lipgloss/v2 v2.0.6 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + dario.cat/mergo v1.0.2 // indirect + deps.dev/api/v3 v3.0.0-20260811035547-133c155ce203 // indirect + deps.dev/api/v3alpha v0.0.0-20260811035547-133c155ce203 // indirect + deps.dev/util/maven v0.0.0-20260928000106-0e158e8db197 // indirect + deps.dev/util/pypi v0.0.0-20260727054525-2946ae4a6141 // indirect + deps.dev/util/resolve v0.0.0-20260727054525-2946ae4a6141 // indirect + deps.dev/util/semver v0.0.0-20260727054525-2946ae4a6141 // indirect + github.com/BurntSushi/toml v1.6.0 // indirect + github.com/CycloneDX/cyclonedx-go v0.12.0 // indirect + github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect + github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect + github.com/ProtonMail/go-crypto v1.4.1 // indirect + github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a // indirect + github.com/Velocidex/ordereddict v0.0.0-20250821063524-02dc06e46238 // indirect + github.com/Velocidex/yaml/v2 v2.2.8 // indirect + github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6 // indirect + github.com/agext/levenshtein v1.2.3 // indirect + github.com/alecthomas/chroma/v2 v2.19.0 // indirect + github.com/anchore/go-lzo v0.1.0 // indirect + github.com/anchore/go-struct-converter v0.1.0 // indirect + github.com/atotto/clipboard v0.1.4 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/ayoubfaouzi/pkcs7 v0.2.3 // indirect + github.com/bazelbuild/buildtools v0.0.0-20260716142318-04cf7de1434f // indirect + github.com/canonical/chisel-manifest v1.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/bubbles v1.0.0 // indirect + github.com/charmbracelet/bubbletea v1.3.10 // indirect + github.com/charmbracelet/colorprofile v0.4.3 // indirect + github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 // indirect + github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 // indirect + github.com/charmbracelet/x/ansi v0.11.8 // indirect + github.com/charmbracelet/x/cellbuf v0.0.15 // indirect + github.com/charmbracelet/x/exp/slice v0.0.0-20260803091719-3755ebad01b1 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/charmbracelet/x/termios v0.1.1 // indirect + github.com/charmbracelet/x/windows v0.2.2 // indirect + github.com/clipperhouse/displaywidth v0.11.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/cloudflare/circl v1.6.4 // indirect + github.com/compose-spec/compose-go/v2 v2.14.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/typeurl/v2 v2.3.0 // indirect + github.com/cyphar/filepath-securejoin v0.7.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/deitch/magic v0.0.0-20240306090643-c67ab88f10cb // indirect + github.com/diskfs/go-diskfs v1.7.0 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/djherbis/times v1.6.0 // indirect + github.com/dlclark/regexp2 v1.12.0 // indirect + github.com/docker/cli v29.7.1+incompatible // indirect + github.com/docker/docker-credential-helpers v0.9.8 // indirect + github.com/docker/go-connections v0.8.1 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/dsoprea/go-exfat v0.0.0-20190906070738-5e932fbdb589 // indirect + github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/edsrzf/mmap-go v1.2.0 // indirect + github.com/elliotwutingfeng/asciiset v0.0.0-20260801111138-45c5fff54b41 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/erikvarga/go-rpmdb v0.0.0-20261001143808-1a11b79cbdd3 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-errors/errors v1.5.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.9.1 // indirect + github.com/go-git/go-git/v5 v5.19.2 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-restruct/restruct v1.2.0-alpha // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/google/go-containerregistry v0.21.8 // indirect + github.com/google/jsonschema-go v0.4.3 // indirect + github.com/google/osv-scalibr v0.5.4-0.20261009010329-5be78a00e3c2 // indirect + github.com/google/osv-scanner/v2 v2.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/css v1.0.1 // indirect + github.com/ianlancetaylor/demangle v0.0.0-20260724033716-83e58baca724 // indirect + github.com/icholy/digest v1.2.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jedib0t/go-pretty/v6 v6.8.3 // indirect + github.com/kevinburke/ssh_config v1.6.0 // indirect + github.com/klauspost/compress v1.20.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/lucasb-eyer/go-colorful v1.4.1 // indirect + github.com/lunixbochs/struc v0.0.0-20241101090106-8d528fa2c543 // indirect + github.com/masahiro331/go-ext4-filesystem v0.0.0-20260423010602-fe51f5b5e52b // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.27 // indirect + github.com/mattn/go-shellwords v1.0.14 // indirect + github.com/michaelkedar/xml v0.0.0-20250626233154-4ec9e090d1cd // indirect + github.com/microcosm-cc/bluemonday v1.0.27 // indirect + github.com/micromdm/plist v0.2.2 // indirect + github.com/moby/buildkit v0.32.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/moby/api v1.55.0 // indirect + github.com/moby/moby/client v0.5.1 // indirect + github.com/modelcontextprotocol/go-sdk v1.8.0 // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/ossf/osv-schema/bindings/go v0.0.0-20260902031056-b388a18021a3 // indirect + github.com/owenrumney/go-sarif/v3 v3.3.1 // indirect + github.com/package-url/packageurl-go v0.1.7 // indirect + github.com/pandatix/go-cvss v0.6.4 // indirect + github.com/pierrec/lz4/v4 v4.1.27 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/xattr v0.4.12 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rust-secure-code/go-rustaudit v0.0.0-20250226111315-e20ec32e963c // indirect + github.com/saferwall/pe v1.6.5 // indirect + github.com/sahilm/fuzzy v0.1.3 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/segmentio/asm v1.2.1 // indirect + github.com/segmentio/encoding v0.5.4 // indirect + github.com/sergi/go-diff v1.4.0 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + github.com/skeema/knownhosts v1.3.2 // indirect + github.com/spdx/gordf v0.0.0-20250128162952-000978ccd6fb // indirect + github.com/spdx/tools-golang v0.5.7 // indirect + github.com/thoas/go-funk v0.9.3 // indirect + github.com/tidwall/gjson v1.19.0 // indirect + github.com/tidwall/jsonc v0.3.3 // indirect + github.com/tidwall/match v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect + github.com/tink-crypto/tink-go/v2 v2.7.0 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect + github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0 // indirect + github.com/ulikunitz/xz v0.5.16 // indirect + github.com/urfave/cli/v3 v3.13.0 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/xhit/go-str2duration/v2 v2.1.0 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + github.com/yuin/goldmark v1.8.5 // indirect + github.com/yuin/goldmark-emoji v1.0.6 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.etcd.io/bbolt v1.5.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.28.0 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect + golang.org/x/crypto v0.58.0 // indirect + golang.org/x/mod v0.41.0 // indirect + golang.org/x/net v0.62.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/telemetry v0.0.0-20260924152758-ed294f943157 // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect + golang.org/x/time v0.15.0 // indirect + golang.org/x/tools v0.51.0 // indirect + golang.org/x/vuln v1.6.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/grpc v1.84.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.56.0 // indirect + osv.dev/bindings/go v0.0.0-20260805021707-3a57b89df3b6 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect + www.velocidex.com/golang/go-ntfs v0.2.1 // indirect + www.velocidex.com/golang/regparser v0.0.0-20250203141505-31e704a67ef7 // indirect +) diff --git a/dev/tools/osv/go.sum b/dev/tools/osv/go.sum new file mode 100644 index 00000000..38f40088 --- /dev/null +++ b/dev/tools/osv/go.sum @@ -0,0 +1,670 @@ +bitbucket.org/creachadair/stringset v0.0.14 h1:t1ejQyf8utS4GZV/4fM+1gvYucggZkfhb+tMobDxYOE= +bitbucket.org/creachadair/stringset v0.0.14/go.mod h1:Ej8fsr6rQvmeMDf6CCWMWGb14H9mz8kmDgPPTdiVT0w= +charm.land/glamour/v2 v2.0.1 h1:xl+r00A4aJWU0z8fgwKd9fQQ4rsphqGUzuEiXZP5n+c= +charm.land/glamour/v2 v2.0.1/go.mod h1:jo9z8XqVKPeEFMVdvCRLGk++RyJ3CdUwgNr7EvXLw3k= +charm.land/lipgloss/v2 v2.0.6 h1:EaGKeuA8FvF+v2BT5VmZd2LoYLaMZJXA5n34th8nCIQ= +charm.land/lipgloss/v2 v2.0.6/go.mod h1:ipDDJNSGa1hlwDtSfW1s2/xR8Vdhbut4PXh2zEKZd0Q= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +deps.dev/api/v3 v3.0.0-20260811035547-133c155ce203 h1:609W3fMlUT5DRVkVUFSpA3a8FvWp0NCAF5auaspsews= +deps.dev/api/v3 v3.0.0-20260811035547-133c155ce203/go.mod h1:n0QKltoCRUXfDqQ6xH3zIIID/QrZ86mglBZvcdqTrCo= +deps.dev/api/v3alpha v0.0.0-20260811035547-133c155ce203 h1:W5ZqGOPq1Yz7nrAXKznIsYjsgjG84jBG5seCaNtaSlI= +deps.dev/api/v3alpha v0.0.0-20260811035547-133c155ce203/go.mod h1:exryK+my03jfhf920mxleMs7s/dwUdVNaRml2Hrbf/w= +deps.dev/util/maven v0.0.0-20260928000106-0e158e8db197 h1:rtUqRyhX/4PBA14cYm13S6Tjp2xNMOhOM0eOSOSTAOc= +deps.dev/util/maven v0.0.0-20260928000106-0e158e8db197/go.mod h1:eGrXziwI7scSGrwIj+5EBHtTeSxAZD/yi8Hb3nFXesA= +deps.dev/util/pypi v0.0.0-20260727054525-2946ae4a6141 h1:WEYLTUP83k5nruc5a8FrgNyWP2s+ArFL308X4tVgX38= +deps.dev/util/pypi v0.0.0-20260727054525-2946ae4a6141/go.mod h1:qmA0z/Lsfa1FMtuLd9JmVZLMHR3GBX/EmbM6z1X3EDU= +deps.dev/util/resolve v0.0.0-20260727054525-2946ae4a6141 h1:d9WOtqC8tF8hfBBt14AfWACsr9/j0Zstgph6np5IBvI= +deps.dev/util/resolve v0.0.0-20260727054525-2946ae4a6141/go.mod h1:C3geprbQ5N3gqIJZl6bsPmG9p6xbRSJAjq07Qm9Fwnw= +deps.dev/util/semver v0.0.0-20260727054525-2946ae4a6141 h1:ZQY7wh3yBPpxAYfi5M4qq1a4ZJ5Z85KbJCL+awX7snA= +deps.dev/util/semver v0.0.0-20260727054525-2946ae4a6141/go.mod h1:jjJweVqtuMQ7Q4zlTQ/kCHpboojkRvpMYlhy/c93DVU= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/CycloneDX/cyclonedx-go v0.12.0 h1:/7Jum36UA6V043tQZ/fE3jf+Nf9gn/qxUFfd7QReMy8= +github.com/CycloneDX/cyclonedx-go v0.12.0/go.mod h1:V2577HhxDDCDLYfkm55WJrz16nHTfyQZwcWUBSG7Z28= +github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI= +github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 h1:0kQAzHq8vLs7Pptv+7TxjdETLf/nIqJpIB4oC6Ba4vY= +github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29/go.mod h1:ZWa7ssZJT30CCDGJ7fk/2SBTq9BIQrrVjrcss0UW2s0= +github.com/Microsoft/hcsshim v0.15.0-rc.1 h1:FbbwtQmiD+BVHynGkx5S65JkLyhkEiiTP8nrpmg2SZw= +github.com/Microsoft/hcsshim v0.15.0-rc.1/go.mod h1:HWvvUPIy9HF6LotILj1G4VyS065rcLQ6tqj6tMUdOfI= +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= +github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a h1:AeXPUzhU0yhID/v5JJEIkjaE85ASe+Vh4Kuv1RSLL+4= +github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a/go.mod h1:ukJBuruT9b24pdgZwWDvOaCYHeS03B7oQPCUWh25bwM= +github.com/Velocidex/ordereddict v0.0.0-20250821063524-02dc06e46238 h1:Q2DFf1f4fiz1/V4cu2wojDKF6/DfjIY7QmfVAr/UnJY= +github.com/Velocidex/ordereddict v0.0.0-20250821063524-02dc06e46238/go.mod h1:+MqO5UMBemyFSm+yRXslbpFTwPUDhFHUf7HPV92twg4= +github.com/Velocidex/yaml/v2 v2.2.8 h1:GUrSy4SBJ6RjGt43k6MeBKtw2z/27gh4A3hfFmFY3No= +github.com/Velocidex/yaml/v2 v2.2.8/go.mod h1:PlXIg/Pxmoja48C1vMHo7C5pauAZvLq/UEPOQ3DsjS4= +github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6 h1:5L8Mj9Co9sJVgW3TpYk2gxGJnDjsYuboNTcRmbtGKGs= +github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6/go.mod h1:3HgLJ9d18kXMLQlJvIY3+FszZYMxCz8WfE2MQ7hDY0w= +github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= +github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/alecthomas/assert v1.0.0 h1:3XmGh/PSuLzDbK3W2gUbRXwgW5lqPkuqvRgeQ30FI5o= +github.com/alecthomas/assert v1.0.0/go.mod h1:va/d2JC+M7F6s+80kl/R3G7FUiW6JzUO+hPhLyJ36ZY= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/chroma/v2 v2.19.0 h1:Im+SLRgT8maArxv81mULDWN8oKxkzboH07CHesxElq4= +github.com/alecthomas/chroma/v2 v2.19.0/go.mod h1:RVX6AvYm4VfYe/zsk7mjHueLDZor3aWCNE14TFlepBk= +github.com/alecthomas/colour v0.1.0 h1:nOE9rJm6dsZ66RGWYSFrXw461ZIt9A6+nHgL7FRrDUk= +github.com/alecthomas/colour v0.1.0/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= +github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/anchore/go-lzo v0.1.0 h1:NgAacnzqPeGH49Ky19QKLBZEuFRqtTG9cdaucc3Vncs= +github.com/anchore/go-lzo v0.1.0/go.mod h1:3kLx0bve2oN1iDwgM1U5zGku1Tfbdb0No5qp1eL1fIk= +github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9dedlWa0OhD30= +github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= +github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ12Gv5o= +github.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/ayoubfaouzi/pkcs7 v0.2.3 h1:XGCYHteXgclHnNlPdCF8aFyoUKwP9VhLQp+VX+hBZ3U= +github.com/ayoubfaouzi/pkcs7 v0.2.3/go.mod h1:u1EPWZOeIdVRo6C0/FVjB91Nsletw+8vZeAaAmeyJvQ= +github.com/bazelbuild/buildtools v0.0.0-20260716142318-04cf7de1434f h1:2mT6QcXmMFtvg7bezs1Fef7nnpJyeCmfWoWNjwtvNZ4= +github.com/bazelbuild/buildtools v0.0.0-20260716142318-04cf7de1434f/go.mod h1:PLNUetjLa77TCCziPsz0EI8a6CUxgC+1jgmWv0H25tg= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/canonical/chisel-manifest v1.2.0 h1:kFB9fBP33CvH+RLbTYLuKtFFICpq2JmH4xtYvBqpj3I= +github.com/canonical/chisel-manifest v1.2.0/go.mod h1:dWUCpU2uSJFbC+p3wEAjZ8uvQdjnHzenJQ1EszkKG4U= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc= +github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E= +github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= +github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= +github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= +github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= +github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 h1:ZR7e0ro+SZZiIZD7msJyA+NjkCNNavuiPBLgerbOziE= +github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834/go.mod h1:aKC/t2arECF6rNOnaKaVU6y4t4ZeHQzqfxedE/VkVhA= +github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 h1:rdnVWKgJpTVXKuKuJyxDJ+NFJdUaUqGvyGy61OcvlbA= +github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886/go.mod h1:nAw0d9PhFp1qdzi2xhQU5YOu5sVpDIHWlaW2Uz/bCro= +github.com/charmbracelet/x/ansi v0.11.8 h1:JMFwp0CgDC2+jcOB162HH5k7I3FVbgFSMMYg7dSPBQQ= +github.com/charmbracelet/x/ansi v0.11.8/go.mod h1:ZNN+3mXny/516oTQPLMPIBeSINvNJJQ8uQXDgbeJxY0= +github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= +github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= +github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f h1:pk6gmGpCE7F3FcjaOEKYriCvpmIN4+6OS/RD0vm4uIA= +github.com/charmbracelet/x/exp/golden v0.0.0-20250806222409-83e3a29d542f/go.mod h1:IfZAMTHB6XkZSeXUqriemErjAWCCzT0LwjKFYCZyw0I= +github.com/charmbracelet/x/exp/slice v0.0.0-20260803091719-3755ebad01b1 h1:mg80wPD5ZV0wDYdZAhHilmwGaTv8uEedVRnNiM0TehA= +github.com/charmbracelet/x/exp/slice v0.0.0-20260803091719-3755ebad01b1/go.mod h1:vqEfX6xzqW1pKKZUUiFOKg0OQ7bCh54Q2vR/tserrRA= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= +github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY= +github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= +github.com/charmbracelet/x/windows v0.2.2 h1:IofanmuvaxnKHuV04sC0eBy/smG6kIKrWG2/jYn2GuM= +github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k= +github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= +github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U= +github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY= +github.com/compose-spec/compose-go/v2 v2.14.0 h1:uaJeo5B3+OVlu+Rx2qLBcAdXPEUUzm5nQrRiGJafRAQ= +github.com/compose-spec/compose-go/v2 v2.14.0/go.mod h1:ZU6zlcweCZKyiB7BVfCizQT9XmkEIMFE+PRZydVcsZg= +github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6dyLYXQ= +github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw= +github.com/containerd/containerd/api v1.11.1 h1:h8nfoDW9+fNsC/9TwiAHj8B1GzXKtR4eFtkhi/X5RLU= +github.com/containerd/containerd/api v1.11.1/go.mod h1:CaQFRu+N1MtbgL6JDOJLUB1hCKESU1lD6MuTJhgtdlw= +github.com/containerd/containerd/v2 v2.3.3 h1:MUNBVVBTBpPll7KPh5GTvkC3cfG03PQLAHVdsUoue9k= +github.com/containerd/containerd/v2 v2.3.3/go.mod h1:rHKGm3VW6wNrINb3x8mNT+w7qYXFVElTt/8HTuxVhD4= +github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg= +github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= +github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v1.0.0-rc.4 h1:M42JrUT4zfZTqtkUwkr0GzmUWbfyO5VO0Q5b3op97T4= +github.com/containerd/platforms v1.0.0-rc.4/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A= +github.com/containerd/plugin v1.1.0 h1:O+7lczNJVMy8rz0YNx3xGB8tTf5qY4i5abF041Ew19U= +github.com/containerd/plugin v1.1.0/go.mod h1:qBTum+A8lJ6lO44A19Eo7y1OlcLj4OWFH1DA/vnHmcc= +github.com/containerd/ttrpc v1.2.9 h1:ha0ak962T0s3CA/RoZ6S6xiWZQF24GrBaEpiGX1uihg= +github.com/containerd/ttrpc v1.2.9/go.mod h1:jjtQRwXm4DL3KsHKW8vDiUOV6wO0hi6IPhmJhxU7aEs= +github.com/containerd/typeurl/v2 v2.3.0 h1:HZHPhRWo5XMy3QGQoPrUzbW/2ckwjfweHmOwlkIrPAQ= +github.com/containerd/typeurl/v2 v2.3.0/go.mod h1:Qk+PAdUYArVj41TnGi6rJ+48RF0PkcTc4i/taoBcK0w= +github.com/cyphar/filepath-securejoin v0.7.0 h1:s0Y3ITPy6sQn5xt54DuYvTF8hu134ooYLUb58DX/HjE= +github.com/cyphar/filepath-securejoin v0.7.0/go.mod h1:ymLGms/u3BYaviIiuKFnUx8EkQEZeK6cInNoAPJA3o4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deitch/magic v0.0.0-20240306090643-c67ab88f10cb h1:4W/2rQ3wzEimF5s+J6OY3ODiQtJZ5W1sForSgogVXkY= +github.com/deitch/magic v0.0.0-20240306090643-c67ab88f10cb/go.mod h1:B3tI9iGHi4imdLi4Asdha1Sc6feLMTfPLXh9IUYmysk= +github.com/diskfs/go-diskfs v1.7.0 h1:vonWmt5CMowXwUc79jWyGrf2DIMeoOjkLlMnQYGVOs8= +github.com/diskfs/go-diskfs v1.7.0/go.mod h1:LhQyXqOugWFRahYUSw47NyZJPezFzB9UELwhpszLP/k= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= +github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= +github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8= +github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/docker/cli v29.7.1+incompatible h1:ILZpP6B7fedIr6ANy824QkDp1WMJuouIq0O2SrBkB2w= +github.com/docker/cli v29.7.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker-credential-helpers v0.9.8 h1:bIREROb7So6PRlq6KTtdS9MPEjC29OQRkFNlvK2OX8Q= +github.com/docker/docker-credential-helpers v0.9.8/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c= +github.com/docker/go-connections v0.8.1 h1:JibmG5hULs5qXSr/cp/w3Pw5fZuStt4MOHMUExb29/M= +github.com/docker/go-connections v0.8.1/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dsoprea/go-exfat v0.0.0-20190906070738-5e932fbdb589 h1:LzrKhEeL5tqo8i86+5a8JgL5cEJBRspVm4FsKRK/gxA= +github.com/dsoprea/go-exfat v0.0.0-20190906070738-5e932fbdb589/go.mod h1:zs3tKt0dOHncKZ7QhimWwN9RP7f6W6CLdjRfscKvvcA= +github.com/dsoprea/go-logging v0.0.0-20190624164917-c4f10aab7696/go.mod h1:Nm/x2ZUNRW6Fe5C3LxdY1PyZY5wmDv/s5dkPJ/VB3iA= +github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd h1:l+vLbuxptsC6VQyQsfD7NnEC8BZuFpz45PgY+pH8YTg= +github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd/go.mod h1:7I+3Pe2o/YSU88W0hWlm9S22W7XI1JFNJ86U0zPKMf8= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= +github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84= +github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= +github.com/elliotwutingfeng/asciiset v0.0.0-20260801111138-45c5fff54b41 h1:k6IHOr6orJ0s8nQ0b8QmuO8RVj0K3WYruDcQfcdqm8g= +github.com/elliotwutingfeng/asciiset v0.0.0-20260801111138-45c5fff54b41/go.mod h1:GLo/8fDswSAniFG+BFIaiSPcK610jyzgEhWYPQwuQdw= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/erikvarga/go-rpmdb v0.0.0-20261001143808-1a11b79cbdd3 h1:rX5hmz+Oru8THw9NJp4pF2P/VPeosuF33nUXS3hD1Lw= +github.com/erikvarga/go-rpmdb v0.0.0-20261001143808-1a11b79cbdd3/go.mod h1:MiEorPk0IChAoCwpg2FXyqVgbNvOlPWZAYHqqIoDNoY= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/gkampitakis/ciinfo v0.3.4 h1:5eBSibVuSMbb/H6Elc0IIEFbkzCJi3lm94n0+U7Z0KY= +github.com/gkampitakis/ciinfo v0.3.4/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-snaps v0.5.23 h1:okh5QR48zpUjpWtu65AtqxdCY8huJq+dEDuUzd1PuKg= +github.com/gkampitakis/go-snaps v0.5.23/go.mod h1:uy3lVzCCRRsAwYqSocyw5fY8xRLCYEfqoOJNxr8HonM= +github.com/glebarez/go-sqlite v1.20.3 h1:89BkqGOXR9oRmG58ZrzgoY/Fhy5x0M+/WV48U5zVrZ4= +github.com/glebarez/go-sqlite v1.20.3/go.mod h1:u3N6D/wftiAzIOJtZl6BmedqxmmkDfH3q+ihjqxC9u0= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.0.2/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= +github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= +github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.9.1 h1:8U73XiOTfINdItHVa6z4Gv7ToObcZ6grkqQbLryLCdA= +github.com/go-git/go-billy/v5 v5.9.1/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= +github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-restruct/restruct v0.0.0-20190418070341-acd4e4c2cb35/go.mod h1:e2k/t2/850rC773ilFYQSoqyJ78SpTx7gtFtOY6/AYA= +github.com/go-restruct/restruct v1.2.0-alpha h1:2Lp474S/9660+SJjpVxoKuWX09JsXHSrdV7Nv3/gkvc= +github.com/go-restruct/restruct v1.2.0-alpha/go.mod h1:KqrpKpn4M8OLznErihXTGLlsXFGeLxHUrLRRI/1YjGk= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/gohugoio/hashstructure v0.6.0 h1:7wMB/2CfXoThFYhdWRGv3u3rUM761Cq29CxUW+NltUg= +github.com/gohugoio/hashstructure v0.6.0/go.mod h1:lapVLk9XidheHG1IQ4ZSbyYrXcaILU1ZEP/+vno5rBQ= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.21.8 h1:Ig/zIsnztdCUNaiNNczE+MoP5xcyUMfvpvfOr1xyMLE= +github.com/google/go-containerregistry v0.21.8/go.mod h1:dP5XNKcL7kMFF/TB3LfvWmVhAcv7iqkHb3oDK8aauTo= +github.com/google/go-cpy v0.0.0-20211218193943-a9c933c06932 h1:5/4TSDzpDnHQ8rKEEQBjRlYx77mHOvXu08oGchxej7o= +github.com/google/go-cpy v0.0.0-20211218193943-a9c933c06932/go.mod h1:cC6EdPbj/17GFCPDK39NRarlMI+kt+O60S12cNB5J9Y= +github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= +github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/osv-scalibr v0.5.4-0.20261009010329-5be78a00e3c2 h1:Wuc6ytG0LYkegdUfSqzg/D6JKfoK+kQnkshSTvluqFg= +github.com/google/osv-scalibr v0.5.4-0.20261009010329-5be78a00e3c2/go.mod h1:rihL22GqgUv42jUlgm2yWy/iSlAY0a2VVNQIAWuID2s= +github.com/google/osv-scanner/v2 v2.7.0 h1:7aOrJ9vB7qbzxC3R+QCXs03SHQiRuM80RJdqZhUEl2g= +github.com/google/osv-scanner/v2 v2.7.0/go.mod h1:0UM0H29x5nYHylj8TapmPkFOxIEck+eJFl/Ocv0tYdM= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/ianlancetaylor/demangle v0.0.0-20260724033716-83e58baca724 h1:QixF8Mcbe87ET7pK/fPbBJ9GXFddmEY8yYMepzMzo30= +github.com/ianlancetaylor/demangle v0.0.0-20260724033716-83e58baca724/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +github.com/icholy/digest v1.2.0 h1:oTbG4IsNOmidJ+421ehG7Ty93yt1yotq13kFMG569yw= +github.com/icholy/digest v1.2.0/go.mod h1:1P1+LzUv48ybX7bu8tVpZ2QWdd+xRuePNuGawHjwRUE= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jedib0t/go-pretty/v6 v6.8.3 h1:yVSk5aemoYHCvcrtqyXklwqcgHQIQzmy/oUzFlmffSQ= +github.com/jedib0t/go-pretty/v6 v6.8.3/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= +github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= +github.com/klauspost/compress v1.20.2 h1:v5tiZvm1KGLyg5d5oXdXe3lsSPrbEUiKoXGia//I7Qo= +github.com/klauspost/compress v1.20.2/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss= +github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lunixbochs/struc v0.0.0-20241101090106-8d528fa2c543 h1:GxMuVb9tJajC1QpbQwYNY1ZAo1EIE8I+UclBjOfjz/M= +github.com/lunixbochs/struc v0.0.0-20241101090106-8d528fa2c543/go.mod h1:vy1vK6wD6j7xX6O6hXe621WabdtNkou2h7uRtTfRMyg= +github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg= +github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/masahiro331/go-ext4-filesystem v0.0.0-20260423010602-fe51f5b5e52b h1:IAYuiQq6BVizPeXTKBoY3Jcin3gGn8No7h31lvH0aKo= +github.com/masahiro331/go-ext4-filesystem v0.0.0-20260423010602-fe51f5b5e52b/go.mod h1:3XMMY1M486mWGTD13WPItg6FsgflQR72ZMAkd+gsyoQ= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.27 h1:Feg/Oou5zI/wnpgDF6omIU0OokC9GxLC/WRknhVlIR0= +github.com/mattn/go-runewidth v0.0.27/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8= +github.com/mattn/go-shellwords v1.0.14 h1:yUKzIgsCnosndOASY6/enly1EAuaXeFSQ7cdyA3OuYg= +github.com/mattn/go-shellwords v1.0.14/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/michaelkedar/xml v0.0.0-20250626233154-4ec9e090d1cd h1:HONhnyx03XM5PTnlrL4dcz0QSJC/Qdi/YH2yJEmpqKc= +github.com/michaelkedar/xml v0.0.0-20250626233154-4ec9e090d1cd/go.mod h1:KUAB0Nhc2O/lzyPLuWF6Jm/HVC4GIRHWpxTWpy14WHM= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/micromdm/plist v0.2.2 h1:a5Yt/coion6hwVEW0da8a5P8IyAchXZ6eC+oBA0uJW8= +github.com/micromdm/plist v0.2.2/go.mod h1:flkfm0od6GzyXBqI28h5sgEyi3iPO28W2t1Zm9LpwWs= +github.com/moby/buildkit v0.32.2 h1:Sfy7+u6dUv/2yuBc9KCoK70Re8atuV8aPZ5UOC068Vc= +github.com/moby/buildkit v0.32.2/go.mod h1:0GB/EJ1d+4VIVqIAgy3asaoGkVXy7IrDfVy7mPhOvg8= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw= +github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0= +github.com/moby/sys/signal v0.7.1/go.mod h1:Se1VGehYokAkrSQwL4tDzHvETwUZlnY7S5XtQ50mQp8= +github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0= +github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/modelcontextprotocol/go-sdk v1.8.0 h1:KIvahhYqwtbeniWVPs3TcXEA7b8jEtwfBpOTAI+Urx4= +github.com/modelcontextprotocol/go-sdk v1.8.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= +github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/ossf/osv-schema/bindings/go v0.0.0-20260902031056-b388a18021a3 h1:mNqW4tPTIPcx3KL5qZCByAHiTwGcwD/eCEfKOEvLA4g= +github.com/ossf/osv-schema/bindings/go v0.0.0-20260902031056-b388a18021a3/go.mod h1:aW9XJyUJfg0r+HogMGMagTt4hF6ibBkH/FZenG/KiyE= +github.com/owenrumney/go-sarif/v3 v3.3.1 h1:bKY8YnWcY8zYTiGVObC5wZLlfEvMkCxK9XMGXBbzEtQ= +github.com/owenrumney/go-sarif/v3 v3.3.1/go.mod h1:72MaugkExDexbSauRuPq6BvUAAqAX0TwoNYMIQyZCMw= +github.com/package-url/packageurl-go v0.1.7 h1:iFWg6tzAjLA6F/qX3M5nZaiMHJgc+p2zxVyr/fY+sZY= +github.com/package-url/packageurl-go v0.1.7/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= +github.com/pandatix/go-cvss v0.6.4 h1:9w2RCO/Q4UTiJyEgpCHRiVc6CfrsFEnkoX+OtATqKio= +github.com/pandatix/go-cvss v0.6.4/go.mod h1:/ukvQnYlrKl3o/DVp7/GO2UZyZheuo/maOK0U1nBEhQ= +github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= +github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM= +github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/rust-secure-code/go-rustaudit v0.0.0-20250226111315-e20ec32e963c h1:8gOLsYwaY2JwlTMT4brS5/9XJdrdIbmk2obvQ748CC0= +github.com/rust-secure-code/go-rustaudit v0.0.0-20250226111315-e20ec32e963c/go.mod h1:kwM/7r/rVluTE8qJbHAffduuqmSv4knVQT2IajGvSiA= +github.com/saferwall/pe v1.6.5 h1:CxgDvikdp9mnLb2kHKVyyRHbK/JSMrtslCaveXxIQ8M= +github.com/saferwall/pe v1.6.5/go.mod h1:aNooU8V9vpqCUCiPb9WWIxEag15MeTjUoDtKQiAsZwY= +github.com/sahilm/fuzzy v0.1.3 h1:juByESSS32nVD81vr6tHmKmA/8zde7gE+x5CLxrzXPU= +github.com/sahilm/fuzzy v0.1.3/go.mod h1:au6//VbVSqu6DFrkL2CfjlJ5iURpNCPeE+1GwY3XsT8= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc= +github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= +github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/skeema/knownhosts v1.3.2 h1:EDL9mgf4NzwMXCTfaxSD/o/a5fxDw/xL9nkU28JjdBg= +github.com/skeema/knownhosts v1.3.2/go.mod h1:bEg3iQAuw+jyiw+484wwFJoKSLwcfd7fqRy+N0QTiow= +github.com/spdx/gordf v0.0.0-20250128162952-000978ccd6fb h1:7G2Czq97VORM5xNRrD8tSQdhoXPRs8s+Otlc7st9TS0= +github.com/spdx/gordf v0.0.0-20250128162952-000978ccd6fb/go.mod h1:uKWaldnbMnjsSAXRurWqqrdyZen1R7kxl8TkmWk2OyM= +github.com/spdx/tools-golang v0.5.7 h1:+sWcKGnhwp3vLdMqPcLdA6QK679vd86cK9hQWH3AwCg= +github.com/spdx/tools-golang v0.5.7/go.mod h1:jg7w0LOpoNAw6OxKEzCoqPC2GCTj45LyTlVmXubDsYw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/terminalstatic/go-xsd-validate v0.1.8 h1:UVrTCy1j3DhwaYTTUF+QYO/Nan13S0tf+Jwi+p45Bf0= +github.com/terminalstatic/go-xsd-validate v0.1.8/go.mod h1:1kb47fi2c6onlf+B7UrrQ9VYraOhcYwFm3iG+J6F4Zo= +github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= +github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= +github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= +github.com/tidwall/jsonc v0.3.3 h1:RVQqL3xFfDkKKXIDsrBiVQiEpBtxoKbmMXONb2H/y2w= +github.com/tidwall/jsonc v0.3.3/go.mod h1:dw+3CIxqHi+t8eFSpzzMlcVYxKp08UP5CD8/uSFCyJE= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tink-crypto/tink-go/v2 v2.7.0 h1:k7QnUXJ1cRDpvoy/5l1FimZqMAArRff8vjUqzi5N04o= +github.com/tink-crypto/tink-go/v2 v2.7.0/go.mod h1:cWNpQ/yAT/QHzAV0kBGMOSJzzYTKofDZdJaUqOPPWCI= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0 h1:2f304B10LaZdB8kkVEaoXvAMVan2tl9AiK4G0odjQtE= +github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0/go.mod h1:278M4p8WsNh3n4a1eqiFcV2FGk7wE5fwUpUom9mK9lE= +github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0= +github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= +github.com/urfave/cli/v3 v3.13.0 h1:Dr6jqMfIyyFsRVn7Nz5mqLsMY+ZMpfh3a0aMs+umPVY= +github.com/urfave/cli/v3 v3.13.0/go.mod h1:vXn6HxPNccJSzQr2QvwVncOKrgYGIHU0HY5h8B2nQj4= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= +github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.8.5 h1:r6N5afV5qj/5S4UTch8agZHJ8UxNCMwX7WjkkJam2NA= +github.com/yuin/goldmark v1.8.5/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs= +github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= +go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= +go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/exp v0.0.0-20260603202125-055de637280b h1:v1uXiEBHo8QA0LiGCo7UgHMzHT4Kdfpl2zmtH5vaP1Q= +golang.org/x/exp v0.0.0-20260603202125-055de637280b/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/telemetry v0.0.0-20260924152758-ed294f943157 h1:DftxASHKl6JolFPqoN+2WrFO53JvkHwRXvFqf8gsivA= +golang.org/x/telemetry v0.0.0-20260924152758-ed294f943157/go.mod h1:i+ivNqjDnTF3WTElsdk5g9V5DTSBYgdNo7xTU9SDwYA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= +golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= +golang.org/x/vuln v1.6.0 h1:FeMO9Rm/HwyduOztbvKcOw+zvDEPr4I4aQNSfevFcKY= +golang.org/x/vuln v1.6.0/go.mod h1:bWlG2493/sjR7ksvicBgMrznH3eYQEyK8ifUYBrqUbg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= +google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/dnaeon/go-vcr.v4 v4.0.7 h1:Mq/RF+mq3QwtEunJSsoTbYPt3elSAmdJhAxrEaqr88I= +gopkg.in/dnaeon/go-vcr.v4 v4.0.7/go.mod h1:cRwV/njsN/D8qNJu4NAXWswz6b4OUh3rMIu4SObbLBg= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= +modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +osv.dev/bindings/go v0.0.0-20260805021707-3a57b89df3b6 h1:1cVzYzGXOOvFVi5B6LOjPeZeqLA59IAowEfYZPwed2g= +osv.dev/bindings/go v0.0.0-20260805021707-3a57b89df3b6/go.mod h1:SeNseWd8ZzLFiMo4CFRy3jaxEcvm5YyQvsiCkjAvgbA= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= +www.velocidex.com/golang/go-ntfs v0.2.1 h1:9oSN0CpBZTmM75F4cEpUdAiOW55afj0ZALpvRt8cBZw= +www.velocidex.com/golang/go-ntfs v0.2.1/go.mod h1:4MSO8W9iNMXyBpjSpxApWfMjJUb9IWFD2Yis5JPZaSY= +www.velocidex.com/golang/regparser v0.0.0-20250203141505-31e704a67ef7 h1:BMX/37sYwX+8JhHt+YNbPfbx7dXG1w1L1mXonNBtjt0= +www.velocidex.com/golang/regparser v0.0.0-20250203141505-31e704a67ef7/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU= diff --git a/docs/architecture/adrs/0003-dependabot-security-pull-requests-only.md b/docs/architecture/adrs/0003-dependabot-security-pull-requests-only.md index df60c8ad..56ce4a07 100644 --- a/docs/architecture/adrs/0003-dependabot-security-pull-requests-only.md +++ b/docs/architecture/adrs/0003-dependabot-security-pull-requests-only.md @@ -2,7 +2,7 @@ * **Status:** Accepted * **Date:** 2026-11-08 -* **Last reviewed:** 2026-11-08 +* **Last reviewed:** 2026-11-09 * **Deciders:** [@nistapp] --- @@ -41,17 +41,18 @@ GitHub's documentation (the Dependabot options reference and "Configuring Depend 3. **One pull request per ecosystem.** A `security-fixes` group joins the minor and patch security fixes. 4. **No `target-branch`.** The default branch is `dev` ([ADR 0002](0002-dev-integrates-main-releases.md)), so every Dependabot pull request goes to `dev` without it. -5. **The dev tools are visible to Dependabot.** Their versions move to two manifests in `dev/tools/`, each with a +5. **The dev tools are visible to Dependabot.** Their versions move to three manifests in `dev/tools/`, each with a lockfile: | Manifest | Lockfile | Tools | Dependabot ecosystem | |---|---|---|---| | [`pyproject.toml`](../../../dev/tools/pyproject.toml) (group `tools`) | `uv.lock` | actionlint-py, check-jsonschema, coverage, detect-secrets, jsonschema, mypy, ruff, skills-ref, uv, zizmor | `uv` | | [`package.json`](../../../dev/tools/package.json) | `package-lock.json` | markdownlint-cli2, mermaid-cli | `npm` | + | [`osv/go.mod`](../../../dev/tools/osv/go.mod) (a `tool` directive) | `osv/go.sum` | osv-scanner ([ADR 0006](0006-security-verb-audits-dependencies-with-osv-scanner.md)) | `gomod` | Nothing installs these projects. The [`Makefile`](../../../Makefile), [`lefthook.yml`](../../../lefthook.yml), [`dev/mermaid_check.py`](../../../dev/mermaid_check.py) and the workflows (`setup-uv` `version-file`) read each - version from them, and still run the tools through `uvx` and `npx`. The root [`.nvmrc`](../../../.nvmrc) pins + version from them, and still run the tools through `uvx`, `npx` and `go tool`. The root [`.nvmrc`](../../../.nvmrc) pins Node for CI's `docs` job. [`test_tool_pins.py`](../../../dev/tests/test_tool_pins.py) fails when a pin is not exact, when a lockfile disagrees with its manifest, or when another file repeats a version. 6. **Data is not a dependency.** The greenfield template (`greenfield/templates/`) and the engine's test fixtures diff --git a/docs/architecture/adrs/0006-security-verb-audits-dependencies-with-osv-scanner.md b/docs/architecture/adrs/0006-security-verb-audits-dependencies-with-osv-scanner.md new file mode 100644 index 00000000..cb52418a --- /dev/null +++ b/docs/architecture/adrs/0006-security-verb-audits-dependencies-with-osv-scanner.md @@ -0,0 +1,91 @@ +# 0006. The Security Verb Audits the Dev Tools' Dependencies with osv-scanner + +* **Status:** Accepted +* **Date:** 2026-11-09 +* **Last reviewed:** 2026-11-09 +* **Deciders:** [@nistapp] + +--- + +## Context + +`make security` ran a secrets scan only. The repository's own audit reported `TOOL-04` PARTIAL: a security verb +exists, but it runs no dependency audit. The engine and the tools have no third-party dependencies. The dev tools +do: about 500 packages across `dev/tools/uv.lock`, `dev/tools/package-lock.json` and their own dependencies +([ADR 0003](0003-dependabot-security-pull-requests-only.md)). + +The engine's `TOOL-04` reads only the seven framework verbs. It reads the recipe text: it does not expand Make +variables or follow a prerequisite target. A separate `make deps-audit` target would leave `TOOL-04` at PARTIAL. + +The owner wants the most effective tool. Contributors may install a toolchain; users may not, and users never run +`make security`: they run the standard-library `audit.pyz` and the skill bundles. + +The first scan on 2026-11-09 found 8 advisories: uv (2), katex, re2js and 4 in Go modules. Five were fixed by +version updates. Three have no fix that reaches this repository and are accepted with a reason. + +## Decision + +1. **osv-scanner runs inside `make security`.** After the secrets scan, the + [`Makefile`](../../../Makefile) runs osv-scanner on every dev-tool lockfile: `dev/tools/uv.lock`, + `dev/tools/package-lock.json` and `dev/tools/osv/go.mod`. The tool name is written in the recipe, so `TOOL-04` + sees it. +2. **osv-scanner is a Go `tool` requirement.** [`dev/tools/osv/go.mod`](../../../dev/tools/osv/go.mod) pins a + tagged release; `go.sum` holds the checksums. `go -C dev/tools/osv tool osv-scanner` builds and runs it. The + version is written once, and Dependabot's `gomod` entry sees it, under the policy of ADR 0003. +3. **A known vulnerability fails the gate.** osv-scanner exits non-zero, so `make security` and `CI Gate` fail. +4. **Exceptions are reviewed and expire.** [`dev/tools/osv-scanner.toml`](../../../dev/tools/osv-scanner.toml) + lists each accepted vulnerability with an `id`, a `reason` and an `ignoreUntil` date at most six months ahead. + [`test_tool_pins.py`](../../../dev/tests/test_tool_pins.py) checks the format and the six-month limit. When the + date passes, the scan fails again. +5. **No offline switch.** `make security` needs the network (osv.dev). CI has it; a contributor without it cannot + run the gate. +6. **Contributors need Go 1.27 or newer.** `make setup` checks for it. CI installs it with `actions/setup-go` from + `go.mod`; the release job does so without a cache. + +--- + +## Alternatives Considered + +| Alternative | Why it was rejected | +|---|---| +| pip-audit plus `npm audit --package-lock-only` | Two tools and a `uv export` step. Two advisory sources instead of OSV, which aggregates GitHub advisories, PyPA and the Go database. osv-scanner reads `uv.lock` directly. | +| A separate `make deps-audit`; `security` stays offline | `TOOL-04` stays PARTIAL. Changing the engine to accept it would bend a check to suit this repository. | +| An `OFFLINE=1` switch that skips the audit | A skip switch on a gate. CI would never set it, but it invites use. | +| osv-scanner binary on `PATH`, version in a pin file | Dependabot cannot see a pin file. Every contributor installs it by hand, at any version. | +| `google/osv-scanner-action` in CI only | CI would run something other than `make security`. Local and CI results could differ. | +| Report only, exit 0 | `TOOL-04` would pass on a check that never blocks. | +| Scan only the Python and Node lockfiles | The Go modules of osv-scanner itself would raise Dependabot alerts that no gate checks. | + +> A later reversal of this decision is folded into this table, so the rejected direction +> keeps its rationale inside the accepted record. + +--- + +## Consequences + +### Positive + +* `TOOL-04` passes. A known vulnerability in any dev tool blocks a merge. +* One advisory source (OSV) and one tool for Python, Node and Go. The same tool fits target repositories of any stack. +* Each accepted vulnerability has a written reason and an end date. + +### Negative / Trade-offs + +* `make security` needs the network and Go. The first run builds osv-scanner (about 30 seconds); later runs use + the Go build cache. +* A new advisory turns every pull request red until someone fixes or accepts it. That is intended. +* `go.mod` and `go.sum` add about 700 lines. A raised indirect Go dependency differs from the set that upstream + osv-scanner tested. + +--- + +## Lifecycle — Living, Current-State ADRs + +* **This ADR MUST describe only the shipped, current design.** There is no `Deprecated`, `Superseded`, or `Rejected` status. +* **Decision changed?** Overwrite the body, keep the file and its number, and fold the reversal into + *Alternatives Considered*. Do NOT create a superseding ADR and do NOT leave a tombstone stub. +* **Decision no longer relevant?** Delete the file, retire its number forever, and update every inbound reference + plus the index table in `docs/architecture/adrs/README.md` in the same change set. +* Archived wording: `git log --follow docs/architecture/adrs/0006-security-verb-audits-dependencies-with-osv-scanner.md`. + +See [`docs/STYLE_GUIDE.md`](../../STYLE_GUIDE.md) § ADR Lifecycle for the canonical rules. diff --git a/docs/architecture/adrs/README.md b/docs/architecture/adrs/README.md index 447f32fc..83fd4b5f 100644 --- a/docs/architecture/adrs/README.md +++ b/docs/architecture/adrs/README.md @@ -13,6 +13,7 @@ The root router is [`docs/README.md`](../../README.md). | [0003](0003-dependabot-security-pull-requests-only.md) | Dependabot Opens Security Pull Requests Only; Major Versions Are Advisory | Accepted | 2026-11-08 | | [0004](0004-plain-language-rules-and-a-prose-check.md) | Plain-Language Rules in One File, Checked by a Standard-Library Prose Check | Accepted | 2026-11-08 | | [0005](0005-release-please-cuts-releases.md) | Release-Please Computes the Version, Writes the Changelog and Tags the Release | Accepted | 2026-11-08 | +| [0006](0006-security-verb-audits-dependencies-with-osv-scanner.md) | The Security Verb Audits the Dev Tools' Dependencies with osv-scanner | Accepted | 2026-11-09 | ## Requirements: `spec/decisions/` diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 883201d6..a3af939a 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -21,6 +21,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | zizmor | Workflow and Dependabot security: injection, broad permissions, unpinned actions, cooldown | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | | check-jsonschema | `.github/workflows/*.yml` and `.github/dependabot.yml` that do not match their SchemaStore schemas | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | | detect-secrets | Key-shaped strings not in `.secrets.baseline` | `make security` | `security` | `dev/tools/pyproject.toml` | +| osv-scanner | A known vulnerability in a dev tool or its dependencies (Python, Node, Go lockfiles), unless [`osv-scanner.toml`](../../dev/tools/osv-scanner.toml) accepts it ([Dependency audit](#dependency-audit)) | `make security` | `security` | `dev/tools/osv/go.mod` | | markdownlint-cli2 | Markdown rule violations (`.markdownlint-cli2.jsonc`) | `make docs-lint`; `make changelog-fix` rewrites `CHANGELOG.md` with the fixes | `docs`; `format-changelog` in `release-please.yml` | `dev/tools/package.json` | | `dev/prose_check.py` | Long sentences, vague words, explanation pages without a diagram ([`COMMUNICATION_RULES.md`](../COMMUNICATION_RULES.md)). Advisory: warnings and notices, never a failure | `make docs-prose [FILES=...]` | `docs` (annotations on changed Markdown) | This repository | | mermaid-cli | A Mermaid block that does not render | `make docs-mermaid FILES=` | `docs` | `dev/tools/package.json` | @@ -31,11 +32,30 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | release-please | Computes the next version from the commit subjects, writes `CHANGELOG.md`, tags and creates the GitHub release (`release-please-config.json`) | — | `release-please.yml`, on each push to `main` | `.github/workflows/release-please.yml` (action pinned by SHA) | | uv | Runs the Python tools through `uvx` and `uv run` | — | All jobs | `dev/tools/pyproject.toml` | | Node | Runs the Node tools through `npx` | — | `docs` | `.nvmrc` | +| Go | Builds and runs osv-scanner through `go tool` | — | `security` | `dev/tools/osv/go.mod` (`go` line) | `make check` runs `format:check`, `typecheck`, `test`, the engine's rule check, `crosscheck`, `recipe-lint` and `workflow-lint`, in sequence. CI's `CI Gate` passes only when the `check`, `security`, `coverage`, `self-audit`, `docs` and `commits` jobs pass ([Branch protection](branch-protection.md#the-required-checks)). +## Dependency audit + +`make security` runs osv-scanner after the secrets scan +([ADR 0006](../architecture/adrs/0006-security-verb-audits-dependencies-with-osv-scanner.md)). It reads the three +dev-tool lockfiles and asks osv.dev for known vulnerabilities. It needs Go 1.27 or newer and the network. The first +run builds osv-scanner, which takes about 30 seconds. + +The scan fails when it finds a vulnerability that [`dev/tools/osv-scanner.toml`](../../dev/tools/osv-scanner.toml) +does not accept. The output has one table row per vulnerability: the OSV link, the package, its version, the fixed +version and the lockfile. To fix a finding, do one of these: + +1. Raise the tool or the dependency to the fixed version. Follow + [Change a tool version](../../RELEASE_PROCESS.md#change-a-tool-version). +2. If no fix reaches this repository, add an entry to `osv-scanner.toml`. Give the `id` and a `reason`. + Set `ignoreUntil` to a date at most six months ahead. `dev/tests/test_tool_pins.py` checks the entry. + +When an `ignoreUntil` date passes, the scan fails again. Look at the advisory again, then fix or renew it. + ## Ratchets A ratchet holds a count that may only go down, or a floor that may only go up. A change that makes a count go up @@ -69,10 +89,11 @@ Each tool version is written once: - Python tools: `dev/tools/pyproject.toml`, group `tools`, locked in `dev/tools/uv.lock`. - Node tools: `dev/tools/package.json`, locked in `dev/tools/package-lock.json`. +- osv-scanner: `dev/tools/osv/go.mod`, a Go `tool` requirement, with checksums in `dev/tools/osv/go.sum`. The `Makefile`, `lefthook.yml`, `dev/mermaid_check.py` and the workflows read each version from those files. -Nothing is installed into the project: the tools run through `uvx` and `npx`. Dependabot watches these files and -opens a pull request only for a security fix that is not a major version +Nothing is installed into the project: the tools run through `uvx`, `npx` and `go tool`. Dependabot watches these +files and opens a pull request only for a security fix that is not a major version ([ADR 0003](../architecture/adrs/0003-dependabot-security-pull-requests-only.md)). `dev/tests/test_tool_pins.py` fails when a version is not exact, when a lockfile disagrees with its manifest, or when another file repeats a version. diff --git a/docs/reference/self-audit.md b/docs/reference/self-audit.md index 4226d8ad..0a551499 100644 --- a/docs/reference/self-audit.md +++ b/docs/reference/self-audit.md @@ -54,13 +54,13 @@ Every check that is not `PASS`, with the reason it is open. | `SEC-01` | PARTIAL | BLOCKER | `/.env` is root-anchored on purpose: the engine's fixtures hold `.env` files as data and must stay committable (comment in `.gitignore`). The check reads a root-only rule as incomplete | | `SEC-03` | PARTIAL | DEGRADER | Same cause as `SEC-01` for the dotenv shapes; the key, certificate and keystore shapes are covered | | `BASE-02` | PARTIAL | DEGRADER | The T5 playbook was applied (D32): `.quality-baseline.json` holds the counts (ruff 0, mypy 0, Markdown lint outside the ratchet 735) and `make format-check`, `make typecheck` and `make docs-lint` fail when one goes up. The check still reads PARTIAL by design: it credits only a numeric budget flag (`--max-warnings N`), and none of these tools has one. Engine issue [#45](https://github.com/Nistapp/codebase-agentic-readiness-framework/issues/45) tracks it; the coverage floor (91) is unchanged and `BASE-04` passes | -| `TOOL-04` | PARTIAL | DEGRADER | **Owner exception** (D33): the repository has no third-party dependencies, so there is nothing for a dependency audit to scan; `make security` is a secrets scan | +| `EXEC-01` | FAIL | BLOCKER | **Owner exception** (2026-11-09): a false positive. osv-scanner's Go module (`dev/tools/osv/go.mod`) pins Go with its `go` directive, but the check reads that directive only from a root `go.mod`. An engine fix is planned (v0.3.3 Track C) | | `HYG-02`, `HYG-06` | FAIL | COSMETIC | Need values only the owner has: `CODEOWNERS` names and a private disclosure contact for `SECURITY.md`. Stay open until supplied (D22) | | `NAV-05` | PARTIAL | COSMETIC | Informational, no work (D34): 44% of public symbols documented (638 of 1438) | | `IDX-02`, `IDX-03`, `TST-02`, `CON-04` | UNKNOWN | mixed | Need a probe, user-level configuration, or `--emit-baseline`; not run here | | `EXEC-02`, `NAV-01` | UNKNOWN | mixed | The repository declares no dependencies and no components, so there is nothing for the check to read | -Recorded decisions that keep a finding open or exempt: `IDX-01` and `TOOL-04` are owner exceptions (D21, D33); +Recorded decisions that keep a finding open or exempt: `IDX-01` and `EXEC-01` are owner exceptions (D21; 2026-11-09); `AGT-08` stays as it is for v0.2, because a stated harness list closes the gap even when a named harness does not read `AGENTS.md` natively, and is revisited in v0.3 (D31); `NAV-05` is informational and unscored (D34). The engine has no way to record an owner exception, so this page is the record. Everything else is either a deliberate design choice stated in the From 4ee9b51057b53fac6ad161ebeeacafc92b110e09 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 11:29:18 +0530 Subject: [PATCH 020/111] build(dev): add a typos spell check to make check `make spell` runs typos 1.51.1 (pinned in dev/tools/pyproject.toml) on every tracked file that .gitignore does not hide. It reads code identifiers as words too. _typos.toml excludes the data paths (fixtures, worked examples, CHANGELOG.md, lockfiles) and holds 11 project terms and code tokens, each with its reason. The repository starts at zero findings, so the check is a hard gate inside make check, with no ratchet. `make spell-fix` writes the suggested fixes, and a lefthook step checks the staged files. The README typo that prompted A3 ("recommneded") fails the check. Roadmap-Step: A3 Co-Authored-By: Claude Opus 5.5 --- .github/dependabot.yml | 2 +- AGENTS.md | 8 +++-- CONTRIBUTING.md | 5 +-- Makefile | 18 +++++++--- _typos.toml | 35 +++++++++++++++++++ dev/tests/test_tool_pins.py | 8 +++-- dev/tools/pyproject.toml | 1 + dev/tools/uv.lock | 19 ++++++++++ ...-dependabot-security-pull-requests-only.md | 2 +- docs/how-to/contribute-a-change.md | 4 +-- docs/reference/dev-tooling.md | 19 ++++++++-- lefthook.yml | 3 ++ 12 files changed, 107 insertions(+), 17 deletions(-) create mode 100644 _typos.toml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5b1aa04e..e0cc26c2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -30,7 +30,7 @@ updates: commit-message: prefix: "ci" - # The Python dev tools (ruff, mypy, detect-secrets, coverage, jsonschema, skills-ref, uv, actionlint-py, zizmor, + # The Python dev tools (ruff, mypy, detect-secrets, coverage, jsonschema, skills-ref, typos, uv, actionlint-py, zizmor, # check-jsonschema), locked in uv.lock. The # Makefile, lefthook.yml and the workflows read each version from dev/tools/pyproject.toml. - package-ecosystem: uv diff --git a/AGENTS.md b/AGENTS.md index d98d38ac..8f207d34 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ make lint # read-only lint make typecheck # mypy, strict, on audit/tools, implement/tools and dev, and the engine's build.py make test # the engine's suite, then the audit tools', the implement tools' and the dev checks' make check # THE gate: format:check, typecheck, test, the engine's --verify-rules, the schema - # cross-check, recipe-lint, workflow-lint (actionlint, zizmor, check-jsonschema) + # cross-check, recipe-lint, workflow-lint (actionlint, zizmor, check-jsonschema), spell (typos) make security # secrets scan, then the dependency audit (osv-scanner; needs Go and the network) make commit-lint # Conventional Commits subjects on this branch (RANGE=origin/dev..HEAD); CI runs it on each PR ``` @@ -47,6 +47,8 @@ This file is read by Claude Code, opencode, Codex, Gemini CLI, Cursor and any ot no vendor-specific instruction file exists beside it. - Run `make check` and `make security` before you propose a change. Do not run an ad-hoc substitute. +- A misspelling fails `make spell` (in `make check`). Fix it, or run `make spell-fix` and review the diff. Add a word + to `_typos.toml` only for a project term or a code token. - When you change Markdown, also run `make docs-lint` (needs Node; lints the ratchet path list in the `Makefile`) and validate every Mermaid block you add (`make docs-mermaid FILES=`). - Python 3.11 or newer (pinned in `.python-version`), `uv` for the pinned dev tools (ruff, mypy, detect-secrets), @@ -56,8 +58,8 @@ no vendor-specific instruction file exists beside it. write a version in the `Makefile`, a hook or a workflow. Nothing is installed into the project. - Local hooks are `lefthook.yml` (a stop on commits to `dev` and `main`, format check on staged Python, - `make docs-lint` on staged ratchet Markdown, `make workflow-lint` on staged `.github/` files, a secrets scan, - and `dev/commit_lint.py` on the commit message). + `make docs-lint` on staged ratchet Markdown, `make workflow-lint` on staged `.github/` files, typos on staged + files, a secrets scan, and `dev/commit_lint.py` on the commit message). The developer runs `lefthook install` once per clone; an agent never does. Hooks do not replace the gate: CI runs `make check`, `make security` and `make docs-lint`. - `make coverage` runs the engine's, the audit tools' and the implement tools' suites under coverage.py. It fails diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db578023..240e802f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,6 +45,7 @@ Run these before you propose a change. CI runs the same commands. | --- | --- | | `make format` | Rewrites formatting in place. Run it before the first `make check` | | `make check` | The gate: formatting and lint, `mypy` strict, the engine's, the tools' and the dev checks' tests, the engine's rule check, the schema cross-check, the recipe linter and the workflow lint. It takes a couple of minutes | +| `make spell` | Part of `make check`. typos on the tracked text; the word list is `_typos.toml`. `make spell-fix` writes the suggested fixes | | `make workflow-lint` | Part of `make check`. actionlint, zizmor and check-jsonschema on `.github/`: workflow mistakes, workflow security, and `dependabot.yml` against its schema. Offline | | `make commit-lint` | Checks that each commit subject on your branch follows Conventional Commits (`RANGE=origin/dev..HEAD` by default). CI runs the same check on every pull request | | `make security` | A secrets scan against `.secrets.baseline`, then a dependency audit of the dev tools (osv-scanner) | @@ -83,8 +84,8 @@ Dependabot watches them for security fixes. To change a version, follow ## Local hooks [`lefthook.yml`](lefthook.yml) stops a commit on `dev` or `main`. It also runs the format check on staged Python, -`make docs-lint` on staged Markdown, `make workflow-lint` on staged `.github/` files, a secrets scan and the -Conventional Commits check (`dev/commit_lint.py`) before a commit lands. Install it +`make docs-lint` on staged Markdown, `make workflow-lint` on staged `.github/` files, the spell check on staged +files, a secrets scan and the Conventional Commits check (`dev/commit_lint.py`) before a commit lands. Install it once per clone with `lefthook install` ([lefthook.dev](https://lefthook.dev)). Hooks do not replace the gate. ## Branches diff --git a/Makefile b/Makefile index 59c30dfb..efafed57 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ # make typecheck mypy, strict, on the typed scope below # make test the engine's unittest suite, then the recipe tools' (the catalogue checks), then the dev checks' # make check THE gate: format:check -> typecheck -> test, plus the engine's rule check, the schema -# cross-check, the recipe linter and the workflow lint +# cross-check, the recipe linter, the workflow lint and the spell check # make security secrets scan against the committed baseline, then the dependency audit (osv-scanner) of # the dev tools' lockfiles; needs Go and the network # make coverage (extra) both unittest suites under coverage.py; fails below the floor in .coveragerc @@ -53,6 +53,9 @@ COVERAGE := uvx coverage@$(call pin,coverage) # and Dependabot security, offline) and check-jsonschema (the SchemaStore schemas it bundles, offline). ACTIONLINT := uvx --from actionlint-py==$(call pin,actionlint-py) actionlint ZIZMOR := uvx zizmor@$(call pin,zizmor) +# The spell check: typos reads _typos.toml (the word list and the excluded data paths) from the repository root, +# and walks the tracked files that .gitignore does not hide. +TYPOS := uvx typos@$(call pin,typos) CHECK_JSONSCHEMA := uvx --from check-jsonschema==$(call pin,check-jsonschema) check-jsonschema # The coverage data stays under the git-ignored build output; settings (the floor) are in .coveragerc. COVERAGE_DATA := $(CURDIR)/dev/dist/coverage/.coverage @@ -81,11 +84,11 @@ TYPED_SOURCES := $(shell find audit/tools implement/tools dev -name '*.py' -not .DEFAULT_GOAL := help .NOTPARALLEL: -.PHONY: help setup format format-check format\:check lint typecheck test coverage verify-rules crosscheck recipe-lint check security update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint +.PHONY: help setup format format-check format\:check lint typecheck test coverage verify-rules crosscheck recipe-lint check security update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint spell spell-fix help: @echo "verbs: format format:check (format-check) lint typecheck test check security" - @echo "extra: setup (checks prerequisites, installs nothing) coverage self-audit update-audit-baseline verify-rules crosscheck recipe-lint workflow-lint commit-lint update-secrets-baseline validate-skills" + @echo "extra: setup (checks prerequisites, installs nothing) coverage self-audit update-audit-baseline verify-rules crosscheck recipe-lint workflow-lint commit-lint spell spell-fix update-secrets-baseline validate-skills" @echo "docs: docs-lint docs-lint-all docs-prose [FILES=...] docs-mermaid FILES=... changelog-fix" # There is nothing to install into the project: the runtime code is standard-library Python and the dev tools run @@ -163,6 +166,13 @@ recipe-lint: $(PYTHON) dev/recipe_lint.py . # Lints the GitHub Actions workflows and checks .github/dependabot.yml. All three tools run offline. +# Read-only: report each misspelling with its file and line. `make spell-fix` writes the suggested fixes; review them. +spell: + $(TYPOS) + +spell-fix: + $(TYPOS) --write-changes + workflow-lint: $(ACTIONLINT) $(ZIZMOR) --offline .github @@ -175,7 +185,7 @@ RANGE ?= origin/dev..HEAD commit-lint: $(PYTHON) dev/commit_lint.py --range $(RANGE) -check: format-check typecheck test verify-rules crosscheck recipe-lint workflow-lint +check: format-check typecheck test verify-rules crosscheck recipe-lint workflow-lint spell # No third-party runtime dependencies exist to audit, so the gate is a secrets scan. The baseline # lists the deliberate key-shaped strings in the fixtures and tests; a new hit fails the scan. diff --git a/_typos.toml b/_typos.toml new file mode 100644 index 00000000..beb7c8e9 --- /dev/null +++ b/_typos.toml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna +# +# The spell check (typos; `make spell`, part of `make check`). typos reads this file from the repository root. +# Add a word here only when it is a project term or a code token, never to hide a real typo. +# How to read the output: docs/reference/dev-tooling.md, "Spell check". + +[files] +# Data, not prose: the engine's fixtures and the worked examples hold deliberate mistakes and third-party text. +# CHANGELOG.md is written by release-please from commit subjects; fix a typo in the commit, not in the file. +extend-exclude = [ + "**/fixtures/**", + "**/testdata/**", + "audit/schema/examples/**", + "CHANGELOG.md", + "*.lock", + "package-lock.json", + "go.sum", + ".secrets.baseline", + ".audit-baseline.json", +] + +[default.extend-words] +# Each entry: the token, then why it is correct here. +mis = "mis" # "mis-cased", the engine's term for AGENTS.md spelled agents.md +unparseable = "unparseable" # a valid spelling, used in the check docs +xdescribe = "xdescribe" # the Jasmine and Mocha skip API that TST checks look for +behaviou = "behaviou" # the regex stem behaviou?r +ignor = "ignor" # a regex stem in dev/eval +execut = "execut" # a regex stem in dev/eval +alls = "alls" # "catch-alls" +Jod = "Jod" # the Node 22 release line's codename +caf = "caf" # "café", a test path with a Unicode escape +directorys = "directorys" # a possessive in test names: test_the_directorys_own_... +repositorys = "repositorys" # a possessive in test names: test_..._this_repositorys_head_... diff --git a/dev/tests/test_tool_pins.py b/dev/tests/test_tool_pins.py index d5b1aa7f..74718869 100644 --- a/dev/tests/test_tool_pins.py +++ b/dev/tests/test_tool_pins.py @@ -72,7 +72,7 @@ def node_pins() -> dict[str, str]: class ManifestsHoldExactPins(unittest.TestCase): def test_every_python_pin_is_exact(self) -> None: pins = python_pins() - for tool in ("ruff", "mypy", "detect-secrets", "coverage", "jsonschema", "skills-ref", "uv"): + for tool in ("ruff", "mypy", "detect-secrets", "coverage", "jsonschema", "skills-ref", "typos", "uv"): self.assertIn(tool, pins) def test_every_node_pin_is_exact(self) -> None: @@ -194,13 +194,17 @@ def dry_run(self, *targets: str) -> str: def test_tool_commands_carry_the_manifest_versions(self) -> None: python = python_pins() node = node_pins() - output = self.dry_run("lint", "typecheck", "security", "docs-lint", "coverage") + output = self.dry_run("lint", "typecheck", "security", "docs-lint", "coverage", "spell") self.assertIn(f"ruff@{python['ruff']}", output) self.assertIn(f"mypy@{python['mypy']}", output) self.assertIn(f"detect-secrets=={python['detect-secrets']}", output) self.assertIn(f"coverage@{python['coverage']}", output) + self.assertIn(f"typos@{python['typos']}", output) self.assertIn(f"markdownlint-cli2@{node['markdownlint-cli2']}", output) + def test_check_runs_the_spell_check(self) -> None: + self.assertIn(f"typos@{python_pins()['typos']}", self.dry_run("check")) + def test_security_audits_every_dev_tool_lockfile(self) -> None: output = self.dry_run("security") self.assertIn("go -C dev/tools/osv tool osv-scanner scan source", output) diff --git a/dev/tools/pyproject.toml b/dev/tools/pyproject.toml index f15377c0..3a49abc3 100644 --- a/dev/tools/pyproject.toml +++ b/dev/tools/pyproject.toml @@ -22,6 +22,7 @@ tools = [ "mypy==2.4.0", "ruff==0.16.10", "skills-ref==0.1.1", + "typos==1.51.1", "uv==0.13.5", "zizmor==1.32.0", ] diff --git a/dev/tools/uv.lock b/dev/tools/uv.lock index fb0c86fc..b7ddb81d 100644 --- a/dev/tools/uv.lock +++ b/dev/tools/uv.lock @@ -281,6 +281,7 @@ tools = [ { name = "mypy" }, { name = "ruff" }, { name = "skills-ref" }, + { name = "typos" }, { name = "uv" }, { name = "zizmor" }, ] @@ -297,6 +298,7 @@ tools = [ { name = "mypy", specifier = "==2.4.0" }, { name = "ruff", specifier = "==0.16.10" }, { name = "skills-ref", specifier = "==0.1.1" }, + { name = "typos", specifier = "==1.51.1" }, { name = "uv", specifier = "==0.13.5" }, { name = "zizmor", specifier = "==1.32.0" }, ] @@ -1092,6 +1094,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] +[[package]] +name = "typos" +version = "1.51.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/0f/cb4959257f1c9837f47826f0d03f72d8d4e7717fee99ebb76a08f2e9a341/typos-1.51.1.tar.gz", hash = "sha256:b0ed99607ea72722d5fe9696645ecfc2419a646e149a1cc9d301bec693ee990e", size = 1854225, upload-time = "2026-10-06T21:07:40.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/68/0292ee8ba639576c2f9fa5a05a1e4ddbb7e3f863ea21da8650df141ad05e/typos-1.51.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:44a6d695ae10df0ce1a0827defc07b5962291af143b926a3e058f6b4a4832d45", size = 3450563, upload-time = "2026-10-06T21:07:22.15Z" }, + { url = "https://files.pythonhosted.org/packages/ec/24/3278c7e782aaa2b445ecb331c4f7791ac6261ace75a858cf9d22f0b1fea8/typos-1.51.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a0788996833c14666111a2c62f19847aa21705a6cb03e547540f11996c147632", size = 3355698, upload-time = "2026-10-06T21:07:24.532Z" }, + { url = "https://files.pythonhosted.org/packages/2d/eb/f8ed4dc1a68dda06086478818ca8125c4b097939b7f57ef08c6854fba9e6/typos-1.51.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0473ae19ccba8e3fff5e2573914b575782568288fd1fae923425f143daca69", size = 8352343, upload-time = "2026-10-06T21:07:26.609Z" }, + { url = "https://files.pythonhosted.org/packages/e0/aa/d620541eee3e942537d33c0d9d0fe9dfb3affda5f44ead383e000ac6b172/typos-1.51.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bb78f99c0fe92825cf580620aa157576e947c181a0b3e6ae211e24149882cb15", size = 7429661, upload-time = "2026-10-06T21:07:28.799Z" }, + { url = "https://files.pythonhosted.org/packages/1e/e1/e359b459ebc2989a4fc630ee4faadc4479b4390b4487dbc86f78dc3a1e52/typos-1.51.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fc63fbfd84bede3db35a5699c80b0a65846ac7768b9bc2e4f8e858f1d6b0da1", size = 7862794, upload-time = "2026-10-06T21:07:30.668Z" }, + { url = "https://files.pythonhosted.org/packages/56/80/0e3846741e9983ad847a384f596ee794607efc05d652eb997dfaef75c2a3/typos-1.51.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0a9dec0c10626606d945ad02f03ec35f31f847584cc63001cccfe41b94097a7d", size = 7195766, upload-time = "2026-10-06T21:07:32.662Z" }, + { url = "https://files.pythonhosted.org/packages/4f/33/61c9b7865ee1d782d1e98488176f83af94f1404aa0be4622f211d5a2252c/typos-1.51.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:05441c0054e6f2ff0720a92b56b39c9ff9460ea30a9baa1d30603b2b709bd59c", size = 8247232, upload-time = "2026-10-06T21:07:34.972Z" }, + { url = "https://files.pythonhosted.org/packages/43/dd/96915ab21d88c645b5e181a66d5b70950102413a9d187548f8c52b452ad2/typos-1.51.1-py3-none-win32.whl", hash = "sha256:e3600f808fdedbeef6a705abe3a31589103842635eca2130cf571c55b9ecfc61", size = 3174015, upload-time = "2026-10-06T21:07:37.088Z" }, + { url = "https://files.pythonhosted.org/packages/bb/b0/e1f67f5688324eb9d5d3c38ffedd72ae36b6a79c52bf6eeec6e1fbfd9706/typos-1.51.1-py3-none-win_amd64.whl", hash = "sha256:88b68c8f7c15b6c04f2f1a558e5fae2aa0d5e3ef46aec1a464a93498d46d99de", size = 3355358, upload-time = "2026-10-06T21:07:38.83Z" }, +] + [[package]] name = "tzdata" version = "2026.6" diff --git a/docs/architecture/adrs/0003-dependabot-security-pull-requests-only.md b/docs/architecture/adrs/0003-dependabot-security-pull-requests-only.md index 56ce4a07..ab441d0b 100644 --- a/docs/architecture/adrs/0003-dependabot-security-pull-requests-only.md +++ b/docs/architecture/adrs/0003-dependabot-security-pull-requests-only.md @@ -46,7 +46,7 @@ GitHub's documentation (the Dependabot options reference and "Configuring Depend | Manifest | Lockfile | Tools | Dependabot ecosystem | |---|---|---|---| - | [`pyproject.toml`](../../../dev/tools/pyproject.toml) (group `tools`) | `uv.lock` | actionlint-py, check-jsonschema, coverage, detect-secrets, jsonschema, mypy, ruff, skills-ref, uv, zizmor | `uv` | + | [`pyproject.toml`](../../../dev/tools/pyproject.toml) (group `tools`) | `uv.lock` | actionlint-py, check-jsonschema, coverage, detect-secrets, jsonschema, mypy, ruff, skills-ref, typos, uv, zizmor | `uv` | | [`package.json`](../../../dev/tools/package.json) | `package-lock.json` | markdownlint-cli2, mermaid-cli | `npm` | | [`osv/go.mod`](../../../dev/tools/osv/go.mod) (a `tool` directive) | `osv/go.sum` | osv-scanner ([ADR 0006](0006-security-verb-audits-dependencies-with-osv-scanner.md)) | `gomod` | diff --git a/docs/how-to/contribute-a-change.md b/docs/how-to/contribute-a-change.md index 9100ba3e..fb33d656 100644 --- a/docs/how-to/contribute-a-change.md +++ b/docs/how-to/contribute-a-change.md @@ -70,8 +70,8 @@ Do these steps once, on each new machine. git switch dev ``` -4. Optional: install the local hooks. They stop a commit on `dev` or `main`. They also run four checks: the - format check, the Markdown lint, the secrets scan and the commit message check: +4. Optional: install the local hooks. They stop a commit on `dev` or `main`. They also run these checks: the + format check, the Markdown lint, the workflow lint, the spell check, the secrets scan and the commit message check: ```bash lefthook install diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index a3af939a..367e95c6 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -19,6 +19,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | `dev/recipe_lint.py` | Broken links, bad `SKILL.md` frontmatter, oversize files, unknown tools, check IDs and requirement IDs | `make recipe-lint` | `check` | This repository | | actionlint | Workflow mistakes: syntax, expressions, action inputs, shellcheck on `run:` blocks | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` (`actionlint-py`) | | zizmor | Workflow and Dependabot security: injection, broad permissions, unpinned actions, cooldown | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | +| typos | Misspelled words in the tracked text, code identifiers included (word list: [`_typos.toml`](../../_typos.toml)) ([Spell check](#spell-check)) | `make spell`; `make spell-fix` writes the fixes | `check` | `dev/tools/pyproject.toml` | | check-jsonschema | `.github/workflows/*.yml` and `.github/dependabot.yml` that do not match their SchemaStore schemas | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | | detect-secrets | Key-shaped strings not in `.secrets.baseline` | `make security` | `security` | `dev/tools/pyproject.toml` | | osv-scanner | A known vulnerability in a dev tool or its dependencies (Python, Node, Go lockfiles), unless [`osv-scanner.toml`](../../dev/tools/osv-scanner.toml) accepts it ([Dependency audit](#dependency-audit)) | `make security` | `security` | `dev/tools/osv/go.mod` | @@ -34,10 +35,23 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | Node | Runs the Node tools through `npx` | — | `docs` | `.nvmrc` | | Go | Builds and runs osv-scanner through `go tool` | — | `security` | `dev/tools/osv/go.mod` (`go` line) | -`make check` runs `format:check`, `typecheck`, `test`, the engine's rule check, `crosscheck`, `recipe-lint` and -`workflow-lint`, in sequence. CI's `CI Gate` passes only when the `check`, `security`, `coverage`, `self-audit`, +`make check` runs `format:check`, `typecheck`, `test`, the engine's rule check, `crosscheck`, `recipe-lint`, +`workflow-lint` and `spell`, in sequence. CI's `CI Gate` passes only when the `check`, `security`, `coverage`, `self-audit`, `docs` and `commits` jobs pass ([Branch protection](branch-protection.md#the-required-checks)). +## Spell check + +`make spell` runs typos on every tracked file that `.gitignore` does not hide. It skips the data paths that +[`_typos.toml`](../../_typos.toml) excludes: the fixtures, the worked examples, `CHANGELOG.md` and the lockfiles. +typos splits code identifiers into words, so a typo inside a variable name fails too. Each finding shows the file, +the line and the suggested word. + +To fix a finding, do one of these: + +1. Correct the word. `make spell-fix` writes every suggested fix; review the diff before you commit. +2. If the word is correct here (a project term, a regex stem or a code token), add it to `[default.extend-words]` in + `_typos.toml`. Add a comment that says why. + ## Dependency audit `make security` runs osv-scanner after the secrets scan @@ -78,6 +92,7 @@ fails its gate. When a count goes down, lower the baseline in the same change. | `format-check` | `make format-check` | Staged Python files | | `docs-lint` | `make docs-lint` | Staged Markdown in the ratchet paths | | `workflow-lint` | `make workflow-lint` | Staged files under `.github/` | +| `spell` | typos, at the pinned version | Staged files, except the paths `_typos.toml` excludes | | `secrets` | detect-secrets on the staged files | Every commit | | `conventional-commits` | `dev/commit_lint.py --message-file` | The commit message | diff --git a/lefthook.yml b/lefthook.yml index 8da6a1a9..4d789fc3 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -23,6 +23,9 @@ pre-commit: workflow-lint: glob: ".github/**" run: make workflow-lint + spell: + # typos on the staged files; _typos.toml holds the word list and the excluded data paths. + run: uvx "typos@$(sed -n 's/^ *"typos==\([^"]*\)".*/\1/p' dev/tools/pyproject.toml)" --force-exclude {staged_files} secrets: # The version is the one pinned in dev/tools/pyproject.toml, where Dependabot sees it. run: uvx --from "$(grep -o 'detect-secrets==[^"]*' dev/tools/pyproject.toml)" detect-secrets-hook --baseline .secrets.baseline {staged_files} # pragma: allowlist secret From 7c7cfa78d27f34ede3b6eb799fd5104a7041ce4b Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 11:51:04 +0530 Subject: [PATCH 021/111] ci: sign build provenance for the release files release.yml now signs one SLSA build provenance statement for the three files in SHA256SUMS (actions/attest v4.2.2, public Sigstore) and attaches the signed bundle to the release as vX.Y.Z.sigstore.json. The provenance records the ref and commit the run started from, not the checkout. So release-please.yml now starts release.yml with --ref vX.Y.Z instead of main, and release.yml refuses a run that did not start from the tag it builds. Permissions move from the workflow to the job, with id-token, attestations and artifact-metadata added. Users keep sha256sum as the default check. Install the skills gains an optional Step 2b: gh attestation verify with --source-ref and --bundle, which works without a GitHub login (checked on gh 2.102.0; a tampered file fails). The release checklist adds the verify step. Roadmap-Step: A5 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release-please.yml | 3 +- .github/workflows/release.yml | 44 +++++++++++++++--- RELEASE_PROCESS.md | 20 +++++--- SECURITY.md | 4 +- dev/tests/test_release_provenance.py | 69 ++++++++++++++++++++++++++++ docs/how-to/install-the-skills.md | 20 ++++++++ docs/reference/dev-tooling.md | 1 + 7 files changed, 145 insertions(+), 16 deletions(-) create mode 100644 dev/tests/test_release_provenance.py diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 4dcc72d3..f2ddfee4 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -122,7 +122,8 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ needs.release-please.outputs.tag_name }} run: | - gh workflow run release.yml --repo "$GITHUB_REPOSITORY" --ref main -f "tag=$TAG" + # --ref is the tag, not main: the build provenance that release.yml signs names this ref and its commit. + gh workflow run release.yml --repo "$GITHUB_REPOSITORY" --ref "$TAG" -f "tag=$TAG" echo "Started release.yml for $TAG. Watch it: gh run list --workflow Release --limit 1" back-merge: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 293b9689..bce0c5f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,12 +3,17 @@ # # Builds the release files for a tag `vX.Y.Z` on `main` and attaches them to its GitHub release: the single-file # engine (audit.pyz), the agentic-readiness-audit and agentic-readiness-implement skill bundles as zips, and their -# SHA-256 sums. Runs the same gates as CI first. +# SHA-256 sums. Runs the same gates as CI first. Then it signs build provenance for the three files (actions/attest, +# public Sigstore) and attaches the signed bundle, so a user can check that a file came from this workflow and tag: +# `gh attestation verify --repo --source-ref refs/tags/ --bundle .sigstore.json`. # -# release-please.yml starts this workflow (workflow_dispatch) after release-please has made the tag and the release. A +# release-please.yml starts this workflow (workflow_dispatch, from the tag: `--ref vX.Y.Z`) after release-please has +# made the tag and the release. The run must start from the tag it builds, because the provenance records the run's +# ref and commit, not the checkout. A # tag pushed by the default GITHUB_TOKEN starts no workflow, so this workflow has no `push: tags` trigger. The # maintainer can also start it by hand for an existing tag; then it creates the release when there is none. No secret -# beyond GITHUB_TOKEN; `contents: write` is the only permission. Actions are pinned by commit SHA. A session of an AI +# beyond GITHUB_TOKEN. The job's permissions are `contents: write` (the release) and, for the provenance, +# `id-token: write`, `attestations: write` and `artifact-metadata: write`. Actions are pinned by commit SHA. A session of an AI # agent never triggers this workflow. How releases work: RELEASE_PROCESS.md. name: Release @@ -20,8 +25,7 @@ on: required: true type: string -permissions: - contents: write +permissions: {} concurrency: group: release-${{ inputs.tag }} @@ -31,6 +35,11 @@ jobs: release: name: release ${{ inputs.tag }} runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + attestations: write + artifact-metadata: write steps: - name: Require a tag of the form vX.Y.Z env: @@ -40,6 +49,17 @@ jobs: echo "::error::'$TAG' is not a release tag of the form vX.Y.Z." exit 1 fi + # The provenance names the ref the run started from. Started from main or a branch, it would name a commit + # that may not be the tagged one. + - name: Require the run to start from the tag + env: + TAG: ${{ inputs.tag }} + RUN_REF: ${{ github.ref }} + run: | + if [[ "$RUN_REF" != "refs/tags/$TAG" ]]; then + echo "::error::This run started from $RUN_REF. Start it from the tag: gh workflow run release.yml --ref $TAG -f tag=$TAG" + exit 1 + fi - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -100,6 +120,18 @@ jobs: (cd "$out" && sha256sum audit.pyz "agentic-readiness-audit-$TAG.zip" "agentic-readiness-implement-$TAG.zip" > SHA256SUMS) cat "$out/SHA256SUMS" echo "RELEASE_DIR=$out" >> "$GITHUB_ENV" + # One signed SLSA build provenance statement for the three files in SHA256SUMS, stored in the repository's + # attestations. The bundle is attached to the release too, so a user can verify without a GitHub login. + - name: Sign build provenance + id: attest + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-checksums: ${{ env.RELEASE_DIR }}/SHA256SUMS + - name: Add the provenance bundle to the release files + env: + TAG: ${{ inputs.tag }} + BUNDLE: ${{ steps.attest.outputs.bundle-path }} + run: cp "$BUNDLE" "$RELEASE_DIR/$TAG.sigstore.json" # release-please has normally made the release already, with the changelog entry as its notes. Then only the # files are attached (--clobber makes a re-run safe). For a hand-started run on a tag with no release, the # release is created here. @@ -112,5 +144,5 @@ jobs: gh release upload "$TAG" "$RELEASE_DIR"/* --repo "$GITHUB_REPOSITORY" --clobber else gh release create "$TAG" "$RELEASE_DIR"/* --repo "$GITHUB_REPOSITORY" --title "$TAG" --verify-tag \ - --notes "See CHANGELOG.md. Verify a download with: sha256sum -c SHA256SUMS" + --notes "See CHANGELOG.md. Verify a download with: sha256sum -c SHA256SUMS (provenance: docs/how-to/install-the-skills.md)" fi diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 812d9093..1e0b068c 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -257,23 +257,27 @@ After the next promotion, the release pull request uses that version. | --- | --- | --- | | `release-please` | Every push to `main` | Opens or updates the release pull request; after its merge, tags `vX.Y.Z` and creates the GitHub release | | `format-changelog` | The release pull request was opened or updated | Runs `make changelog-fix` (markdownlint `--fix`) on `CHANGELOG.md` and pushes the fix to the release pull request. release-please writes `*` bullets and double blank lines, which the `docs-lint` ratchet refuses | -| `publish` | A release was created | Starts `release.yml` with the tag (a dispatch is the one event that `GITHUB_TOKEN` may start a workflow with) | +| `publish` | A release was created | Starts `release.yml` from the tag, with `--ref vX.Y.Z` (a dispatch is the one event that `GITHUB_TOKEN` may start a workflow with) | | `back-merge` | A release was created | Creates `chore/back-merge-vX.Y.Z` at the release commit and opens its pull request into `dev` | [`release.yml`](.github/workflows/release.yml) runs on a dispatch with a tag. It does these steps in sequence: -1. It makes sure that the tag has the form `vX.Y.Z` and that its commit is on `main`. +1. It makes sure that the tag has the form `vX.Y.Z`, that the run started from that tag, and that its commit is on + `main`. The run must start from the tag because the build provenance records the run's ref and commit. 2. It makes sure that the engine's `__version__` equals the tag. 3. It runs `make check` and `make security`. A tag on a red commit publishes nothing. 4. It runs `python3 audit/engine/build.py`. This builds the reproducible single-file engine, `audit.pyz`. 5. It runs `make validate-skills`. This builds the `agentic-readiness-audit` and `agentic-readiness-implement` skill bundles and validates both against the Agent Skills specification. 6. It zips the two bundles from `dev/dist/skills/` and writes `SHA256SUMS` for `audit.pyz` and the two zips. -7. It attaches the four files to the GitHub release. If the tag has no release (a run started by hand), it creates +7. It signs build provenance for the three files in `SHA256SUMS` (`actions/attest`, public Sigstore). GitHub stores + the attestation. The signed bundle is written to `vX.Y.Z.sigstore.json`. +8. It attaches the five files to the GitHub release. If the tag has no release (a run started by hand), it creates one. Every action in both workflows is pinned by commit SHA. To check a download, run `sha256sum -c SHA256SUMS` in the -folder that holds all four files. +folder that holds the files. To check where a file came from, follow +[Step 2b of Install the skills](docs/how-to/install-the-skills.md#step-2b-optional-check-the-build-provenance). ## If the release fails @@ -284,7 +288,7 @@ with no files. 2. If the cause was outside the code (for example a network failure), run the workflow again for the same tag: ```bash - gh workflow run release.yml --ref main -f tag=vX.Y.Z + gh workflow run release.yml --ref vX.Y.Z -f tag=vX.Y.Z ``` 3. If the code must change, fix it on a work branch from `dev` and merge it. Then promote `dev` again. release-please @@ -391,6 +395,8 @@ promotion ([Step 1](#step-1-make-sure-that-dev-is-ready)): a merge commit. - [ ] The release pull request shows the version you expect, its `CHANGELOG.md` entry reads well, and it is merged with a merge commit. -- [ ] `release.yml` passed; the release lists `audit.pyz`, both skill zips and `SHA256SUMS`; and - `sha256sum -c SHA256SUMS` passes on the downloads. +- [ ] `release.yml` passed; the release lists `audit.pyz`, both skill zips, `SHA256SUMS` and `vX.Y.Z.sigstore.json`; + `sha256sum -c SHA256SUMS` passes on the downloads; and + `gh attestation verify audit.pyz --repo Nistapp/codebase-agentic-readiness-framework --source-ref refs/tags/vX.Y.Z` + passes. - [ ] The back-merge pull request into `dev` is merged, and its branch is deleted. diff --git a/SECURITY.md b/SECURITY.md index 36f7d352..e359b5a3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -25,5 +25,5 @@ This policy covers what this repository ships and runs: Examples of what we want to hear about: a tool that writes outside the places its documentation names, a recipe step that leads an agent to expose a secret or run a command the user did not approve, or a release asset that does not -match its `SHA256SUMS`. Problems in the framework documents themselves (wrong or unsafe advice) are welcome as -reports too. +match its `SHA256SUMS` or fails `gh attestation verify`. Problems in the framework documents themselves (wrong or +unsafe advice) are welcome as reports too. diff --git a/dev/tests/test_release_provenance.py b/dev/tests/test_release_provenance.py new file mode 100644 index 00000000..ebbd09ec --- /dev/null +++ b/dev/tests/test_release_provenance.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna + +"""The release workflow signs build provenance for the files it publishes, from the tag it builds. + +The provenance records the ref and commit that the run started from, not the commit it checked out. So +``release-please.yml`` must start ``release.yml`` from the tag, and ``release.yml`` must refuse any other ref. The +signed bundle is attached to the release, because ``gh attestation verify`` without ``--bundle`` needs a GitHub login. +How users verify: ``docs/how-to/install-the-skills.md``. The workflows are YAML; the standard library has no YAML +parser, so these tests read the text. +""" + +from __future__ import annotations + +import re +import unittest + +import _dev_paths # noqa: F401 (puts dev/ on sys.path) +from _dev_paths import REPO + +RELEASE = REPO / ".github" / "workflows" / "release.yml" +RELEASE_PLEASE = REPO / ".github" / "workflows" / "release-please.yml" +PROVENANCE_PERMISSIONS = ("contents: write", "id-token: write", "attestations: write", "artifact-metadata: write") + + +def job_permissions(text: str) -> list[str]: + """The lines of the first job-level ``permissions:`` block (indented four spaces).""" + block = re.search(r"(?m)^ permissions:\n((?: \S.*\n)+)", text) + return [line.strip() for line in block.group(1).splitlines()] if block else [] + + +class ReleaseSignsProvenance(unittest.TestCase): + def setUp(self) -> None: + self.text = RELEASE.read_text(encoding="utf-8") + + def test_attests_the_files_in_the_checksums(self) -> None: + self.assertRegex(self.text, r"uses: actions/attest@[0-9a-f]{40} # v\d+\.\d+\.\d+") + self.assertIn("subject-checksums: ${{ env.RELEASE_DIR }}/SHA256SUMS", self.text) + + def test_attests_after_the_files_are_collected_and_before_the_upload(self) -> None: + collect = self.text.index("name: Collect the release files") + attest = self.text.index("uses: actions/attest@") + upload = self.text.index("name: Attach the files to the release") + self.assertLess(collect, attest) + self.assertLess(attest, upload) + + def test_the_bundle_is_a_release_file(self) -> None: + self.assertIn('cp "$BUNDLE" "$RELEASE_DIR/$TAG.sigstore.json"', self.text) + + def test_permissions_are_on_the_job_only(self) -> None: + self.assertRegex(self.text, r"(?m)^permissions: \{\}$") + self.assertEqual(sorted(job_permissions(self.text)), sorted(PROVENANCE_PERMISSIONS)) + + def test_refuses_a_run_that_did_not_start_from_the_tag(self) -> None: + self.assertIn("RUN_REF: ${{ github.ref }}", self.text) + self.assertIn('if [[ "$RUN_REF" != "refs/tags/$TAG" ]]; then', self.text) + guard = self.text.index("name: Require the run to start from the tag") + self.assertLess(guard, self.text.index("name: Gates")) + + +class ReleasePleaseStartsFromTheTag(unittest.TestCase): + def test_dispatch_uses_the_tag_as_the_ref(self) -> None: + text = RELEASE_PLEASE.read_text(encoding="utf-8") + self.assertIn('gh workflow run release.yml --repo "$GITHUB_REPOSITORY" --ref "$TAG" -f "tag=$TAG"', text) + self.assertNotIn("--ref main", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/how-to/install-the-skills.md b/docs/how-to/install-the-skills.md index e4cf28ba..7335d9fc 100644 --- a/docs/how-to/install-the-skills.md +++ b/docs/how-to/install-the-skills.md @@ -10,6 +10,7 @@ - [ ] Python 3.11 or newer on `PATH` - [ ] `curl` (or a browser), `unzip` and `sha256sum` (on macOS: `shasum -a 256`) +- [ ] Optional, for Step 2b: the [GitHub CLI](https://cli.github.com/) (`gh`). This page was checked with 2.102.0 The quicker path is a clone: see the quick start in the root [`README.md`](../../README.md). Use this page when you want the skills inside another project instead. @@ -34,6 +35,25 @@ sha256sum --ignore-missing -c SHA256SUMS Expect `OK` for each zip. On macOS, use `shasum -a 256 --ignore-missing -c SHA256SUMS`. Do not use a zip that fails this check. +## Step 2b (optional): Check the build provenance + +The checksums prove that a file was not damaged. The build provenance also proves that the file was built by this +repository's release workflow, from the tag you picked. + +1. Also download `.sigstore.json` from the release page. +2. Run this command for each zip. Replace `` with the tag, for example `v0.3.3`: + + ```bash + gh attestation verify agentic-readiness-audit-.zip \ + --repo Nistapp/codebase-agentic-readiness-framework \ + --source-ref refs/tags/ \ + --bundle .sigstore.json + ``` + +The command exits with 0 when the file passes. Do not use a file that fails. With `--bundle`, you do not need a +GitHub login. Without it, `gh` fetches the attestation from GitHub and asks you to run `gh auth login` first. +Releases up to v0.3.2 have no provenance. + ## Step 3: Unzip into the skills folder Unzip both into the `.claude/skills/` folder of the project where you start Claude Code: diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 367e95c6..ea9ee8f9 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -29,6 +29,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | skills-ref | A skill bundle that breaks the Agent Skills specification | `make validate-skills` | Release only | `dev/tools/pyproject.toml` | | `dev/commit_lint.py` | A commit subject that is not Conventional Commits | `make commit-lint` | `commits` | This repository | | The audit engine | A readiness finding that `.audit-baseline.json` does not have | `make self-audit` | `self-audit` | This repository | +| actions/attest | Signs build provenance for the release files; users check it with `gh attestation verify` ([Install the skills](../how-to/install-the-skills.md#step-2b-optional-check-the-build-provenance)) | — | `release.yml`, release only | `.github/workflows/release.yml` (action pinned by SHA) | | CodeQL | Security flaws in the Python code and the workflows. Advisory: alerts go to the Security tab ([Code scanning](branch-protection.md#code-scanning)) | — | `codeql.yml`, outside `CI Gate` | `.github/workflows/codeql.yml` (action pinned by SHA) | | release-please | Computes the next version from the commit subjects, writes `CHANGELOG.md`, tags and creates the GitHub release (`release-please-config.json`) | — | `release-please.yml`, on each push to `main` | `.github/workflows/release-please.yml` (action pinned by SHA) | | uv | Runs the Python tools through `uvx` and `uv run` | — | All jobs | `dev/tools/pyproject.toml` | From 5c26847aef164f1787a3133978fdab5d204ff74a Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 12:07:31 +0530 Subject: [PATCH 022/111] ci: fix zizmor's pedantic findings and gate on the pedantic persona zizmor's default persona hid 13 help-level findings. Each permission now carries a comment that says why it is needed (6), the CI Gate job reads the needed jobs' results through env instead of expanding them into the script (6), and the source-branch check cancels an older run for the same pull request (1). make workflow-lint now runs zizmor with --persona pedantic, so these cannot come back unnoticed. Roadmap-Step: A6 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 14 ++++++++------ .github/workflows/codeql.yml | 4 ++-- .github/workflows/pr-source-branch.yml | 5 +++++ .github/workflows/release-please.yml | 14 +++++++------- .github/workflows/release.yml | 8 ++++---- Makefile | 5 +++-- dev/tests/test_release_provenance.py | 4 ++-- docs/reference/dev-tooling.md | 2 +- 8 files changed, 32 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a182ff4..2c183f32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,13 +175,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Require check, security, coverage, self-audit, docs and commits to have succeeded + # The results come in through env, never expanded into the script (zizmor: template-injection). + env: + check: ${{ needs.check.result }} + security: ${{ needs.security.result }} + coverage: ${{ needs.coverage.result }} + self_audit: ${{ needs.self-audit.result }} + docs: ${{ needs.docs.result }} + commits: ${{ needs.commits.result }} run: | - check="${{ needs.check.result }}" - security="${{ needs.security.result }}" - coverage="${{ needs.coverage.result }}" - self_audit="${{ needs.self-audit.result }}" - docs="${{ needs.docs.result }}" - commits="${{ needs.commits.result }}" echo "check: $check" echo "security: $security" echo "coverage: $coverage" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 408a9ccf..ba479909 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -27,8 +27,8 @@ jobs: name: analyze (${{ matrix.language }}) runs-on: ubuntu-latest permissions: - contents: read - security-events: write + contents: read # check out the code + security-events: write # upload the analysis to the Security tab strategy: fail-fast: false matrix: diff --git a/.github/workflows/pr-source-branch.yml b/.github/workflows/pr-source-branch.yml index 9835a409..8e37d736 100644 --- a/.github/workflows/pr-source-branch.yml +++ b/.github/workflows/pr-source-branch.yml @@ -18,6 +18,11 @@ on: branches: [main] types: [opened, reopened, synchronize, edited] +# A newer event on the same pull request makes an older run pointless. +concurrency: + group: pr-source-branch-${{ github.event.pull_request.number }} + cancel-in-progress: true + permissions: {} jobs: diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index f2ddfee4..4d64652d 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -37,9 +37,9 @@ jobs: name: release-please runs-on: ubuntu-latest permissions: - contents: write - issues: write - pull-requests: write + contents: write # create the tag and the GitHub release + issues: write # create the release labels that release-please uses + pull-requests: write # open and update the release pull request outputs: release_created: ${{ steps.release.outputs.release_created }} tag_name: ${{ steps.release.outputs.tag_name }} @@ -71,7 +71,7 @@ jobs: if: needs.release-please.outputs.prs_created == 'true' && needs.release-please.outputs.pr_branch != '' runs-on: ubuntu-latest permissions: - contents: write + contents: write # push the markdownlint fix of CHANGELOG.md to the release branch steps: - name: Make a GitHub App token id: app-token @@ -115,7 +115,7 @@ jobs: if: needs.release-please.outputs.release_created == 'true' runs-on: ubuntu-latest permissions: - actions: write + actions: write # start release.yml with a workflow dispatch steps: - name: Start release.yml for the new tag env: @@ -132,8 +132,8 @@ jobs: if: needs.release-please.outputs.release_created == 'true' runs-on: ubuntu-latest permissions: - contents: write - pull-requests: write + contents: write # create the back-merge branch + pull-requests: write # open its pull request into dev steps: - name: Make a GitHub App token id: app-token diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bce0c5f8..a7fc50f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,10 +36,10 @@ jobs: name: release ${{ inputs.tag }} runs-on: ubuntu-latest permissions: - contents: write - id-token: write - attestations: write - artifact-metadata: write + contents: write # create the release and attach the files + id-token: write # mint the OIDC token that Sigstore signs the provenance with + attestations: write # store the provenance in the repository's attestations + artifact-metadata: write # record the release files as attested artifacts steps: - name: Require a tag of the form vX.Y.Z env: diff --git a/Makefile b/Makefile index efafed57..89035fb2 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,8 @@ QUALITY_BASELINE := $(PYTHON) dev/quality_baseline.py check DEP_LOCKFILES := dev/tools/uv.lock dev/tools/package-lock.json dev/tools/osv/go.mod COVERAGE := uvx coverage@$(call pin,coverage) # Workflow and GitHub-config checks: actionlint (syntax, expressions, shellcheck on run: blocks), zizmor (workflow -# and Dependabot security, offline) and check-jsonschema (the SchemaStore schemas it bundles, offline). +# and Dependabot security, offline, at the pedantic persona: its help-level findings fail too) and check-jsonschema +# (the SchemaStore schemas it bundles, offline). ACTIONLINT := uvx --from actionlint-py==$(call pin,actionlint-py) actionlint ZIZMOR := uvx zizmor@$(call pin,zizmor) # The spell check: typos reads _typos.toml (the word list and the excluded data paths) from the repository root, @@ -175,7 +176,7 @@ spell-fix: workflow-lint: $(ACTIONLINT) - $(ZIZMOR) --offline .github + $(ZIZMOR) --offline --persona pedantic .github $(CHECK_JSONSCHEMA) --builtin-schema vendor.github-workflows .github/workflows/*.yml $(CHECK_JSONSCHEMA) --builtin-schema vendor.dependabot .github/dependabot.yml diff --git a/dev/tests/test_release_provenance.py b/dev/tests/test_release_provenance.py index ebbd09ec..911c88ab 100644 --- a/dev/tests/test_release_provenance.py +++ b/dev/tests/test_release_provenance.py @@ -24,9 +24,9 @@ def job_permissions(text: str) -> list[str]: - """The lines of the first job-level ``permissions:`` block (indented four spaces).""" + """The lines of the first job-level ``permissions:`` block (indented four spaces), without their comments.""" block = re.search(r"(?m)^ permissions:\n((?: \S.*\n)+)", text) - return [line.strip() for line in block.group(1).splitlines()] if block else [] + return [line.split("#")[0].strip() for line in block.group(1).splitlines()] if block else [] class ReleaseSignsProvenance(unittest.TestCase): diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index ea9ee8f9..36c6c49d 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -18,7 +18,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | jsonschema | The tools' schema validator disagrees with the reference package | `make crosscheck` | `check` | `dev/tools/pyproject.toml` | | `dev/recipe_lint.py` | Broken links, bad `SKILL.md` frontmatter, oversize files, unknown tools, check IDs and requirement IDs | `make recipe-lint` | `check` | This repository | | actionlint | Workflow mistakes: syntax, expressions, action inputs, shellcheck on `run:` blocks | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` (`actionlint-py`) | -| zizmor | Workflow and Dependabot security: injection, broad permissions, unpinned actions, cooldown | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | +| zizmor | Workflow and Dependabot security: injection, broad permissions, unpinned actions, cooldown. Runs at the pedantic persona, so a permission without a comment or a missing concurrency limit fails too | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | | typos | Misspelled words in the tracked text, code identifiers included (word list: [`_typos.toml`](../../_typos.toml)) ([Spell check](#spell-check)) | `make spell`; `make spell-fix` writes the fixes | `check` | `dev/tools/pyproject.toml` | | check-jsonschema | `.github/workflows/*.yml` and `.github/dependabot.yml` that do not match their SchemaStore schemas | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | | detect-secrets | Key-shaped strings not in `.secrets.baseline` | `make security` | `security` | `dev/tools/pyproject.toml` | From f807d6346bdf8972e7182d79751f044f54f4799a Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 12:11:50 +0530 Subject: [PATCH 023/111] fix(dev): ignore .env files at any depth The .env rules in .gitignore were root-anchored, so a nested .env (for example tools/x/.env) could be committed by accident; the self-audit reported SEC-01 and SEC-03 PARTIAL for it. The rules now match at any depth. The engine's fixture .env files are already tracked and stay tracked; a new one is added with `git add -f`, like the key material. The audit baseline drops both findings (Phase 1 score 0.8525 -> 0.873). Roadmap-Step: A6 Co-Authored-By: Claude Opus 5.5 --- .audit-baseline.json | 184 ++++++++++--------------------------------- .gitignore | 13 +-- .secrets.baseline | 8 +- 3 files changed, 52 insertions(+), 153 deletions(-) diff --git a/.audit-baseline.json b/.audit-baseline.json index 19e63241..1c482e86 100644 --- a/.audit-baseline.json +++ b/.audit-baseline.json @@ -129,7 +129,7 @@ "verb": "test" }, { - "command": "make format-check && make typecheck && make test && make verify-rules && make crosscheck && make recipe-lint && make workflow-lint", + "command": "make format-check && make typecheck && make test && make verify-rules && make crosscheck && make recipe-lint && make workflow-lint && make spell", "resolved": true, "runner": "Makefile", "verb": "check" @@ -343,7 +343,7 @@ "verb": "test" }, { - "command": "make format-check && make typecheck && make test && make verify-rules && make crosscheck && make recipe-lint && make workflow-lint", + "command": "make format-check && make typecheck && make test && make verify-rules && make crosscheck && make recipe-lint && make workflow-lint && make spell", "resolved": true, "runner": "Makefile", "verb": "check" @@ -582,6 +582,7 @@ "Makefile", "audit/engine/", "AGENTS.md", + "_typos.toml", ".python-version", "dev/tools/pyproject.toml", "dev/tools/package.json", @@ -645,7 +646,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "the prohibitions section of AGENTS.md names paths that exist: Makefile, audit/engine/, AGENTS.md, .python-version, dev/tools/pyproject.toml, dev/tools/package.json, dev/tools/osv/go.mod, lefthook.yml (+45 more)", + "summary": "the prohibitions section of AGENTS.md names paths that exist: Makefile, audit/engine/, AGENTS.md, _typos.toml, .python-version, dev/tools/pyproject.toml, dev/tools/package.json, dev/tools/osv/go.mod (+46 more)", "tier": "B", "title": "Allow/deny artifact present", "verdict": "PASS" @@ -804,7 +805,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "test verb on Makefile (cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests); 84 test file(s)", + "summary": "test verb on Makefile (cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests); 85 test file(s)", "tier": "A", "title": "Test verb exists, suite non-empty", "verdict": "PASS" @@ -837,7 +838,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "skip/xfail census across 84 test file(s): python 1; total 1", + "summary": "skip/xfail census across 85 test file(s): python 1; total 1", "tier": "B", "title": "Skip / xfail census", "verdict": "PASS" @@ -922,10 +923,10 @@ }, { "data": { - "denominator": 1152, + "denominator": 1155, "kind": "ratio", "method": "python: module-level def/class not starting with '_'", - "numerator": 459, + "numerator": 460, "unit": "public symbols" }, "id": "NAV-05", @@ -934,33 +935,33 @@ "scored": false, "severity": "COSMETIC", "status": "implemented", - "summary": "459/1152 public symbols documented (40%); method: python: module-level def/class not starting with '_'", + "summary": "460/1155 public symbols documented (40%); method: python: module-level def/class not starting with '_'", "tier": "B", "title": "Public-symbol documentation coverage", "verdict": "PARTIAL" }, { "data": { - "covered": [], - "kind": "secret_shapes", - "missing": [], - "root_only": [ + "covered": [ "dotenv", "dotenv variant" ], + "kind": "secret_shapes", + "missing": [], + "root_only": [], "shapes": [ { - "covered": false, + "covered": true, "label": "dotenv", "probe": ".env", - "root_only": true, + "root_only": false, "suggested": ".env" }, { - "covered": false, + "covered": true, "label": "dotenv variant", "probe": ".env.local", - "root_only": true, + "root_only": false, "suggested": ".env.*" } ] @@ -971,10 +972,10 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "no tracked .env, but ignore rules are incomplete: root-anchored only: dotenv, dotenv variant (the rule covers the root, not nested files such as `a/.env`)", + "summary": "no tracked .env and ignore rules cover the shape", "tier": "B", "title": ".env untracked and ignored", - "verdict": "PARTIAL" + "verdict": "PASS" }, { "data": { @@ -1015,6 +1016,8 @@ { "data": { "covered": [ + "dotenv", + "dotenv variant", "ssh private key", "PEM", "key", @@ -1027,23 +1030,20 @@ ], "kind": "secret_shapes", "missing": [], - "root_only": [ - "dotenv", - "dotenv variant" - ], + "root_only": [], "shapes": [ { - "covered": false, + "covered": true, "label": "dotenv", "probe": ".env", - "root_only": true, + "root_only": false, "suggested": ".env" }, { - "covered": false, + "covered": true, "label": "dotenv variant", "probe": ".env.local", - "root_only": true, + "root_only": false, "suggested": ".env.*" }, { @@ -1117,10 +1117,10 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "covered: ssh private key, PEM, key, PKCS#12, PFX, keystore, certificate (.crt), certificate (.cer), AWS credentials; root-anchored only: dotenv, dotenv variant (the rule covers the root, not nested files such as `a/.env`)", + "summary": "covered: dotenv, dotenv variant, ssh private key, PEM, key, PKCS#12, PFX, keystore, certificate (.crt), certificate (.cer), AWS credentials", "tier": "B", "title": "Ignore patterns cover secret shapes", - "verdict": "PARTIAL" + "verdict": "PASS" }, { "data": {}, @@ -1303,57 +1303,6 @@ "statement": "An agent cannot discover the codebase through an MCP index because no MCP configuration names codebase-memory-mcp for this repository.", "verdict": "FAIL" }, - { - "check": "SEC-01", - "evidence": [ - { - "line": null, - "note": "__pycache__", - "path": ".gitignore" - }, - { - "line": null, - "note": "*.py[cod]", - "path": ".gitignore" - }, - { - "line": null, - "note": ".mypy_cache", - "path": ".gitignore" - }, - { - "line": null, - "note": ".ruff_cache", - "path": ".gitignore" - }, - { - "line": null, - "note": "/dev/dist", - "path": ".gitignore" - }, - { - "line": null, - "note": "/audit/engine/dist", - "path": ".gitignore" - }, - { - "line": null, - "note": "/dist", - "path": ".gitignore" - }, - { - "line": null, - "note": "*.pyz", - "path": ".gitignore" - } - ], - "id": "SEC-01", - "phase": 1, - "remediation": "Add the .env shape without a leading slash (for example `.env` and `.env.*`) to .gitignore so an environment file cannot be committed by accident, at any depth.", - "severity": "BLOCKER", - "statement": "An agent cannot trust that a future .env stays out of history because ignore rules are incomplete (root-anchored only: dotenv, dotenv variant (the rule covers the root, not nested files such as `a/.env`)).", - "verdict": "PARTIAL" - }, { "check": "BASE-02", "evidence": [ @@ -1394,70 +1343,19 @@ "severity": "DEGRADER", "statement": "An agent cannot state how many violations exist today because a counting surface exists but no numeric violation budget or count is committed.", "verdict": "PARTIAL" - }, - { - "check": "SEC-03", - "evidence": [ - { - "line": null, - "note": "__pycache__", - "path": ".gitignore" - }, - { - "line": null, - "note": "*.py[cod]", - "path": ".gitignore" - }, - { - "line": null, - "note": ".mypy_cache", - "path": ".gitignore" - }, - { - "line": null, - "note": ".ruff_cache", - "path": ".gitignore" - }, - { - "line": null, - "note": "/dev/dist", - "path": ".gitignore" - }, - { - "line": null, - "note": "/audit/engine/dist", - "path": ".gitignore" - }, - { - "line": null, - "note": "/dist", - "path": ".gitignore" - }, - { - "line": null, - "note": "*.pyz", - "path": ".gitignore" - } - ], - "id": "SEC-03", - "phase": 1, - "remediation": "Add the missing shapes to .gitignore \u2014 for example `.env.*`, `*.pem`, `*.key`, `*.p12`, `*.pfx`, `*.keystore`, `id_rsa*`, `*.crt`, `*.cer`, `.aws/*`.", - "severity": "DEGRADER", - "statement": "An agent cannot trust that secret shapes cannot be committed because ignore rules are incomplete (root-anchored only: dotenv, dotenv variant (the rule covers the root, not nested files such as `a/.env`)).", - "verdict": "PARTIAL" } ], "inventory": { - "files_inspected": 339, - "files_set_aside": 460, - "files_skipped": 3, + "files_inspected": 341, + "files_set_aside": 456, + "files_skipped": 7, "set_aside": [ { "files": 3, "path": "audit/engine/examples" }, { - "files": 276, + "files": 273, "path": "audit/engine/tests/fixtures" }, { @@ -1477,7 +1375,7 @@ "path": "docs/templates" }, { - "files": 39, + "files": 38, "path": "greenfield/templates" } ], @@ -1528,15 +1426,15 @@ } ], "provenance": { - "framework_revision": "b678cd339ce6114fd1ed6df18a68d6b5596a8557", + "framework_revision": "3ac756b9ad29656d1f56e2a5406257033faf668c", "git_branch": "chore/v0.3.3-track-a-tooling", "git_dirty": true, - "git_sha": "b678cd339ce6114fd1ed6df18a68d6b5596a8557", + "git_sha": "3ac756b9ad29656d1f56e2a5406257033faf668c", "phase": 1, "probes_enabled": false, "ruleset_hash": "970e6792f98eeeb5", "ruleset_revision": "2026-11-06.1", - "scanned_at": "2026-11-09T05:17:36+00:00", + "scanned_at": "2026-11-09T06:37:40+00:00", "target": "/home/kc/Projects/Nistapp-agentic-frameworks/car-framework-track-a", "tool": "python-agentic-audit", "user_config": { @@ -1567,12 +1465,12 @@ "test_frameworks": [] }, "summary": { - "blockers": 3, + "blockers": 2, "checks": { "attest": 0, "fail": 2, - "partial": 5, - "pass": 58, + "partial": 3, + "pass": 60, "unknown": 4 }, "checks_applicable": 69, @@ -1580,8 +1478,8 @@ "checks_scoreable": 55, "cosmetic": 0, "coverage_note": "55 of 55 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", - "degraders": 2, - "phase1_score": 0.8525 + "degraders": 1, + "phase1_score": 0.873 }, "unattested": [ "index-in-use", diff --git a/.gitignore b/.gitignore index 716406b2..890a31e3 100644 --- a/.gitignore +++ b/.gitignore @@ -15,12 +15,13 @@ __pycache__/ **/.gradle/ **/.pytest_cache/ -# environment files (secrets never committed; templates are) -# `.env` is root-anchored on purpose: the engine's fixtures contain `.env` files as data (tests/fixtures/*/.env) and -# must stay committable. The root file is the only one a developer creates here. -/.env -/.env.* -!/.env.example +# environment files (secrets never committed; templates are), at any depth +# The engine's fixtures hold `.env` files as data (audit/engine/tests/fixtures/*/.env). They are already tracked, and +# git keeps tracking them. A new fixture `.env` is added with `git add -f` and a note in its test, like the key +# material below. +.env +.env.* +!.env.example # secret and key material by shape. A fixture that needs one of these is added with `git add -f` and a note in its # test; nothing in the repository's own configuration should ever be one. diff --git a/.secrets.baseline b/.secrets.baseline index 2d2cda55..598eced0 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -131,16 +131,16 @@ { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", - "hashed_secret": "f7a50539d1173ad5105d5dc9fe93786e0e3fec89", + "hashed_secret": "3597bef0211ce89d6702c53efaffb61a440b30c7", "is_verified": false, - "line_number": 1531 + "line_number": 1429 }, { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", "hashed_secret": "228ea6c3984c8d1226e0ffc33455f16dce652d01", "is_verified": false, - "line_number": 1537 + "line_number": 1435 } ], "audit/engine/audit/rules/checks/con.py": [ @@ -1458,5 +1458,5 @@ } ] }, - "generated_at": "2026-11-09T06:52:39Z" + "generated_at": "2026-11-09T06:52:43Z" } From c0cc4a7623d57350b3c0eafd424902322fb9a76e Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 12:15:56 +0530 Subject: [PATCH 024/111] ci: add an advisory OpenSSF Scorecard workflow Scorecard scores the repository's security practices on each push to dev and weekly, and uploads the result to the Security tab (category scorecard). publish_results is false: nothing goes to the public OpenSSF API, so there is no public score page or badge until the maintainer decides to publish. No ruleset requires the workflow. Roadmap-Step: A6 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/scorecard.yml | 43 +++++++++++++++++++++++++++++ docs/reference/branch-protection.md | 13 +++++++-- docs/reference/dev-tooling.md | 1 + 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/scorecard.yml diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 00000000..1530b8ca --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna +# +# OpenSSF Scorecard: scores this repository's security practices (branch protection, pinned actions, token +# permissions, signed releases, ...) and uploads the result to the Security tab as code-scanning alerts. Advisory: +# no ruleset requires it (docs/reference/branch-protection.md, "Code scanning"). +# `publish_results: false`: nothing is sent to the public OpenSSF API, so there is no public score page or badge. +# Turning it on is the maintainer's decision; it also needs `id-token: write` on the job. +# Scorecard runs on the default branch only (`dev`). +name: Scorecard + +on: + push: + branches: [dev] + schedule: + - cron: "29 4 * * 1" + +permissions: {} + +concurrency: + group: scorecard-${{ github.ref }} + cancel-in-progress: true + +jobs: + analysis: + name: analysis + runs-on: ubuntu-latest + permissions: + contents: read # check out the code and read the repository's settings through the API + security-events: write # upload the result to the Security tab + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ossf/scorecard-action@fa7117db83c52d787ae5a89f350999701505d78e # v2.5.0 + with: + results_file: results.sarif + results_format: sarif + publish_results: false + - uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + sarif_file: results.sarif + category: scorecard diff --git a/docs/reference/branch-protection.md b/docs/reference/branch-protection.md index 76d43191..56c55d52 100644 --- a/docs/reference/branch-protection.md +++ b/docs/reference/branch-protection.md @@ -106,7 +106,14 @@ workflows. It runs on each pull request into `dev` or `main`, on each push to `d scan the engine's test fixtures (`audit/engine/tests/fixtures/`), because they hold flawed code on purpose. It does not scan JavaScript or TypeScript, because the only such files are those fixtures. -Code scanning is advisory. No ruleset requires the CodeQL workflow, and an alert does not block a merge. The +[`.github/workflows/scorecard.yml`](../../.github/workflows/scorecard.yml) runs OpenSSF Scorecard on each push to +`dev` and every Monday. It scores the repository's security practices, such as pinned actions, token permissions and +signed releases. Each low score shows as an alert in the Security tab, in the `scorecard` category. The results are +not published: `publish_results` is `false`, so there is no public score page and no badge. Publishing is the +maintainer's decision. + +Code scanning is advisory. No ruleset requires the CodeQL or the Scorecard workflow, and an alert does not block a +merge. The maintainer handles an alert in the Security tab: 1. Fix the code in a pull request into `dev`. The alert closes when the fix reaches `dev`. @@ -127,8 +134,8 @@ On 2026-11-08 the default branch changed from `main` to `dev`. On 2026-11-07 these settings were also turned on: private vulnerability reporting, secret scanning, push protection and Dependabot security updates (see [`SECURITY.md`](../../SECURITY.md)). -On 2026-11-09 the CodeQL workflow was added. Code scanning default setup stays off: the workflow replaces it (see -[Code scanning](#code-scanning)). +On 2026-11-09 the CodeQL and Scorecard workflows were added. Code scanning default setup stays off: the CodeQL +workflow replaces it (see [Code scanning](#code-scanning)). On 2026-11-08 "Automatically delete head branches" was turned on. GitHub deletes the head branch of a pull request when the pull request is merged. It does not delete `dev` after a promotion pull request, because the `dev` ruleset diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 36c6c49d..97433e0a 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -31,6 +31,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | The audit engine | A readiness finding that `.audit-baseline.json` does not have | `make self-audit` | `self-audit` | This repository | | actions/attest | Signs build provenance for the release files; users check it with `gh attestation verify` ([Install the skills](../how-to/install-the-skills.md#step-2b-optional-check-the-build-provenance)) | — | `release.yml`, release only | `.github/workflows/release.yml` (action pinned by SHA) | | CodeQL | Security flaws in the Python code and the workflows. Advisory: alerts go to the Security tab ([Code scanning](branch-protection.md#code-scanning)) | — | `codeql.yml`, outside `CI Gate` | `.github/workflows/codeql.yml` (action pinned by SHA) | +| OpenSSF Scorecard | Weak security practices of the repository (Branch-Protection, Pinned-Dependencies, Token-Permissions, Signed-Releases, ...). Advisory: alerts go to the Security tab; results are not published ([Code scanning](branch-protection.md#code-scanning)) | — | `scorecard.yml`, on push to `dev` and weekly | `.github/workflows/scorecard.yml` (action pinned by SHA) | | release-please | Computes the next version from the commit subjects, writes `CHANGELOG.md`, tags and creates the GitHub release (`release-please-config.json`) | — | `release-please.yml`, on each push to `main` | `.github/workflows/release-please.yml` (action pinned by SHA) | | uv | Runs the Python tools through `uvx` and `uv run` | — | All jobs | `dev/tools/pyproject.toml` | | Node | Runs the Node tools through `npx` | — | `docs` | `.nvmrc` | From 65024fb26b6cdc04d071ee448257a38c8b2bda10 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 12:22:11 +0530 Subject: [PATCH 025/111] docs(self-audit): refresh the self-audit page and neutralise the baseline The page quoted the 2026-11-06 report (0.8402, 7 findings, ruleset 2026-11-06.1). It now states the current report on ruleset 2026-11-09.1 (with Track C's CI-05): 0.875, 70 checks (56 scoreable), 61 PASS and 3 findings (EXEC-01 and IDX-01 BLOCKER, BASE-02 DEGRADER). It records what closed since (HYG-02, HYG-06, TOOL-04, SEC-01, SEC-03), the EXEC-02 PARTIAL reason, and the 2026-11-09 review of NAV-05 (left as it is). The report is published as schema/examples/self-audit/2026-11-09.json. The baseline carried absolute local paths in provenance.target and the IDX-01 summary, already on dev since a12ffce; both are neutralised as the page's "Neutralised path" section requires. Roadmap-Step: A6 Co-Authored-By: Claude Opus 5.5 --- .audit-baseline.json | 57 +- .secrets.baseline | 26 +- .../examples/self-audit/2026-11-09.json | 1504 +++++++++++++++++ docs/reference/self-audit.md | 43 +- 4 files changed, 1582 insertions(+), 48 deletions(-) create mode 100644 audit/schema/examples/self-audit/2026-11-09.json diff --git a/.audit-baseline.json b/.audit-baseline.json index 1c482e86..0f82325f 100644 --- a/.audit-baseline.json +++ b/.audit-baseline.json @@ -22,7 +22,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "index store could not be located; searched: user-level locations (not read: --no-user-config), /home/kc/Projects/Nistapp-agentic-frameworks/car-framework-track-a/.codebase-memory/graph.db.zst", + "summary": "index store could not be located; searched: user-level locations (not read: --no-user-config), /.codebase-memory/graph.db.zst", "tier": "A", "title": "An index exists for this checkout", "verdict": "UNKNOWN" @@ -523,6 +523,19 @@ "title": "CI-only steps flagged", "verdict": "PASS" }, + { + "data": {}, + "id": "CI-05", + "pack": "CI", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "github: local actionlint (Makefile), actionlint (dev/tools/pyproject.toml), check-jsonschema (Makefile), zizmor (Makefile), zizmor (dev/tools/pyproject.toml), CI zizmor (.github/workflows/ci.yml)", + "tier": "B", + "title": "Workflow files are linted", + "verdict": "PASS" + }, { "data": {}, "id": "BASE-01", @@ -923,10 +936,10 @@ }, { "data": { - "denominator": 1155, + "denominator": 1159, "kind": "ratio", "method": "python: module-level def/class not starting with '_'", - "numerator": 460, + "numerator": 462, "unit": "public symbols" }, "id": "NAV-05", @@ -935,7 +948,7 @@ "scored": false, "severity": "COSMETIC", "status": "implemented", - "summary": "460/1155 public symbols documented (40%); method: python: module-level def/class not starting with '_'", + "summary": "462/1159 public symbols documented (40%); method: python: module-level def/class not starting with '_'", "tier": "B", "title": "Public-symbol documentation coverage", "verdict": "PARTIAL" @@ -1346,8 +1359,8 @@ } ], "inventory": { - "files_inspected": 341, - "files_set_aside": 456, + "files_inspected": 342, + "files_set_aside": 463, "files_skipped": 7, "set_aside": [ { @@ -1355,15 +1368,15 @@ "path": "audit/engine/examples" }, { - "files": 273, + "files": 277, "path": "audit/engine/tests/fixtures" }, { - "files": 32, + "files": 33, "path": "audit/schema/examples" }, { - "files": 92, + "files": 94, "path": "dev/eval/expected/fixtures" }, { @@ -1426,16 +1439,16 @@ } ], "provenance": { - "framework_revision": "3ac756b9ad29656d1f56e2a5406257033faf668c", + "framework_revision": "801faf42cf1104512148020a8195f7138d5b55ac", "git_branch": "chore/v0.3.3-track-a-tooling", - "git_dirty": true, - "git_sha": "3ac756b9ad29656d1f56e2a5406257033faf668c", + "git_dirty": false, + "git_sha": "801faf42cf1104512148020a8195f7138d5b55ac", "phase": 1, "probes_enabled": false, - "ruleset_hash": "970e6792f98eeeb5", - "ruleset_revision": "2026-11-06.1", - "scanned_at": "2026-11-09T06:37:40+00:00", - "target": "/home/kc/Projects/Nistapp-agentic-frameworks/car-framework-track-a", + "ruleset_hash": "04e5d0ff9063e012", + "ruleset_revision": "2026-11-09.1", + "scanned_at": "2026-11-09T06:56:35+00:00", + "target": ".", "tool": "python-agentic-audit", "user_config": { "paths_read": [], @@ -1470,16 +1483,16 @@ "attest": 0, "fail": 2, "partial": 3, - "pass": 60, + "pass": 61, "unknown": 4 }, - "checks_applicable": 69, - "checks_implemented": 69, - "checks_scoreable": 55, + "checks_applicable": 70, + "checks_implemented": 70, + "checks_scoreable": 56, "cosmetic": 0, - "coverage_note": "55 of 55 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", + "coverage_note": "56 of 56 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", "degraders": 1, - "phase1_score": 0.873 + "phase1_score": 0.875 }, "unattested": [ "index-in-use", diff --git a/.secrets.baseline b/.secrets.baseline index 598eced0..792df653 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -131,16 +131,16 @@ { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", - "hashed_secret": "3597bef0211ce89d6702c53efaffb61a440b30c7", + "hashed_secret": "c7f38d4728cc28cf34fd049666a73c7d01edc606", "is_verified": false, - "line_number": 1429 + "line_number": 1442 }, { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", - "hashed_secret": "228ea6c3984c8d1226e0ffc33455f16dce652d01", + "hashed_secret": "78d6ecebdb129eed66e4824377d8ca833d9b3932", "is_verified": false, - "line_number": 1435 + "line_number": 1448 } ], "audit/engine/audit/rules/checks/con.py": [ @@ -438,6 +438,22 @@ "line_number": 1518 } ], + "audit/schema/examples/self-audit/2026-11-09.json": [ + { + "type": "Hex High Entropy String", + "filename": "audit/schema/examples/self-audit/2026-11-09.json", + "hashed_secret": "c7f38d4728cc28cf34fd049666a73c7d01edc606", + "is_verified": false, + "line_number": 1442 + }, + { + "type": "Hex High Entropy String", + "filename": "audit/schema/examples/self-audit/2026-11-09.json", + "hashed_secret": "78d6ecebdb129eed66e4824377d8ca833d9b3932", + "is_verified": false, + "line_number": 1448 + } + ], "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json": [ { "type": "Hex High Entropy String", @@ -1458,5 +1474,5 @@ } ] }, - "generated_at": "2026-11-09T06:52:43Z" + "generated_at": "2026-11-09T06:56:41Z" } diff --git a/audit/schema/examples/self-audit/2026-11-09.json b/audit/schema/examples/self-audit/2026-11-09.json new file mode 100644 index 00000000..0f82325f --- /dev/null +++ b/audit/schema/examples/self-audit/2026-11-09.json @@ -0,0 +1,1504 @@ +{ + "candidate_components": [], + "checks": [ + { + "data": {}, + "id": "IDX-01", + "pack": "IDX", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "no project-level MCP configuration found (user-level configuration not read: --no-user-config)", + "tier": "A", + "title": "codebase-memory-mcp registered for this repository", + "verdict": "FAIL" + }, + { + "data": {}, + "id": "IDX-02", + "pack": "IDX", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "index store could not be located; searched: user-level locations (not read: --no-user-config), /.codebase-memory/graph.db.zst", + "tier": "A", + "title": "An index exists for this checkout", + "verdict": "UNKNOWN" + }, + { + "data": {}, + "id": "IDX-03", + "pack": "IDX", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "index freshness is a Tier-C probe; re-run with --run-gates to compare the indexed revision against HEAD", + "tier": "C", + "title": "Index freshness", + "verdict": "UNKNOWN" + }, + { + "data": {}, + "id": "AGT-01", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "3 nested instruction file(s)", + "tier": "A", + "title": "Instruction file present, canonical, and non-trivial", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-02", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "single-root repository: no component-level instruction files required", + "tier": "A", + "title": "Component coverage by instruction files", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-03", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "docs/, the style guide and the scratch contract are named", + "tier": "B", + "title": "Documentation contract named", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-04", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "definition of done ties a doc trigger to the same change set", + "tier": "B", + "title": "Definition of done stated", + "verdict": "PASS" + }, + { + "data": { + "kind": "verb_surface", + "missing": [], + "resolved_count": 7, + "verbs": [ + { + "command": "$(RUFF) format audit implement dev", + "resolved": true, + "runner": "Makefile", + "verb": "format" + }, + { + "command": "make format-check", + "resolved": true, + "runner": "Makefile", + "verb": "format:check" + }, + { + "command": "$(RUFF) check audit implement dev", + "resolved": true, + "runner": "Makefile", + "verb": "lint" + }, + { + "command": "$(MYPY) --config-file mypy.ini $(TYPED_SOURCES) && $(QUALITY_BASELINE) mypy -- $(MYPY) --config-file mypy.ini $(TYPED_SOURCES)", + "resolved": true, + "runner": "Makefile", + "verb": "typecheck" + }, + { + "command": "cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests", + "resolved": true, + "runner": "Makefile", + "verb": "test" + }, + { + "command": "make format-check && make typecheck && make test && make verify-rules && make crosscheck && make recipe-lint && make workflow-lint && make spell", + "resolved": true, + "runner": "Makefile", + "verb": "check" + }, + { + "command": "git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline && go -C dev/tools/osv tool osv-scanner scan source --no-call-analysis=go --config=$(CURDIR)/dev/tools/osv-scanner.toml $(foreach f,$(DEP_LOCKFILES),--lockfile=$(CURDIR)/$(f))", + "resolved": true, + "runner": "Makefile", + "verb": "security" + } + ] + }, + "id": "AGT-05", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "all 7 verbs named in AGENTS.md and resolved on a runner", + "tier": "B", + "title": "Seven verbs named", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-06", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "19 boundary rules reference real paths", + "tier": "B", + "title": "Architectural boundaries declared", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-07", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "prohibitions cover tests, generated files, the default branch and secrets", + "tier": "B", + "title": "Do-not-do / deny rules present", + "verdict": "PASS" + }, + { + "data": { + "kind": "harness_matrix", + "reached": 20, + "reached_tools": [ + "Augment Code", + "Bolt (StackBlitz)", + "Charm Crush", + "Cline", + "Cursor", + "Devin (Cognition)", + "Firebase Studio (ex-IDX)", + "GitHub Copilot / VS Code", + "Goose (Block)", + "JetBrains Junie", + "Jules (Google)", + "Kiro (AWS)", + "OpenAI Codex CLI", + "Roo Code", + "Sourcegraph Amp", + "Trae", + "Warp", + "Windsurf / Devin Desktop", + "Zed", + "opencode" + ], + "total": 28, + "unreached_tools": [ + "Aider", + "Amazon Q Developer", + "Anthropic Claude Code", + "Continue", + "Google Gemini CLI", + "Qwen Code", + "Replit Agent", + "Void" + ] + }, + "id": "AGT-08", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "20/28 harnesses reached; AGENTS.md:46 states the supported harnesses (Anthropic Claude Code, Cursor, Google Gemini CLI, OpenAI Codex CLI, opencode)", + "tier": "B", + "title": "No competing instruction files, and the reach matrix is stated", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-09", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "the branch model and the release boundary are stated", + "tier": "B", + "title": "Branch and release boundary declared", + "verdict": "PASS" + }, + { + "data": {}, + "id": "AGT-10", + "pack": "AGT", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "", + "tier": "B", + "title": "Instruction indirection resolved", + "verdict": "PASS" + }, + { + "data": {}, + "id": "DOC-01", + "pack": "DOC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "44 markdown file(s) under docs/", + "tier": "A", + "title": "docs/ exists", + "verdict": "PASS" + }, + { + "data": {}, + "id": "DOC-02", + "pack": "DOC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "docs/STYLE_GUIDE.md present and named in AGENTS.md", + "tier": "A", + "title": "docs/STYLE_GUIDE.md exists", + "verdict": "PASS" + }, + { + "data": {}, + "id": "DOC-03", + "pack": "DOC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "artefacts/ present, ignored by artefacts/.gitignore, and untracked", + "tier": "B", + "title": "artefacts/ exists and is ignored", + "verdict": "PASS" + }, + { + "data": {}, + "id": "DOC-04", + "pack": "DOC", + "phase": 1, + "scored": true, + "severity": "COSMETIC", + "status": "implemented", + "summary": "6 ADR(s) indexed in docs/architecture/adrs/README.md", + "tier": "A", + "title": "ADR directory and index", + "verdict": "PASS" + }, + { + "data": { + "kind": "verb_surface", + "missing": [], + "resolved_count": 7, + "verbs": [ + { + "command": "$(RUFF) format audit implement dev", + "resolved": true, + "runner": "Makefile", + "verb": "format" + }, + { + "command": "make format-check", + "resolved": true, + "runner": "Makefile", + "verb": "format:check" + }, + { + "command": "$(RUFF) check audit implement dev", + "resolved": true, + "runner": "Makefile", + "verb": "lint" + }, + { + "command": "$(MYPY) --config-file mypy.ini $(TYPED_SOURCES) && $(QUALITY_BASELINE) mypy -- $(MYPY) --config-file mypy.ini $(TYPED_SOURCES)", + "resolved": true, + "runner": "Makefile", + "verb": "typecheck" + }, + { + "command": "cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests", + "resolved": true, + "runner": "Makefile", + "verb": "test" + }, + { + "command": "make format-check && make typecheck && make test && make verify-rules && make crosscheck && make recipe-lint && make workflow-lint && make spell", + "resolved": true, + "runner": "Makefile", + "verb": "check" + }, + { + "command": "git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline && go -C dev/tools/osv tool osv-scanner scan source --no-call-analysis=go --config=$(CURDIR)/dev/tools/osv-scanner.toml $(foreach f,$(DEP_LOCKFILES),--lockfile=$(CURDIR)/$(f))", + "resolved": true, + "runner": "Makefile", + "verb": "security" + } + ] + }, + "id": "CMD-01", + "pack": "CMD", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "format -> Makefile; format:check -> Makefile; lint -> Makefile; typecheck -> Makefile; test -> Makefile; check -> Makefile; security -> Makefile", + "tier": "B", + "title": "Seven verbs resolvable on one runner", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CMD-02", + "pack": "CMD", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "check (Makefile) composes: format-check, typecheck, test", + "tier": "B", + "title": "check composes the read-only verbs", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CMD-03", + "pack": "CMD", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "one runner defines the gate", + "tier": "B", + "title": "No divergent second definition", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CMD-04", + "pack": "CMD", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "no read-only verb invokes a write verb", + "tier": "B", + "title": "Write verbs separated from read-only verbs", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TOOL-01", + "pack": "TOOL", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "formatter ruff.toml + Makefile format verb", + "tier": "B", + "title": "Formatter configured", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TOOL-02", + "pack": "TOOL", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "linter ruff.toml + Makefile lint verb", + "tier": "B", + "title": "Linter configured", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TOOL-03", + "pack": "TOOL", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "strict config: mypy.ini", + "tier": "A", + "title": "Type checker configured", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TOOL-04", + "pack": "TOOL", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "security -> git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline && go -C dev/tools/osv tool osv-scanner scan source --no-call-analysis=go --config=$(CURDIR)/dev/tools/osv-scanner.toml $(foreach f,$(DEP_LOCKFILES),--lockfile=$(CURDIR)/$(f))", + "tier": "A", + "title": "Dependency / security scan verb", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TOOL-05", + "pack": "TOOL", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "wired: lefthook.yml", + "tier": "B", + "title": "Local hook chain active", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TOOL-06", + "pack": "TOOL", + "phase": 1, + "scored": true, + "severity": "COSMETIC", + "status": "implemented", + "summary": "commit rule: lefthook.yml:commit-msg", + "tier": "A", + "title": "Commit convention enforced", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CI-01", + "pack": "CI", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "PR-triggered: .github/workflows/ci.yml (github); .github/workflows/codeql.yml (github); .github/workflows/pr-source-branch.yml (github)", + "tier": "B", + "title": "Pipeline present, triggered on PRs", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CI-02", + "pack": "CI", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "CI verbs: check, security; local read-only verbs not exercised: format:check, lint, test, typecheck", + "tier": "B", + "title": "Local\u2194CI parity", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CI-03", + "pack": "CI", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "no CI-only steps", + "tier": "B", + "title": "CI-only steps flagged", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CI-05", + "pack": "CI", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "github: local actionlint (Makefile), actionlint (dev/tools/pyproject.toml), check-jsonschema (Makefile), zizmor (Makefile), zizmor (dev/tools/pyproject.toml), CI zizmor (.github/workflows/ci.yml)", + "tier": "B", + "title": "Workflow files are linted", + "verdict": "PASS" + }, + { + "data": {}, + "id": "BASE-01", + "pack": "BASE", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "baseline/suppression mechanism present: .audit-baseline.json, .quality-baseline.json, .secrets.baseline", + "tier": "A", + "title": "Baseline artifact exists", + "verdict": "PASS" + }, + { + "data": {}, + "id": "BASE-02", + "pack": "BASE", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "counting surface present but no committed count/budget: .audit-baseline.json, .quality-baseline.json, .secrets.baseline, Makefile:lint, Makefile:security, Makefile:typecheck; method: static configuration scan; the target's own tools are never run", + "tier": "B", + "title": "Current violation counts measured", + "verdict": "PARTIAL" + }, + { + "data": {}, + "id": "BASE-03", + "pack": "BASE", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "ratchet present: baseline compared in CI (.github/workflows/ci.yml)", + "tier": "B", + "title": "No-new-violations mechanism", + "verdict": "PASS" + }, + { + "data": {}, + "id": "BASE-04", + "pack": "BASE", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "coverage threshold configured: .coveragerc", + "tier": "A", + "title": "Coverage floor configured", + "verdict": "PASS" + }, + { + "data": { + "kind": "path_list", + "paths": [ + "Makefile", + "audit/engine/", + "AGENTS.md", + "_typos.toml", + ".python-version", + "dev/tools/pyproject.toml", + "dev/tools/package.json", + "dev/tools/osv/go.mod", + "lefthook.yml", + "dev", + ".github/", + "dev/commit_lint.py", + ".coveragerc", + ".quality-baseline.json", + ".secrets.baseline", + ".skip", + "README.md", + "docs/", + "greenfield/", + "docs/framework/", + "docs/STYLE_GUIDE.md", + "docs/README.md", + "spec/", + "REQ-...", + "spec/README.md", + "audit/", + "SKILL.md", + "checks/", + "prompts/", + "catalogue.json", + "stacks/", + "schema/", + "tools/", + "audit/AGENTS.md", + "audit/engine/AGENTS.md", + "implement/", + "playbooks/", + "state.md", + "progress.py", + "implement/docs/architecture/adrs/", + "implement/AGENTS.md", + "dev/", + "dev/dist/skills/", + "dev/eval/", + "artefacts/", + "docs/architecture/adrs/README.md", + ".env", + "RELEASE_PROCESS.md", + "CHANGELOG.md", + ".release-please-manifest.json", + ".github/rulesets/", + "docs/reference/branch-protection.md", + "audit/tools/", + "implement/tools/", + "audit/engine/tests/", + "audit/tools/tests/", + "implement/tools/tests/", + "dev/tests/" + ], + "truncated": false + }, + "id": "CON-01", + "pack": "CON", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "the prohibitions section of AGENTS.md names paths that exist: Makefile, audit/engine/, AGENTS.md, _typos.toml, .python-version, dev/tools/pyproject.toml, dev/tools/package.json, dev/tools/osv/go.mod (+46 more)", + "tier": "B", + "title": "Allow/deny artifact present", + "verdict": "PASS" + }, + { + "data": { + "kind": "path_list", + "paths": [ + "audit/engine/tests", + "audit/tools/tests", + "dev/tests", + "implement/tools/tests" + ], + "truncated": false + }, + "id": "CON-02", + "pack": "CON", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "the prohibitions section of AGENTS.md names the test directories (audit/engine/tests/, audit/tools/tests/, dev/tests/, implement/tools/tests/)", + "tier": "B", + "title": "Test directories protected", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CON-03", + "pack": "CON", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "artefacts/ present, ignored by artefacts/.gitignore, and untracked", + "tier": "B", + "title": "artefacts/ excluded from index and agent reads", + "verdict": "PASS" + }, + { + "data": {}, + "id": "CON-04", + "pack": "CON", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "draft not emitted; pass --emit-baseline to write do-not-do.draft.md beside the report", + "tier": "\u2014", + "title": "Draft allow/deny emitted", + "verdict": "UNKNOWN" + }, + { + "data": {}, + "id": "EXEC-01", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "unpinned: go", + "tier": "A", + "title": "Toolchain pinned / wrapper committed", + "verdict": "FAIL" + }, + { + "data": {}, + "id": "EXEC-02", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "consistency unverified: dev/tools/osv/go.sum, dev/tools/package-lock.json, dev/tools/uv.lock", + "tier": "B", + "title": "Lockfile committed and consistent", + "verdict": "PARTIAL" + }, + { + "data": {}, + "id": "EXEC-03", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "documented setup command: AGENTS.md: make setup", + "tier": "A", + "title": "Non-interactive setup path", + "verdict": "PASS" + }, + { + "data": { + "kind": "env_keys", + "missing": [], + "referenced": [ + "TMPDIR" + ], + "templates": [] + }, + "id": "EXEC-04", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "1 referenced key(s) declared", + "tier": "B", + "title": "Env-var inventory complete", + "verdict": "PASS" + }, + { + "data": {}, + "id": "EXEC-05", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "no tracked generated or vendored paths", + "tier": "B", + "title": "No generated artifacts committed in-tree", + "verdict": "PASS" + }, + { + "data": {}, + "id": "EXEC-06", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "15 documented command(s) resolve", + "tier": "B", + "title": "Documented commands resolve", + "verdict": "PASS" + }, + { + "data": {}, + "id": "EXEC-07", + "pack": "EXEC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "lint: ruff.toml; types: mypy.ini", + "tier": "B", + "title": "No competing configs for one concern", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TST-01", + "pack": "TST", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "test verb on Makefile (cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests && $(PYTHON) -m unittest discover -s audit/tools/tests && $(PYTHON) -m unittest discover -s implement/tools/tests && $(PYTHON) -m unittest discover -s dev/tests); 85 test file(s)", + "tier": "A", + "title": "Test verb exists, suite non-empty", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TST-02", + "pack": "TST", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "probe not run: probes disabled (pass --run-gates)", + "tier": "C", + "title": "Suite status known", + "verdict": "UNKNOWN" + }, + { + "data": { + "breakdown": { + "python": 1 + }, + "kind": "counter", + "unit": "skip/xfail markers", + "value": 1 + }, + "id": "TST-03", + "pack": "TST", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "skip/xfail census across 85 test file(s): python 1; total 1", + "tier": "B", + "title": "Skip / xfail census", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TST-04", + "pack": "TST", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "fast path documented: AGENTS.md:36 `python3 -m unittest tests.test_cmd_pack -k verbs` (scope: README.md, AGENTS.md, CONTRIBUTING.md)", + "tier": "B", + "title": "Fast path documented", + "verdict": "PASS" + }, + { + "data": {}, + "id": "TST-05", + "pack": "TST", + "phase": 1, + "scored": true, + "severity": "COSMETIC", + "status": "implemented", + "summary": "coverage configured: .coveragerc", + "tier": "A", + "title": "Coverage configuration present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "NAV-01", + "pack": "NAV", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "single-root repository declares package.json", + "tier": "A", + "title": "Declared component boundaries", + "verdict": "PASS" + }, + { + "data": {}, + "id": "NAV-02", + "pack": "NAV", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "entry point(s): audit/engine/audit/__main__.py, audit/engine/build.py, audit/tools/engine.py, audit/tools/facts.py, audit/tools/render_report.py, audit/tools/report.py", + "tier": "B", + "title": "Obvious entry points", + "verdict": "PASS" + }, + { + "data": {}, + "id": "NAV-03", + "pack": "NAV", + "phase": 1, + "scored": true, + "severity": "COSMETIC", + "status": "implemented", + "summary": "oversized(>524288 B): 0; catch-all dirs(>50 files): 0; binaries: 0; minified: 0", + "tier": "B", + "title": "Structural red flags", + "verdict": "PASS" + }, + { + "data": {}, + "id": "NAV-04", + "pack": "NAV", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "docs/architecture/; an ADR directory", + "tier": "A", + "title": "Architecture orientation artifact", + "verdict": "PASS" + }, + { + "data": { + "denominator": 1159, + "kind": "ratio", + "method": "python: module-level def/class not starting with '_'", + "numerator": 462, + "unit": "public symbols" + }, + "id": "NAV-05", + "pack": "NAV", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "462/1159 public symbols documented (40%); method: python: module-level def/class not starting with '_'", + "tier": "B", + "title": "Public-symbol documentation coverage", + "verdict": "PARTIAL" + }, + { + "data": { + "covered": [ + "dotenv", + "dotenv variant" + ], + "kind": "secret_shapes", + "missing": [], + "root_only": [], + "shapes": [ + { + "covered": true, + "label": "dotenv", + "probe": ".env", + "root_only": false, + "suggested": ".env" + }, + { + "covered": true, + "label": "dotenv variant", + "probe": ".env.local", + "root_only": false, + "suggested": ".env.*" + } + ] + }, + "id": "SEC-01", + "pack": "SEC", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "no tracked .env and ignore rules cover the shape", + "tier": "B", + "title": ".env untracked and ignored", + "verdict": "PASS" + }, + { + "data": { + "allowlisted": 2, + "allowlisted_by": [ + ".secrets.baseline" + ], + "files": [], + "kind": "credential_matrix", + "source": "tracked text files", + "suppressed": [ + { + "lines": [ + 38 + ], + "path": "audit/engine/tests/test_report_render.py" + }, + { + "lines": [ + 41 + ], + "path": "audit/engine/tests/test_sec_pack.py" + } + ], + "total": 0 + }, + "id": "SEC-02", + "pack": "SEC", + "phase": 1, + "scored": true, + "severity": "BLOCKER", + "status": "implemented", + "summary": "0 credential-shaped lines outside the allowlist in tracked text files; 2 allowlisted by `.secrets.baseline`", + "tier": "B", + "title": "No key-shaped strings in tracked files", + "verdict": "PASS" + }, + { + "data": { + "covered": [ + "dotenv", + "dotenv variant", + "ssh private key", + "PEM", + "key", + "PKCS#12", + "PFX", + "keystore", + "certificate (.crt)", + "certificate (.cer)", + "AWS credentials" + ], + "kind": "secret_shapes", + "missing": [], + "root_only": [], + "shapes": [ + { + "covered": true, + "label": "dotenv", + "probe": ".env", + "root_only": false, + "suggested": ".env" + }, + { + "covered": true, + "label": "dotenv variant", + "probe": ".env.local", + "root_only": false, + "suggested": ".env.*" + }, + { + "covered": true, + "label": "ssh private key", + "probe": "id_rsa", + "root_only": false, + "suggested": "id_rsa*" + }, + { + "covered": true, + "label": "PEM", + "probe": "server.pem", + "root_only": false, + "suggested": "*.pem" + }, + { + "covered": true, + "label": "key", + "probe": "server.key", + "root_only": false, + "suggested": "*.key" + }, + { + "covered": true, + "label": "PKCS#12", + "probe": "server.p12", + "root_only": false, + "suggested": "*.p12" + }, + { + "covered": true, + "label": "PFX", + "probe": "server.pfx", + "root_only": false, + "suggested": "*.pfx" + }, + { + "covered": true, + "label": "keystore", + "probe": "app.keystore", + "root_only": false, + "suggested": "*.keystore" + }, + { + "covered": true, + "label": "certificate (.crt)", + "probe": "server.crt", + "root_only": false, + "suggested": "*.crt" + }, + { + "covered": true, + "label": "certificate (.cer)", + "probe": "server.cer", + "root_only": false, + "suggested": "*.cer" + }, + { + "covered": true, + "label": "AWS credentials", + "probe": ".aws/credentials", + "root_only": false, + "suggested": ".aws/*" + } + ] + }, + "id": "SEC-03", + "pack": "SEC", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "covered: dotenv, dotenv variant, ssh private key, PEM, key, PKCS#12, PFX, keystore, certificate (.crt), certificate (.cer), AWS credentials", + "tier": "B", + "title": "Ignore patterns cover secret shapes", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-01", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "README.md present and non-placeholder", + "tier": "A", + "title": "README present and non-placeholder", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-02", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "CODEOWNERS present at .github/CODEOWNERS", + "tier": "A", + "title": "CODEOWNERS present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-03", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "pull-request template present at .github/pull_request_template.md", + "tier": "A", + "title": "Pull-request template present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-04", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "2 issue template(s) present", + "tier": "A", + "title": "Issue templates present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-05", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "CONTRIBUTING present at CONTRIBUTING.md", + "tier": "A", + "title": "CONTRIBUTING present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-06", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "SECURITY present at SECURITY.md", + "tier": "A", + "title": "SECURITY present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-07", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "CHANGELOG.md last changed 0 day(s) before HEAD", + "tier": "A", + "title": "CHANGELOG present and recent", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-08", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "release document present at RELEASE_PROCESS.md", + "tier": "A", + "title": "Release process documented", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-09", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "dependency automation configured at .github/dependabot.yml", + "tier": "A", + "title": "Dependency update automation configured", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-10", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "license present at LICENSE", + "tier": "A", + "title": "License present", + "verdict": "PASS" + }, + { + "data": {}, + "id": "HYG-11", + "pack": "HYG", + "phase": 1, + "scored": false, + "severity": "COSMETIC", + "status": "implemented", + "summary": "branch protection declared as code at docs/reference/branch-protection.md", + "tier": "B", + "title": "Branch protection declared as code", + "verdict": "PASS" + } + ], + "components": [ + { + "agents_md": "present", + "declared": true, + "declared_by": "implicit", + "manifest": "package.json", + "name": "(root)", + "path": "." + } + ], + "findings": [ + { + "check": "EXEC-01", + "evidence": [], + "id": "EXEC-01:go", + "phase": 1, + "remediation": "Commit the pin appropriate to go \u2014 .tool-versions \u2014 so the agent's runtime is the repository's, not the host's.", + "severity": "BLOCKER", + "statement": "An agent cannot reproduce the go toolchain because the detected go project pins no toolchain version.", + "verdict": "FAIL" + }, + { + "check": "IDX-01", + "evidence": [ + { + "line": null, + "note": null, + "path": "" + } + ], + "id": "IDX-01", + "phase": 1, + "remediation": "Register the server in a project config (for example `opencode.json`) or a user config (`~/.config/opencode/opencode.json`).", + "severity": "BLOCKER", + "statement": "An agent cannot discover the codebase through an MCP index because no MCP configuration names codebase-memory-mcp for this repository.", + "verdict": "FAIL" + }, + { + "check": "BASE-02", + "evidence": [ + { + "line": null, + "note": null, + "path": ".audit-baseline.json" + }, + { + "line": null, + "note": null, + "path": ".quality-baseline.json" + }, + { + "line": null, + "note": null, + "path": ".secrets.baseline" + }, + { + "line": null, + "note": null, + "path": "Makefile:lint" + }, + { + "line": null, + "note": null, + "path": "Makefile:security" + }, + { + "line": null, + "note": null, + "path": "Makefile:typecheck" + } + ], + "id": "BASE-02", + "phase": 1, + "remediation": "Commit the count or configure a budget (`--max-warnings N`, `max-issues`, or a `violation budget`) so a regression has a baseline number to compare against.", + "severity": "DEGRADER", + "statement": "An agent cannot state how many violations exist today because a counting surface exists but no numeric violation budget or count is committed.", + "verdict": "PARTIAL" + } + ], + "inventory": { + "files_inspected": 342, + "files_set_aside": 463, + "files_skipped": 7, + "set_aside": [ + { + "files": 3, + "path": "audit/engine/examples" + }, + { + "files": 277, + "path": "audit/engine/tests/fixtures" + }, + { + "files": 33, + "path": "audit/schema/examples" + }, + { + "files": 94, + "path": "dev/eval/expected/fixtures" + }, + { + "files": 13, + "path": "dev/eval/implement-fixtures/testdata" + }, + { + "files": 5, + "path": "docs/templates" + }, + { + "files": 38, + "path": "greenfield/templates" + } + ], + "truncated": false + }, + "not_applicable": [], + "probes": [ + { + "command": [ + "cd", + "$(ENGINE)", + "&&", + "$(PYTHON)", + "-m", + "unittest", + "discover", + "-s", + "tests", + "&&", + "$(PYTHON)", + "-m", + "unittest", + "discover", + "-s", + "audit/tools/tests", + "&&", + "$(PYTHON)", + "-m", + "unittest", + "discover", + "-s", + "implement/tools/tests", + "&&", + "$(PYTHON)", + "-m", + "unittest", + "discover", + "-s", + "dev/tests" + ], + "duration_s": 0.0, + "exit_code": null, + "output_tail": "", + "redactions": 0, + "refused_reason": "probes disabled (pass --run-gates)", + "timed_out": false, + "verb": "test" + } + ], + "provenance": { + "framework_revision": "801faf42cf1104512148020a8195f7138d5b55ac", + "git_branch": "chore/v0.3.3-track-a-tooling", + "git_dirty": false, + "git_sha": "801faf42cf1104512148020a8195f7138d5b55ac", + "phase": 1, + "probes_enabled": false, + "ruleset_hash": "04e5d0ff9063e012", + "ruleset_revision": "2026-11-09.1", + "scanned_at": "2026-11-09T06:56:35+00:00", + "target": ".", + "tool": "python-agentic-audit", + "user_config": { + "paths_read": [], + "read_allowed": false + }, + "version": "0.3.2" + }, + "schema_version": "2", + "stack": { + "ci_providers": [ + "github" + ], + "ecosystems": [ + "go", + "node", + "python" + ], + "hook_managers": [ + "lefthook" + ], + "notes": [], + "package_managers": [ + "gomod", + "npm", + "uv" + ], + "test_frameworks": [] + }, + "summary": { + "blockers": 2, + "checks": { + "attest": 0, + "fail": 2, + "partial": 3, + "pass": 61, + "unknown": 4 + }, + "checks_applicable": 70, + "checks_implemented": 70, + "checks_scoreable": 56, + "cosmetic": 0, + "coverage_note": "56 of 56 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", + "degraders": 1, + "phase1_score": 0.875 + }, + "unattested": [ + "index-in-use", + "baseline-adequacy", + "suite-trustworthiness", + "doc-accuracy", + "gate-honoured" + ] +} diff --git a/docs/reference/self-audit.md b/docs/reference/self-audit.md index 0a551499..382a8e1a 100644 --- a/docs/reference/self-audit.md +++ b/docs/reference/self-audit.md @@ -1,7 +1,7 @@ # Self-Audit > **Audience:** Everyone -> **Source of truth:** [`schema/examples/self-audit/2026-11-06.json`](../../audit/schema/examples/self-audit/2026-11-06.json) +> **Source of truth:** [`schema/examples/self-audit/2026-11-09.json`](../../audit/schema/examples/self-audit/2026-11-09.json) > (the engine's report for this repository) and the root [`.audit-baseline.json`](../../.audit-baseline.json) > (the same report, kept as the CI ratchet). This page restates the report; the report wins. > **Status:** Live. Regenerated by hand when the baseline changes (see [Keep it true](#keep-it-true)). @@ -16,16 +16,19 @@ score from falling. | Field | Value | | --- | --- | -| Phase-1 score | **0.8402** | -| Date | 2026-11-06 | -| Engine | `python-agentic-audit` 0.1.0 | -| Ruleset | `2026-11-06.1` (hash `970e6792f98eeeb5`) | -| Applicable checks | 69 (55 scoreable) | -| Findings | 7: 2 BLOCKER, 3 DEGRADER, 2 COSMETIC | +| Phase-1 score | **0.875** | +| Date | 2026-11-09 | +| Engine | `python-agentic-audit` 0.3.2 | +| Ruleset | `2026-11-09.1` (hash `04e5d0ff9063e012`) | +| Applicable checks | 70 (56 scoreable) | +| Findings | 3: 2 BLOCKER, 1 DEGRADER | | Scope | the whole repository except `artefacts/` and `dist/`; no user-level configuration read | -`HYG-05`, `HYG-07`, `HYG-08` and `HYG-09` closed with `CONTRIBUTING.md`, the root changelog, `RELEASE_PROCESS.md` and -`.github/dependabot.yml` (MR08); the score did not move because those checks are cosmetic. +Since the 2026-11-06 report (0.8402): `HYG-02` closed with `.github/CODEOWNERS` and `HYG-06` with the security policy, +`SECURITY.md`. On 2026-11-09, `TOOL-04` closed with the dependency audit in +`make security` ([ADR 0006](../architecture/adrs/0006-security-verb-audits-dependencies-with-osv-scanner.md)), and +`SEC-01` and `SEC-03` closed when the `.env` rules in `.gitignore` began to match at any depth. `EXEC-01` opened as +a false positive (below). The new check `CI-05` (workflow files are linted) passes through `make workflow-lint`. The score was 0.6762 on 2026-11-05, before the repository adopted the Phase 1 requirements it asks of others (see the [changelog](../../audit/CHANGELOG.md) and the engine's @@ -33,14 +36,14 @@ The score was 0.6762 on 2026-11-05, before the repository adopted the Phase 1 re | Verdict | Checks | | --- | --: | -| PASS | 55 | -| PARTIAL | 5 | -| FAIL | 3 | -| UNKNOWN | 6 | +| PASS | 61 | +| PARTIAL | 3 | +| FAIL | 2 | +| UNKNOWN | 4 | | ATTEST | 0 | -| **Total** | **69** | +| **Total** | **70** | -`UNKNOWN` and `ATTEST` earn no credit. Six checks are `UNKNOWN` because their evidence needs a Tier-C probe +`UNKNOWN` and `ATTEST` earn no credit. Four checks are `UNKNOWN` because their evidence needs a Tier-C probe (`--run-gates`, not used here) or the user-level configuration that `--no-user-config` leaves unread, so this score understates the repository rather than certifying it. @@ -51,18 +54,16 @@ Every check that is not `PASS`, with the reason it is open. | Check | Verdict | Severity | Why it is open | | --- | --- | --- | --- | | `IDX-01` | FAIL | BLOCKER | **Owner exception** (D21): no project `.mcp.json`. The index is optional here, and a project file would make every contributor's harness offer to run a binary. Review in v0.3 | -| `SEC-01` | PARTIAL | BLOCKER | `/.env` is root-anchored on purpose: the engine's fixtures hold `.env` files as data and must stay committable (comment in `.gitignore`). The check reads a root-only rule as incomplete | -| `SEC-03` | PARTIAL | DEGRADER | Same cause as `SEC-01` for the dotenv shapes; the key, certificate and keystore shapes are covered | | `BASE-02` | PARTIAL | DEGRADER | The T5 playbook was applied (D32): `.quality-baseline.json` holds the counts (ruff 0, mypy 0, Markdown lint outside the ratchet 735) and `make format-check`, `make typecheck` and `make docs-lint` fail when one goes up. The check still reads PARTIAL by design: it credits only a numeric budget flag (`--max-warnings N`), and none of these tools has one. Engine issue [#45](https://github.com/Nistapp/codebase-agentic-readiness-framework/issues/45) tracks it; the coverage floor (91) is unchanged and `BASE-04` passes | | `EXEC-01` | FAIL | BLOCKER | **Owner exception** (2026-11-09): a false positive. osv-scanner's Go module (`dev/tools/osv/go.mod`) pins Go with its `go` directive, but the check reads that directive only from a root `go.mod`. An engine fix is planned (v0.3.3 Track C) | -| `HYG-02`, `HYG-06` | FAIL | COSMETIC | Need values only the owner has: `CODEOWNERS` names and a private disclosure contact for `SECURITY.md`. Stay open until supplied (D22) | -| `NAV-05` | PARTIAL | COSMETIC | Informational, no work (D34): 44% of public symbols documented (638 of 1438) | +| `EXEC-02` | PARTIAL | BLOCKER | No finding: the lockfiles (`dev/tools/uv.lock`, `dev/tools/package-lock.json`, `dev/tools/osv/go.sum`) are committed, but the engine does not run the tools, so it cannot confirm that they match their manifests. `dev/tests/test_tool_pins.py` checks that in `make test` | +| `NAV-05` | PARTIAL | COSMETIC | No finding, informational and unscored (D34): 40% of public symbols documented (460 of 1155). Reviewed on 2026-11-09 and left as it is: a ruff `D1` ratchet would baseline about 700 violations for a cosmetic check, and mypy strict and the tests carry the meaning | | `IDX-02`, `IDX-03`, `TST-02`, `CON-04` | UNKNOWN | mixed | Need a probe, user-level configuration, or `--emit-baseline`; not run here | -| `EXEC-02`, `NAV-01` | UNKNOWN | mixed | The repository declares no dependencies and no components, so there is nothing for the check to read | Recorded decisions that keep a finding open or exempt: `IDX-01` and `EXEC-01` are owner exceptions (D21; 2026-11-09); `AGT-08` stays as it is for v0.2, because a stated harness list closes the gap even when a named harness does not read -`AGENTS.md` natively, and is revisited in v0.3 (D31); `NAV-05` is informational and unscored (D34). The engine has no +`AGENTS.md` natively, and is revisited in v0.3 (D31); `NAV-05` is informational and unscored (D34; reviewed again +2026-11-09). The engine has no way to record an owner exception, so this page is the record. Everything else is either a deliberate design choice stated in the repository, a value waiting for the owner, planned work or open debt; none is hidden in the score. From 003c6e03e2c4279bf7b9629f7e5d4efc1008af56 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 12:30:48 +0530 Subject: [PATCH 026/111] chore: refresh the self-audit baseline for ruleset 2026-11-09.1 The merge from dev brought CI-05 (workflow files are linted). It passes here, so the self-audit now reads 70 checks and a Phase-1 score of 0.879. The baseline, the published example and docs/reference/self-audit.md move to the new ruleset; the findings are unchanged. Roadmap-Step: B4 Co-Authored-By: Claude Opus 5.5 --- .audit-baseline.json | 49 ++++++++++++------- .secrets.baseline | 18 +++---- .../examples/self-audit/2026-11-09.json | 49 ++++++++++++------- docs/reference/self-audit.md | 10 ++-- 4 files changed, 76 insertions(+), 50 deletions(-) diff --git a/.audit-baseline.json b/.audit-baseline.json index b07bad98..3aa96f6a 100644 --- a/.audit-baseline.json +++ b/.audit-baseline.json @@ -523,6 +523,19 @@ "title": "CI-only steps flagged", "verdict": "PASS" }, + { + "data": {}, + "id": "CI-05", + "pack": "CI", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "github: local actionlint (Makefile), actionlint (dev/tools/pyproject.toml), check-jsonschema (Makefile), zizmor (Makefile), zizmor (dev/tools/pyproject.toml), CI none", + "tier": "B", + "title": "Workflow files are linted", + "verdict": "PASS" + }, { "data": {}, "id": "BASE-01", @@ -921,10 +934,10 @@ }, { "data": { - "denominator": 1178, + "denominator": 1182, "kind": "ratio", "method": "python: module-level def/class not starting with '_'", - "numerator": 470, + "numerator": 472, "unit": "public symbols" }, "id": "NAV-05", @@ -933,7 +946,7 @@ "scored": false, "severity": "COSMETIC", "status": "implemented", - "summary": "470/1178 public symbols documented (40%); method: python: module-level def/class not starting with '_'", + "summary": "472/1182 public symbols documented (40%); method: python: module-level def/class not starting with '_'", "tier": "B", "title": "Public-symbol documentation coverage", "verdict": "PARTIAL" @@ -1459,7 +1472,7 @@ ], "inventory": { "files_inspected": 342, - "files_set_aside": 461, + "files_set_aside": 467, "files_skipped": 3, "set_aside": [ { @@ -1467,7 +1480,7 @@ "path": "audit/engine/examples" }, { - "files": 276, + "files": 280, "path": "audit/engine/tests/fixtures" }, { @@ -1475,7 +1488,7 @@ "path": "audit/schema/examples" }, { - "files": 92, + "files": 94, "path": "dev/eval/expected/fixtures" }, { @@ -1538,15 +1551,15 @@ } ], "provenance": { - "framework_revision": "98a13e1b84271c38f8b41c79d67eefa5e0f1f45d", + "framework_revision": "e0693eb82c2b975e579e53ceff19849da09ceccc", "git_branch": "feat/v0.3.3-track-b-mcp", - "git_dirty": true, - "git_sha": "98a13e1b84271c38f8b41c79d67eefa5e0f1f45d", + "git_dirty": false, + "git_sha": "e0693eb82c2b975e579e53ceff19849da09ceccc", "phase": 1, "probes_enabled": false, - "ruleset_hash": "970e6792f98eeeb5", - "ruleset_revision": "2026-11-06.1", - "scanned_at": "2026-11-09T06:47:54+00:00", + "ruleset_hash": "04e5d0ff9063e012", + "ruleset_revision": "2026-11-09.1", + "scanned_at": "2026-11-09T06:52:56+00:00", "target": ".", "tool": "python-agentic-audit", "user_config": { @@ -1580,16 +1593,16 @@ "attest": 0, "fail": 1, "partial": 6, - "pass": 58, + "pass": 59, "unknown": 4 }, - "checks_applicable": 69, - "checks_implemented": 69, - "checks_scoreable": 55, + "checks_applicable": 70, + "checks_implemented": 70, + "checks_scoreable": 56, "cosmetic": 0, - "coverage_note": "55 of 55 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", + "coverage_note": "56 of 56 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", "degraders": 4, - "phase1_score": 0.877 + "phase1_score": 0.879 }, "unattested": [ "index-in-use", diff --git a/.secrets.baseline b/.secrets.baseline index f87ab764..0c474b27 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -131,16 +131,16 @@ { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", - "hashed_secret": "70e5aa2e9690505b775c6a71c8d9969d78fc0b4a", + "hashed_secret": "43c01a9474d2a59e421a95a0ced74d6d055afe98", "is_verified": false, - "line_number": 1541 + "line_number": 1554 }, { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", - "hashed_secret": "228ea6c3984c8d1226e0ffc33455f16dce652d01", + "hashed_secret": "78d6ecebdb129eed66e4824377d8ca833d9b3932", "is_verified": false, - "line_number": 1547 + "line_number": 1560 } ], "audit/engine/audit/rules/checks/con.py": [ @@ -442,16 +442,16 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/self-audit/2026-11-09.json", - "hashed_secret": "70e5aa2e9690505b775c6a71c8d9969d78fc0b4a", + "hashed_secret": "43c01a9474d2a59e421a95a0ced74d6d055afe98", "is_verified": false, - "line_number": 1541 + "line_number": 1554 }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/self-audit/2026-11-09.json", - "hashed_secret": "228ea6c3984c8d1226e0ffc33455f16dce652d01", + "hashed_secret": "78d6ecebdb129eed66e4824377d8ca833d9b3932", "is_verified": false, - "line_number": 1547 + "line_number": 1560 } ], "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json": [ @@ -1474,5 +1474,5 @@ } ] }, - "generated_at": "2026-11-09T06:52:02Z" + "generated_at": "2026-11-09T06:53:01Z" } diff --git a/audit/schema/examples/self-audit/2026-11-09.json b/audit/schema/examples/self-audit/2026-11-09.json index b07bad98..3aa96f6a 100644 --- a/audit/schema/examples/self-audit/2026-11-09.json +++ b/audit/schema/examples/self-audit/2026-11-09.json @@ -523,6 +523,19 @@ "title": "CI-only steps flagged", "verdict": "PASS" }, + { + "data": {}, + "id": "CI-05", + "pack": "CI", + "phase": 1, + "scored": true, + "severity": "DEGRADER", + "status": "implemented", + "summary": "github: local actionlint (Makefile), actionlint (dev/tools/pyproject.toml), check-jsonschema (Makefile), zizmor (Makefile), zizmor (dev/tools/pyproject.toml), CI none", + "tier": "B", + "title": "Workflow files are linted", + "verdict": "PASS" + }, { "data": {}, "id": "BASE-01", @@ -921,10 +934,10 @@ }, { "data": { - "denominator": 1178, + "denominator": 1182, "kind": "ratio", "method": "python: module-level def/class not starting with '_'", - "numerator": 470, + "numerator": 472, "unit": "public symbols" }, "id": "NAV-05", @@ -933,7 +946,7 @@ "scored": false, "severity": "COSMETIC", "status": "implemented", - "summary": "470/1178 public symbols documented (40%); method: python: module-level def/class not starting with '_'", + "summary": "472/1182 public symbols documented (40%); method: python: module-level def/class not starting with '_'", "tier": "B", "title": "Public-symbol documentation coverage", "verdict": "PARTIAL" @@ -1459,7 +1472,7 @@ ], "inventory": { "files_inspected": 342, - "files_set_aside": 461, + "files_set_aside": 467, "files_skipped": 3, "set_aside": [ { @@ -1467,7 +1480,7 @@ "path": "audit/engine/examples" }, { - "files": 276, + "files": 280, "path": "audit/engine/tests/fixtures" }, { @@ -1475,7 +1488,7 @@ "path": "audit/schema/examples" }, { - "files": 92, + "files": 94, "path": "dev/eval/expected/fixtures" }, { @@ -1538,15 +1551,15 @@ } ], "provenance": { - "framework_revision": "98a13e1b84271c38f8b41c79d67eefa5e0f1f45d", + "framework_revision": "e0693eb82c2b975e579e53ceff19849da09ceccc", "git_branch": "feat/v0.3.3-track-b-mcp", - "git_dirty": true, - "git_sha": "98a13e1b84271c38f8b41c79d67eefa5e0f1f45d", + "git_dirty": false, + "git_sha": "e0693eb82c2b975e579e53ceff19849da09ceccc", "phase": 1, "probes_enabled": false, - "ruleset_hash": "970e6792f98eeeb5", - "ruleset_revision": "2026-11-06.1", - "scanned_at": "2026-11-09T06:47:54+00:00", + "ruleset_hash": "04e5d0ff9063e012", + "ruleset_revision": "2026-11-09.1", + "scanned_at": "2026-11-09T06:52:56+00:00", "target": ".", "tool": "python-agentic-audit", "user_config": { @@ -1580,16 +1593,16 @@ "attest": 0, "fail": 1, "partial": 6, - "pass": 58, + "pass": 59, "unknown": 4 }, - "checks_applicable": 69, - "checks_implemented": 69, - "checks_scoreable": 55, + "checks_applicable": 70, + "checks_implemented": 70, + "checks_scoreable": 56, "cosmetic": 0, - "coverage_note": "55 of 55 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", + "coverage_note": "56 of 56 scoreable checks are implemented in this ruleset revision; every other check reports UNKNOWN and earns no credit, so this report understates readiness rather than certifying it", "degraders": 4, - "phase1_score": 0.877 + "phase1_score": 0.879 }, "unattested": [ "index-in-use", diff --git a/docs/reference/self-audit.md b/docs/reference/self-audit.md index af108c1e..b6a05776 100644 --- a/docs/reference/self-audit.md +++ b/docs/reference/self-audit.md @@ -16,11 +16,11 @@ score from falling. | Field | Value | | --- | --- | -| Phase-1 score | **0.8770** | +| Phase-1 score | **0.8790** | | Date | 2026-11-09 | | Engine | `python-agentic-audit` 0.3.2 | -| Ruleset | `2026-11-06.1` (hash `970e6792f98eeeb5`) | -| Applicable checks | 69 (55 scoreable) | +| Ruleset | `2026-11-09.1` (hash `04e5d0ff9063e012`) | +| Applicable checks | 70 (56 scoreable) | | Findings | 5: 1 BLOCKER, 4 DEGRADER | | Scope | the whole repository except `artefacts/` and `dist/`; no user-level configuration read | @@ -33,12 +33,12 @@ The score was 0.6762 on 2026-11-05, before the repository adopted the Phase 1 re | Verdict | Checks | | --- | --: | -| PASS | 58 | +| PASS | 59 | | PARTIAL | 6 | | FAIL | 1 | | UNKNOWN | 4 | | ATTEST | 0 | -| **Total** | **69** | +| **Total** | **70** | `UNKNOWN` and `ATTEST` earn no credit. Four checks are `UNKNOWN` because their evidence needs a Tier-C probe (`--run-gates`, not used here) or the user-level configuration that `--no-user-config` leaves unread, so this score From c03533162ec10b77f544b96245a22fe31e2092d8 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 13:12:24 +0530 Subject: [PATCH 027/111] feat(engine): HYG-09 reads ecosystem coverage and the target-branch trap HYG-09 only checked that a Dependabot or Renovate file existed. It now reads the file for two defects that hold whatever update policy the owner chose. Coverage: each ecosystem a manifest names, plus github-actions with GitHub CI, needs an entry (Renovate covers all unless enabledManagers leaves one out). The trap: a Dependabot entry that sets target-branch with ignore or groups, whose options then do not reach security updates. Either is PARTIAL; an unreadable file is UNKNOWN. The policy itself (security only, majors, cooldown) is never a verdict. HYG-09 stays COSMETIC and unscored. Ruleset 2026-11-09.2. Two one-defect fixtures with eval expectations; the presence test now uses a minimal valid config. Rubric, engine catalogue page and changelog updated. Self-audit: HYG-09 PASS. Roadmap-Step: C2 Co-Authored-By: Claude Opus 5.5 --- .secrets.baseline | 20 +- audit/checks/13-hygiene-informational.md | 45 +++- audit/engine/audit/__init__.py | 2 +- audit/engine/audit/rules/checks/hyg.py | 228 ++++++++++++++++-- .../02-check-catalogue.md | 2 +- audit/engine/docs/reference/changelog.md | 1 + audit/engine/tests/fixtures/README.md | 2 + .../.github/dependabot.yml | 11 + .../hyg-09-target-branch-trap/package.json | 5 + .../.github/dependabot.yml | 7 + .../hyg-09-uncovered-ecosystem/package.json | 5 + audit/engine/tests/test_fixture_matrix.py | 2 + audit/engine/tests/test_hyg09_policy.py | 130 ++++++++++ audit/engine/tests/test_hyg_pack.py | 2 + audit/engine/tests/test_hygiene_presence.py | 2 +- .../fixtures/hyg-09-target-branch-trap.json | 14 ++ .../fixtures/hyg-09-uncovered-ecosystem.json | 14 ++ 17 files changed, 460 insertions(+), 32 deletions(-) create mode 100644 audit/engine/tests/fixtures/hyg-09-target-branch-trap/.github/dependabot.yml create mode 100644 audit/engine/tests/fixtures/hyg-09-target-branch-trap/package.json create mode 100644 audit/engine/tests/fixtures/hyg-09-uncovered-ecosystem/.github/dependabot.yml create mode 100644 audit/engine/tests/fixtures/hyg-09-uncovered-ecosystem/package.json create mode 100644 audit/engine/tests/test_hyg09_policy.py create mode 100644 dev/eval/expected/fixtures/hyg-09-target-branch-trap.json create mode 100644 dev/eval/expected/fixtures/hyg-09-uncovered-ecosystem.json diff --git a/.secrets.baseline b/.secrets.baseline index ae38561d..262edb05 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1159,6 +1159,24 @@ "line_number": 4 } ], + "dev/eval/expected/fixtures/hyg-09-target-branch-trap.json": [ + { + "type": "Hex High Entropy String", + "filename": "dev/eval/expected/fixtures/hyg-09-target-branch-trap.json", + "hashed_secret": "96adac10ca0ce3d5a63f77d51c63aab87a6a5cb2", + "is_verified": false, + "line_number": 4 + } + ], + "dev/eval/expected/fixtures/hyg-09-uncovered-ecosystem.json": [ + { + "type": "Hex High Entropy String", + "filename": "dev/eval/expected/fixtures/hyg-09-uncovered-ecosystem.json", + "hashed_secret": "a252014b20b554444d5737385a8d18d6c7dfe1c4", + "is_verified": false, + "line_number": 4 + } + ], "dev/eval/expected/fixtures/hyg-10-no-license.json": [ { "type": "Hex High Entropy String", @@ -1458,5 +1476,5 @@ } ] }, - "generated_at": "2026-11-09T05:58:50Z" + "generated_at": "2026-11-09T07:42:18Z" } diff --git a/audit/checks/13-hygiene-informational.md b/audit/checks/13-hygiene-informational.md index 82135d58..7e3e0904 100644 --- a/audit/checks/13-hygiene-informational.md +++ b/audit/checks/13-hygiene-informational.md @@ -244,27 +244,46 @@ Pitfalls A repository automating its release entirely through `.github/w ## HYG-09 — Dependency update automation configured -Why it matters Dependencies should be patched on a schedule, not discovered stale during an incident. +Why it matters Dependencies should be patched on a schedule, not discovered stale during an incident. A + configuration that silently misses an ecosystem, or whose rules do not reach security + updates, looks like a policy and is not one. Severity COSMETIC Engine `HYG-09`. Dependabot (`.github/dependabot.yml`/`.yaml`, or the same names at the root) or Renovate configuration (`renovate.json[5]`, `.renovaterc[.json]`, - `.github/renovate.json`, `renovate.config.js`) at any of its conventional locations is - `PASS`. None found is `FAIL`. + `.github/renovate.json`, `renovate.config.js`) at any of its conventional locations. + The engine reads it for two defects that hold whatever policy the owner chose: + (1) coverage: each ecosystem a manifest names, plus `github-actions` when GitHub CI is + detected, has an entry (Node `npm`/`bun`, Python `pip`/`uv`, JVM `maven`/`gradle`, Go + `gomod`, Rust `cargo`, .NET `nuget`, Ruby `bundler`, PHP `composer`); Renovate covers + every manager unless `enabledManagers` leaves one out; (2) the `target-branch` trap: a + Dependabot entry that sets `target-branch` together with `ignore` or `groups`. Method automated -Look for The fixed set of conventional Dependabot/Renovate configuration filenames, each at its - documented location. -Rubric PASS: a Dependabot or Renovate configuration file exists at a conventional location. - PARTIAL: not used by this check. +Look for The fixed set of conventional filenames; in a Dependabot file, each `updates:` item's + `package-ecosystem`, `target-branch`, `ignore` and `groups`; in a Renovate JSON file, + `enabledManagers`. +Rubric PASS: a configuration exists, every detected ecosystem has an entry, and no entry has + the trap. + PARTIAL: a configuration exists, but a detected ecosystem has no entry, or an entry sets + `target-branch` together with `ignore` or `groups`. FAIL: no file from the fixed configuration set exists at any conventional location. NOT_APPLICABLE: never. - UNKNOWN when: never. -Evidence to cite A `file` evidence item citing the configuration path found, for PASS. + UNKNOWN when: every configuration found is unreadable (a Dependabot file with no + `updates:` list, invalid Renovate JSON), or an ecosystem is uncovered and one of the + files could not be read. `renovate.json5` and `renovate.config.js` count on presence. +Evidence to cite A `file` evidence item citing the configuration path, and for the trap the line of + `target-branch`. Absence proof A FAIL states that none of the fixed Dependabot/Renovate filenames were found at any of - their conventional locations. + their conventional locations. A coverage PARTIAL names each uncovered ecosystem and the + `package-ecosystem` values that would cover it. Remediation `add-repo-hygiene` -Pitfalls A third-party dependency-update bot configured entirely through an external SaaS - dashboard, with no file committed to the repository, is invisible to this check and - reads `FAIL` — the engine can only see what is committed. +Pitfalls The update policy (security pull requests only, majors advisory, a cooldown) is the + owner's choice and never a verdict; do not mark a repository down for weekly version + updates. `target-branch` matters because Dependabot drops an entry's options for + security updates when it is set; security pull requests always go to the default + branch. Coverage is per ecosystem, not per directory: a monorepo with one `npm` entry + for `/` passes even if a nested `package.json` has no entry. A bot configured entirely + through an external dashboard, with no committed file, reads `FAIL` — the engine can + only see what is committed. ## HYG-10 — License present diff --git a/audit/engine/audit/__init__.py b/audit/engine/audit/__init__.py index 3ffc8709..c1690ecc 100644 --- a/audit/engine/audit/__init__.py +++ b/audit/engine/audit/__init__.py @@ -24,7 +24,7 @@ #: Bumped whenever a rule pack, a check, or the instruction-variant table changes. #: Recorded in every report's provenance block so a report can be attributed to an #: exact catalogue revision (docs: Scan Engine & Tiering, § 7 Determinism). -__ruleset_revision__ = "2026-11-09.1" +__ruleset_revision__ = "2026-11-09.2" #: Highest framework phase this build evaluates. Phase 1 only in v1. __phase_ceiling__ = 1 diff --git a/audit/engine/audit/rules/checks/hyg.py b/audit/engine/audit/rules/checks/hyg.py index 89ea7d36..e9fa89f8 100644 --- a/audit/engine/audit/rules/checks/hyg.py +++ b/audit/engine/audit/rules/checks/hyg.py @@ -32,7 +32,15 @@ * **HYG-08** — a text/markdown document whose filename contains "release" (a release process or release-notes page), at the same places. CI workflow files named `release.yml` are not documents and do not count. -* **HYG-09** — Dependabot or Renovate configuration at any of its conventional locations. +* **HYG-09** — Dependabot or Renovate configuration at any of its conventional locations, read for two defects + that hold whatever update policy the owner chose. (1) **Coverage:** every ecosystem the stack names from a + manifest, plus `github-actions` when GitHub CI is detected, has a Dependabot entry of an accepted + `package-ecosystem` (:data:`_DEPENDABOT_ECOSYSTEMS`); Renovate covers every manager unless `enabledManagers` + leaves one out (:data:`_RENOVATE_MANAGERS`). (2) **The `target-branch` trap:** a Dependabot entry that sets + `target-branch` together with `ignore` or `groups`, whose options then do not apply to security updates. Either + defect is `PARTIAL`. A configuration whose content cannot be read (no `updates:` list, invalid JSON) is + ``UNKNOWN`` when it is the only one or when an ecosystem is uncovered; `renovate.json5` and `renovate.config.js` + count on presence. Coverage is per ecosystem, not per directory. * **HYG-10** — a root license file: `LICENSE*`, `LICENCE*`, `COPYING*`, `COPYRIGHT*` or `UNLICENSE*` (case-insensitive, bare name or document suffix). * **HYG-11** — branch protection declared as code, where the provider allows it. A @@ -52,12 +60,17 @@ from __future__ import annotations import datetime as dt +import json +import re import subprocess +from dataclasses import dataclass, field from pathlib import Path from audit.evaluate import CheckOutcome from audit.findings import Evidence, Finding, Verdict, statement +from audit.rules.checks._common import _indent from audit.scan import Inventory +from audit.stack import Stack from audit.rules.payloads import Payload #: A README below this many characters of trimmed text is present but not documentation. @@ -88,6 +101,39 @@ "renovate.config.js", ) +#: Renovate files the engine can parse with the standard library; the others count on presence. +_RENOVATE_JSON = frozenset({"renovate.json", ".renovaterc", ".renovaterc.json"}) +_RENOVATE_UNPARSED = frozenset({"renovate.json5", "renovate.config.js"}) + +#: Stack ecosystem (or ``github`` for the CI provider) -> the Dependabot ``package-ecosystem`` values that cover it. +_DEPENDABOT_ECOSYSTEMS: dict[str, tuple[str, ...]] = { + "node": ("npm", "bun"), + "python": ("pip", "uv"), + "jvm": ("maven", "gradle"), + "go": ("gomod",), + "rust": ("cargo",), + "dotnet": ("nuget",), + "ruby": ("bundler",), + "php": ("composer",), + "github": ("github-actions",), +} + +#: The same, as Renovate manager names for ``enabledManagers``. +_RENOVATE_MANAGERS: dict[str, tuple[str, ...]] = { + "node": ("npm", "bun"), + "python": ("pep621", "pip_requirements", "pip_setup", "pip-compile", "pipenv", "poetry", "setuptools"), + "jvm": ("maven", "gradle"), + "go": ("gomod",), + "rust": ("cargo",), + "dotnet": ("nuget",), + "ruby": ("bundler",), + "php": ("composer",), + "github": ("github-actions",), +} + +#: The Dependabot entry options that the owner expects to bind security updates, and that ``target-branch`` drops. +_TRAPPED_OPTIONS = frozenset({"ignore", "groups"}) + def _outcome( spec, verdict: Verdict, summary: str = "", findings: list[Finding] | None = None, data: Payload | None = None @@ -488,24 +534,176 @@ def check_hyg08(*, spec, target, inventory, stack, components, session) -> Check # =========================================================================== +@dataclass +class _UpdateEntry: + """One item of a Dependabot ``updates:`` list: its ecosystem, its option keys and their line numbers.""" + + line: int + ecosystem: str | None = None + keys: dict[str, int] = field(default_factory=dict) + + +@dataclass +class _AutomationConfig: + """What one dependency-update file tells the check. ``covers`` is ``None`` when it covers every ecosystem.""" + + path: str + readable: bool = True + covers: set[str] | None = None + traps: list[_UpdateEntry] = field(default_factory=list) + + +def _scalar(value: str) -> str: + value = re.sub(r"\s+#.*$", "", value).strip() + return value.strip("\"'") + + +def _dependabot_entries(text: str) -> list[_UpdateEntry] | None: + """Minimal reader for the ``updates:`` list of a Dependabot file; ``None`` when there is no such list. + + No YAML library is available (stdlib only), so this understands block style: ``- key: value`` items and + their sibling keys. Anchors, flow-style items and multi-document files are not interpreted. + """ + lines = text.splitlines() + try: + start = next(i for i, line in enumerate(lines) if re.match(r"^updates:\s*(#.*)?$", line)) + except StopIteration: + return None + entries: list[_UpdateEntry] = [] + item_indent: int | None = None + key_indent = 0 + for number, line in enumerate(lines[start + 1 :], start=start + 2): + if not line.strip() or line.lstrip().startswith("#"): + continue + indent = _indent(line) + stripped = line.strip() + if stripped.startswith("- ") and (item_indent is None or indent == item_indent): + item_indent = indent + key_indent = indent + 2 + entries.append(_UpdateEntry(line=number)) + stripped = stripped[2:].strip() + indent = key_indent + elif item_indent is None or indent <= item_indent: + break + key = re.match(r"^([A-Za-z0-9_-]+):(.*)$", stripped) + if entries and key and indent == key_indent: + name, value = key.group(1), key.group(2) + entries[-1].keys.setdefault(name, number) + if name == "package-ecosystem": + entries[-1].ecosystem = _scalar(value) + return entries + + +def _read_dependabot(rel: str, text: str | None) -> _AutomationConfig: + entries = _dependabot_entries(text or "") + if entries is None: + return _AutomationConfig(rel, readable=False, covers=set()) + named = {entry.ecosystem for entry in entries if entry.ecosystem} + covers = {eco for eco, accepted in _DEPENDABOT_ECOSYSTEMS.items() if named & set(accepted)} + traps = [entry for entry in entries if "target-branch" in entry.keys and _TRAPPED_OPTIONS & set(entry.keys)] + return _AutomationConfig(rel, covers=covers, traps=traps) + + +def _read_renovate(rel: str, text: str | None) -> _AutomationConfig: + if _basename(rel) in _RENOVATE_UNPARSED: + return _AutomationConfig(rel) + try: + config = json.loads(text or "") + except ValueError: + return _AutomationConfig(rel, readable=False, covers=set()) + if not isinstance(config, dict): + return _AutomationConfig(rel, readable=False, covers=set()) + enabled = config.get("enabledManagers") + if not isinstance(enabled, list): + return _AutomationConfig(rel) + names = {str(name) for name in enabled} + return _AutomationConfig(rel, covers={eco for eco, accepted in _RENOVATE_MANAGERS.items() if names & set(accepted)}) + + +def _required_coverage(stack: Stack) -> set[str]: + """The ecosystems a dependency-update policy must reach: those a manifest names, plus GitHub Actions.""" + required = {eco for eco in stack.ecosystems - stack.source_only if eco in _DEPENDABOT_ECOSYSTEMS} + if "github" in stack.ci_providers: + required.add("github") + return required + + +def _coverage_label(eco: str) -> str: + return "github-actions" if eco == "github" else eco + + def check_hyg09(*, spec, target, inventory, stack, components, session) -> CheckOutcome: found = _present(inventory, *_DEPENDENCY_AUTOMATION) - if found: - return _outcome(spec, Verdict.PASS, f"dependency automation configured at {found[0]}") - return _outcome( - spec, - Verdict.FAIL, - "no dependency update automation", - [ + if not found: + return _outcome( + spec, + Verdict.FAIL, + "no dependency update automation", + [ + _finding( + spec, + Verdict.FAIL, + "keep dependencies patched without manual sweeps", + "no Dependabot or Renovate configuration exists", + remediation="Add .github/dependabot.yml (or renovate.json) with an update schedule.", + ), + ], + ) + + configs = [ + _read_renovate(rel, inventory.read(rel)) if "renovate" in rel else _read_dependabot(rel, inventory.read(rel)) + for rel in found + ] + required = _required_coverage(stack) + covered: set[str] = set() + for config in configs: + covered |= required if config.covers is None else config.covers + missing = sorted(required - covered) + unreadable = [config.path for config in configs if not config.readable] + + findings = [ + _finding( + spec, + Verdict.PARTIAL, + "rely on the update policy reaching security updates", + f"the `{entry.ecosystem or 'unnamed'}` entry sets `target-branch` together with " + f"`{'`, `'.join(sorted(_TRAPPED_OPTIONS & set(entry.keys)))}`, and Dependabot drops those options for " + "security updates, which always go to the default branch", + evidence=[Evidence(config.path, line=entry.keys["target-branch"], note="target-branch")], + path=f"{config.path}:{entry.keys['target-branch']}", + remediation="Remove `target-branch` when it names the default branch; otherwise make that branch the " + "default, so the entry's options apply to security updates too.", + ) + for config in configs + for entry in config.traps + ] + if missing and not unreadable: + labels = ", ".join(_coverage_label(eco) for eco in missing) + accepted = "; ".join(f"{_coverage_label(eco)}: {' or '.join(_DEPENDABOT_ECOSYSTEMS[eco])}" for eco in missing) + findings.append( _finding( spec, - Verdict.FAIL, - "keep dependencies patched without manual sweeps", - "no Dependabot or Renovate configuration exists", - remediation="Add .github/dependabot.yml (or renovate.json) with an update schedule.", - ), - ], - ) + Verdict.PARTIAL, + "keep every ecosystem's dependencies under the update policy", + f"the dependency-update configuration has no entry for {labels}", + evidence=[Evidence(config.path) for config in configs], + path=configs[0].path, + remediation=f"Add an update entry for each detected ecosystem (Dependabot `package-ecosystem` " + f"{accepted}), or the matching Renovate manager to `enabledManagers`.", + ) + ) + if findings: + return _outcome( + spec, Verdict.PARTIAL, f"dependency automation at {found[0]} has {len(findings)} defect(s)", findings + ) + if missing or len(unreadable) == len(configs): + return _outcome( + spec, + Verdict.UNKNOWN, + f"dependency automation at {', '.join(unreadable)} could not be read; coverage is not settled", + ) + reach = ", ".join(_coverage_label(eco) for eco in sorted(required)) or "no ecosystem with a manifest" + return _outcome(spec, Verdict.PASS, f"dependency automation configured at {found[0]}; covers {reach}") # =========================================================================== diff --git a/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md b/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md index c35f88f8..3b5410a9 100644 --- a/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md +++ b/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md @@ -220,7 +220,7 @@ pattern is *named* when it stands in a code span of the section. The audit also | `HYG-06` | SECURITY | A | Present | Implemented | | `HYG-07` | CHANGELOG | A | Present and dated recently | Implemented | | `HYG-08` | Release process documented | A | A release document exists | Implemented | -| `HYG-09` | Dependency update automation | A | Dependabot, Renovate, or equivalent configured | Implemented | +| `HYG-09` | Dependency update automation | A | Dependabot or Renovate configured, an entry for each detected ecosystem, and no `target-branch` beside `ignore` or `groups` | Implemented | | `HYG-10` | License | A | A license file exists | Implemented | | `HYG-11` | Branch protection | B | Protection declared by configuration-as-code where the provider allows | Implemented | diff --git a/audit/engine/docs/reference/changelog.md b/audit/engine/docs/reference/changelog.md index 78fe5d4a..4832257a 100644 --- a/audit/engine/docs/reference/changelog.md +++ b/audit/engine/docs/reference/changelog.md @@ -13,6 +13,7 @@ the record of how; git has the rest. | Revision | Date | Scoreable checks | What it is | |---|---|---|---| +| `2026-11-09.2` | 2026-11-09 | 56 | v0.3.3 C2: `HYG-09` (informational, unscored) reads the Dependabot or Renovate file. `PARTIAL` when a detected ecosystem (from a manifest, plus `github-actions` with GitHub CI) has no entry, or when a Dependabot entry sets `target-branch` together with `ignore` or `groups`; `UNKNOWN` when the file cannot be read. The update policy itself is the owner's choice and never a verdict. No score moves | | `2026-11-09.1` | 2026-11-09 | 56 | v0.3.3 C1: new check `CI-05` (workflow files are linted, DEGRADER). It reads the linters each CI provider supports (actionlint, zizmor, check-jsonschema schemas, `glab ci lint`, `circleci config validate`): local ⇒ `PASS`, CI only ⇒ `PARTIAL`, none ⇒ `FAIL`, no workflow or Jenkins only ⇒ `UNKNOWN`. Every repository with workflow files and no linter gains a `CI-05` finding | | `2026-11-06.1` | 2026-11-06 | 55 | MR06.4: `TST-04` accepts `python3 -m unittest -k ` as a single-test form; `AGT-08` takes a line of the root `AGENTS.md` that says "read by" or "supported harnesses" and names a known harness as the answer to the reach gap (#26) | | `2026-11-05.4` | 2026-11-05 | 55 | MR06.3: `NAV-02` entry point is a guard line, `DOC-04` ignores the ADR template, `--out` without a suffix, `--no-user-config` and `provenance.user_config` | diff --git a/audit/engine/tests/fixtures/README.md b/audit/engine/tests/fixtures/README.md index 4ef6515d..ffaf7ed5 100644 --- a/audit/engine/tests/fixtures/README.md +++ b/audit/engine/tests/fixtures/README.md @@ -115,6 +115,8 @@ fixtures/ con-02-specs-are-not-tests/ `specs/` holds feature specifications, `tests/` is named -> CON-02 PASS con-03-artefacts-not-ignored/ artefacts/ present, no ignore rule -> CON-03 FAIL hyg-01-placeholder-readme/ README is a TODO stub -> HYG-01 FAIL + hyg-09-uncovered-ecosystem/ package.json, a Dependabot file with only a pip entry -> HYG-09 PARTIAL + hyg-09-target-branch-trap/ the npm entry sets target-branch and ignore -> HYG-09 PARTIAL hyg-10-no-license/ real README, no LICENSE at the root -> HYG-10 FAIL hyg-minimal/ no hygiene artifacts at all -> HYG-01…10 FAIL, HYG-11 UNKNOWN governed-minimal/ the smallest repository that passes every implemented AGT content check, plus its cross-pack neighbours exercised by the matrix test diff --git a/audit/engine/tests/fixtures/hyg-09-target-branch-trap/.github/dependabot.yml b/audit/engine/tests/fixtures/hyg-09-target-branch-trap/.github/dependabot.yml new file mode 100644 index 00000000..87a82f7f --- /dev/null +++ b/audit/engine/tests/fixtures/hyg-09-target-branch-trap/.github/dependabot.yml @@ -0,0 +1,11 @@ +# The npm entry sets target-branch and ignore: the ignore rule does not apply to security updates. HYG-09 PARTIAL. +version: 2 +updates: + - package-ecosystem: npm + directory: / + schedule: + interval: weekly + target-branch: develop + ignore: + - dependency-name: "*" + update-types: ["version-update:semver-major"] diff --git a/audit/engine/tests/fixtures/hyg-09-target-branch-trap/package.json b/audit/engine/tests/fixtures/hyg-09-target-branch-trap/package.json new file mode 100644 index 00000000..1e188665 --- /dev/null +++ b/audit/engine/tests/fixtures/hyg-09-target-branch-trap/package.json @@ -0,0 +1,5 @@ +{ + "name": "hyg-09-target-branch-trap", + "version": "0.1.0", + "private": true +} diff --git a/audit/engine/tests/fixtures/hyg-09-uncovered-ecosystem/.github/dependabot.yml b/audit/engine/tests/fixtures/hyg-09-uncovered-ecosystem/.github/dependabot.yml new file mode 100644 index 00000000..b690320c --- /dev/null +++ b/audit/engine/tests/fixtures/hyg-09-uncovered-ecosystem/.github/dependabot.yml @@ -0,0 +1,7 @@ +# A Node project whose Dependabot file has no npm entry: HYG-09 PARTIAL. +version: 2 +updates: + - package-ecosystem: pip + directory: / + schedule: + interval: weekly diff --git a/audit/engine/tests/fixtures/hyg-09-uncovered-ecosystem/package.json b/audit/engine/tests/fixtures/hyg-09-uncovered-ecosystem/package.json new file mode 100644 index 00000000..98d43651 --- /dev/null +++ b/audit/engine/tests/fixtures/hyg-09-uncovered-ecosystem/package.json @@ -0,0 +1,5 @@ +{ + "name": "hyg-09-uncovered-ecosystem", + "version": "0.1.0", + "private": true +} diff --git a/audit/engine/tests/test_fixture_matrix.py b/audit/engine/tests/test_fixture_matrix.py index 9f49af25..267c14df 100644 --- a/audit/engine/tests/test_fixture_matrix.py +++ b/audit/engine/tests/test_fixture_matrix.py @@ -170,6 +170,8 @@ "idx-02-no-index": {"IDX-02": "UNKNOWN", "IDX-03": "UNKNOWN"}, # -- HYG (informational, reported never scored) ------------------------------------------- "hyg-01-placeholder-readme": {"HYG-01": "FAIL", "HYG-10": "FAIL"}, + "hyg-09-uncovered-ecosystem": {"HYG-09": "PARTIAL"}, + "hyg-09-target-branch-trap": {"HYG-09": "PARTIAL"}, "hyg-10-no-license": {"HYG-01": "PASS", "HYG-10": "FAIL"}, "hyg-minimal": {f"HYG-{n:02d}": "FAIL" for n in range(1, 11)} | {"HYG-11": "UNKNOWN"}, # -- the AGT-content positive fixture ----------------------------------------------------- diff --git a/audit/engine/tests/test_hyg09_policy.py b/audit/engine/tests/test_hyg09_policy.py new file mode 100644 index 00000000..b09b0853 --- /dev/null +++ b/audit/engine/tests/test_hyg09_policy.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna + +"""`HYG-09` reads the dependency-update file for coverage per ecosystem and the `target-branch` trap. + +The update policy itself (security pull requests only, majors advisory, cooldown) is the owner's choice and never +a verdict. Two defects hold whatever the policy: an ecosystem with no entry, and a Dependabot entry whose `ignore` +or `groups` options `target-branch` keeps from applying to security updates. +""" + +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import support # noqa: E402 + +PACKAGE_JSON = '{"name": "x", "version": "0.1.0", "private": true}\n' +WORKFLOW = ( + "name: CI\non: [pull_request]\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - run: make check\n" +) + + +def dependabot(*entries: str) -> str: + return "version: 2\nupdates:\n" + "".join(entries) + + +def entry(ecosystem: str, *extra: str) -> str: + lines = [f" - package-ecosystem: {ecosystem}", " directory: /", " schedule:", " interval: weekly"] + return "\n".join(lines + [f" {line}" for line in extra]) + "\n" + + +class Hyg09PolicyTests(unittest.TestCase): + def check(self, files: dict[str, str]) -> dict: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for rel, text in files.items(): + support.write(root, rel, text) + report = support.run_audit(root) + check = next(c for c in report["checks"] if c["id"] == "HYG-09") + check["findings"] = [f for f in report.get("findings", []) if f["check"] == "HYG-09"] + return check + + def test_every_ecosystem_covered_passes(self): + check = self.check( + { + "package.json": PACKAGE_JSON, + "pyproject.toml": "[project]\nname = 'x'\n", + ".github/workflows/ci.yml": WORKFLOW, + ".github/dependabot.yml": dependabot(entry("npm"), entry("uv"), entry("github-actions")), + } + ) + self.assertEqual(check["verdict"], "PASS") + self.assertIn("covers github-actions, node, python", check["summary"]) + + def test_github_ci_needs_a_github_actions_entry(self): + check = self.check( + { + "package.json": PACKAGE_JSON, + ".github/workflows/ci.yml": WORKFLOW, + ".github/dependabot.yml": dependabot(entry("npm")), + } + ) + self.assertEqual(check["verdict"], "PARTIAL") + self.assertIn("no entry for github-actions", check["findings"][0]["statement"]) + + def test_items_at_the_updates_indent_are_read(self): + text = "version: 2\nupdates:\n- package-ecosystem: 'npm' # quoted, with a comment\n directory: /\n" + check = self.check({"package.json": PACKAGE_JSON, "dependabot.yml": text}) + self.assertEqual(check["verdict"], "PASS") + + def test_target_branch_alone_is_not_the_trap(self): + check = self.check( + {"package.json": PACKAGE_JSON, ".github/dependabot.yml": dependabot(entry("npm", "target-branch: develop"))} + ) + self.assertEqual(check["verdict"], "PASS") + + def test_target_branch_with_groups_is_the_trap_and_cites_its_line(self): + text = dependabot(entry("npm", "target-branch: develop", "groups:", " all:", " patterns: ['*']")) + check = self.check({"package.json": PACKAGE_JSON, ".github/dependabot.yml": text}) + self.assertEqual(check["verdict"], "PARTIAL") + self.assertEqual(check["findings"][0]["evidence"][0]["line"], 7) + self.assertIn("`groups`", check["findings"][0]["statement"]) + + def test_a_source_only_python_project_needs_no_entry(self): + check = self.check({"src/app.py": "print('x')\n", ".github/dependabot.yml": dependabot(entry("docker"))}) + self.assertEqual(check["verdict"], "PASS") + + def test_a_dependabot_file_without_an_updates_list_is_unknown(self): + check = self.check({"package.json": PACKAGE_JSON, ".github/dependabot.yml": "version: 2\n"}) + self.assertEqual(check["verdict"], "UNKNOWN") + + def test_renovate_without_enabled_managers_covers_everything(self): + check = self.check( + {"package.json": PACKAGE_JSON, ".github/workflows/ci.yml": WORKFLOW, "renovate.json": '{"extends": []}'} + ) + self.assertEqual(check["verdict"], "PASS") + + def test_renovate_enabled_managers_that_leave_out_an_ecosystem_is_partial(self): + check = self.check({"package.json": PACKAGE_JSON, "renovate.json": '{"enabledManagers": ["pep621"]}'}) + self.assertEqual(check["verdict"], "PARTIAL") + self.assertIn("no entry for node", check["findings"][0]["statement"]) + + def test_renovate_json5_counts_on_presence(self): + check = self.check({"package.json": PACKAGE_JSON, "renovate.json5": "{ extends: [] }"}) + self.assertEqual(check["verdict"], "PASS") + + def test_invalid_renovate_json_is_unknown(self): + check = self.check({"package.json": PACKAGE_JSON, "renovate.json": "{ not json"}) + self.assertEqual(check["verdict"], "UNKNOWN") + + def test_a_second_tool_can_cover_what_the_first_leaves_out(self): + check = self.check( + { + "package.json": PACKAGE_JSON, + ".github/workflows/ci.yml": WORKFLOW, + ".github/dependabot.yml": dependabot(entry("github-actions")), + "renovate.json": '{"enabledManagers": ["npm"]}', + } + ) + self.assertEqual(check["verdict"], "PASS") + + +if __name__ == "__main__": + unittest.main() diff --git a/audit/engine/tests/test_hyg_pack.py b/audit/engine/tests/test_hyg_pack.py index 4e6f520a..b50ebc74 100644 --- a/audit/engine/tests/test_hyg_pack.py +++ b/audit/engine/tests/test_hyg_pack.py @@ -34,6 +34,8 @@ #: fixture directory -> {check id: expected verdict} FIXTURES: dict[str, dict[str, str]] = { "hyg-01-placeholder-readme": {"HYG-01": "FAIL", "HYG-10": "FAIL"}, + "hyg-09-uncovered-ecosystem": {"HYG-09": "PARTIAL"}, + "hyg-09-target-branch-trap": {"HYG-09": "PARTIAL"}, "hyg-10-no-license": {"HYG-01": "PASS", "HYG-10": "FAIL"}, "hyg-minimal": { "HYG-01": "FAIL", diff --git a/audit/engine/tests/test_hygiene_presence.py b/audit/engine/tests/test_hygiene_presence.py index d77335fb..b19764f0 100644 --- a/audit/engine/tests/test_hygiene_presence.py +++ b/audit/engine/tests/test_hygiene_presence.py @@ -63,7 +63,7 @@ def test_the_conventional_locations_still_pass(self): "Makefile": MAKE, ".github/CODEOWNERS": "* @a\n", ".github/pull_request_template.md": "x", - ".github/dependabot.yml": "x", + ".github/dependabot.yml": "version: 2\nupdates:\n - package-ecosystem: github-actions\n", "CONTRIBUTING.md": "x", "docs/SECURITY.md": "x", "docs/how-to/release-process.md": "x", diff --git a/dev/eval/expected/fixtures/hyg-09-target-branch-trap.json b/dev/eval/expected/fixtures/hyg-09-target-branch-trap.json new file mode 100644 index 00000000..75a7cf9f --- /dev/null +++ b/dev/eval/expected/fixtures/hyg-09-target-branch-trap.json @@ -0,0 +1,14 @@ +{ + "expected_version": "0.1", + "repo_label": "hyg-09-target-branch-trap", + "repo_sha": "8d6da8e873c18b65ead44b1e2397ff3dc12a5ef5", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-09-target-branch-trap' (audit/engine/tests/fixtures/hyg-09-target-branch-trap), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "checks": [ + { + "id": "HYG-09", + "acceptable_verdicts": [ + "PARTIAL" + ] + } + ] +} diff --git a/dev/eval/expected/fixtures/hyg-09-uncovered-ecosystem.json b/dev/eval/expected/fixtures/hyg-09-uncovered-ecosystem.json new file mode 100644 index 00000000..81f25b5c --- /dev/null +++ b/dev/eval/expected/fixtures/hyg-09-uncovered-ecosystem.json @@ -0,0 +1,14 @@ +{ + "expected_version": "0.1", + "repo_label": "hyg-09-uncovered-ecosystem", + "repo_sha": "17bc601cdf67a32857dc021e17d6c1baf6f52ce8", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-09-uncovered-ecosystem' (audit/engine/tests/fixtures/hyg-09-uncovered-ecosystem), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "checks": [ + { + "id": "HYG-09", + "acceptable_verdicts": [ + "PARTIAL" + ] + } + ] +} From 81b00021a9d3697bfd0bc0845723c1db86375761 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 13:20:05 +0530 Subject: [PATCH 028/111] docs(implement): write or fix the dependency-update file for HYG-09 Step 8 of add-repo-hygiene no longer only verifies. It keeps the tool the repository uses (Dependabot on GitHub, Renovate elsewhere when there is none), writes one entry per detected ecosystem plus github-actions, and removes a target-branch that names the default branch; any other target is an owner question. A new preference, deps.update_policy, picks the policy: security-only (the default), security-and-minor or all. Turning on security updates stays an owner action. The Verification table gains its own HYG-09 row. The Python, Node and Java profiles name their Dependabot ecosystems and Renovate managers, and the profile anatomy lists the field. Roadmap-Step: C2 Co-Authored-By: Claude Opus 5.5 --- audit/stacks/README.md | 3 +- audit/stacks/java.md | 3 +- audit/stacks/python.md | 4 ++- audit/stacks/typescript-node.md | 3 +- implement/playbooks/add-repo-hygiene.md | 40 ++++++++++++++++++++----- 5 files changed, 42 insertions(+), 11 deletions(-) diff --git a/audit/stacks/README.md b/audit/stacks/README.md index 066acb9a..f3b39fb1 100644 --- a/audit/stacks/README.md +++ b/audit/stacks/README.md @@ -34,7 +34,8 @@ Detection signals files and dependencies that identify the stack and its fr Command-surface map how each verb is usually realised, and the runner (npm scripts / Makefile / Taskfile / Gradle / Maven) Tooling defaults recommended formatter, linter, type checker, test runner, coverage tool, - dependency scanner, hook manager and commit-message rule, with "alternatives + dependency scanner, hook manager, commit-message rule and dependency-update + ecosystem (Dependabot and Renovate names), with "alternatives you may find" so an existing choice is respected, never replaced without being asked Execution determinism version pinning and lockfile expectations for this stack diff --git a/audit/stacks/java.md b/audit/stacks/java.md index b14d0960..83f0f9f9 100644 --- a/audit/stacks/java.md +++ b/audit/stacks/java.md @@ -43,7 +43,8 @@ Dependency-Check Gradle plugin (`org.owasp.dependencycheck`), registering `depen task name that does not match `security` literally, so a repository applying the plugin still reads `CMD-01` `FAIL` for that verb unless a `security` task delegates to it (see Pitfalls). Hook manager: none is standard; a committed `.githooks/` with `core.hooksPath` set by a build task, or lefthook (no JVM plugin -needed). Commit rule: a `commit-msg` script or lefthook stage. +needed). Commit rule: a `commit-msg` script or lefthook stage. Dependency updates: Dependabot +`package-ecosystem: gradle` or `maven`; Renovate's `gradle` and `maven` managers. ## Execution determinism diff --git a/audit/stacks/python.md b/audit/stacks/python.md index 2f1866cd..bba43175 100644 --- a/audit/stacks/python.md +++ b/audit/stacks/python.md @@ -33,7 +33,9 @@ Flake8/Pylint on an older one; config in `pyproject.toml`'s `[tool.ruff]`/`[tool `mypy.ini`'s `strict = true`. Test runner: pytest. Dependency scanner: `pip-audit` (works against any of pip/uv/poetry's resolved environment) or `safety check`. Hook manager: pre-commit (`.pre-commit-config.yaml`); lefthook in a mixed repository. Commit rule: a pre-commit `commit-msg` hook -(conventional-pre-commit, commitizen or gitlint). +(conventional-pre-commit, commitizen or gitlint). Dependency updates: Dependabot `package-ecosystem: pip` +(pip, pipenv, poetry, pip-compile) or `uv` (a `uv.lock`); Renovate's `pep621`, `pip_requirements`, +`poetry` and `pipenv` managers. ## Execution determinism diff --git a/audit/stacks/typescript-node.md b/audit/stacks/typescript-node.md index 54256dc4..9f1fbc1c 100644 --- a/audit/stacks/typescript-node.md +++ b/audit/stacks/typescript-node.md @@ -43,7 +43,8 @@ seen across the corpus (§Verified) — a third option, Node's own built-in test Dependency scanner: the package manager's own `audit` command; `audit-ci` or Snyk are alternatives that pin a fixed severity threshold in config rather than a CLI flag. Hook manager: Husky + lint-staged (`prepare: husky`), or simple-git-hooks; lefthook when there is no root `package.json`. Commit rule: -commitlint in `.husky/commit-msg`. +commitlint in `.husky/commit-msg`. Dependency updates: Dependabot `package-ecosystem: npm` (npm, Yarn and +pnpm lockfiles) or `bun`; Renovate's `npm` manager. ## Execution determinism diff --git a/implement/playbooks/add-repo-hygiene.md b/implement/playbooks/add-repo-hygiene.md index 44ba919a..8b12c2ff 100644 --- a/implement/playbooks/add-repo-hygiene.md +++ b/implement/playbooks/add-repo-hygiene.md @@ -28,7 +28,7 @@ action in the state, never an invented contact. | HYG-06 | [13-hygiene-informational.md](../../audit/checks/13-hygiene-informational.md#hyg-06--security-present) | A `SECURITY` document: how to report privately, with owner-supplied contact values as a named placeholder | | HYG-07 | [13-hygiene-informational.md](../../audit/checks/13-hygiene-informational.md#hyg-07--changelog-present-and-recent) | Verification, and a decision: a changelog the release tool writes is not hand-written (Steps 6) | | HYG-08 | [13-hygiene-informational.md](../../audit/checks/13-hygiene-informational.md#hyg-08--release-process-documented) | A short document (name contains `release`) describing the release process in prose | -| HYG-09 | [13-hygiene-informational.md](../../audit/checks/13-hygiene-informational.md#hyg-09--dependency-update-automation-configured) | Verification only: a Dependabot or Renovate file already exists in most repositories | +| HYG-09 | [13-hygiene-informational.md](../../audit/checks/13-hygiene-informational.md#hyg-09--dependency-update-automation-configured) | An update entry for each detected ecosystem, no `target-branch` beside `ignore` or `groups`, and the owner's update policy (Step 8) | | HYG-10 | [13-hygiene-informational.md](../../audit/checks/13-hygiene-informational.md#hyg-10--license-present) | Verification only; the playbook never chooses a licence | | HYG-11 | [13-hygiene-informational.md](../../audit/checks/13-hygiene-informational.md#hyg-11--branch-protection-declared-as-code) | A page whose path contains `branch-protection` that records the rules in the repository's own words | @@ -66,6 +66,7 @@ observation. | `ci.branch_protection`, `ci.branch_protection_runbook` | Whether the rules are enforced or only recorded; who may change the setting | | `agents_md.release_tags` | `never-tag-or-release` means the release page says agents never tag | | `format.mass_reformat` | `no`: new Markdown only, no reformatting of existing pages | +| `deps.update_policy` | `security-only` (default), `security-and-minor` or `all`: which pull requests the dependency-update tool opens (Step 8) | ## Inputs to ask @@ -81,6 +82,8 @@ the policy files, and for nothing else: repository: a handle is per repository. Default shape: keep the existing path rules, add a default `*` line for the owner, and add the second-reviewer line as a comment. **Required code-owner review stays off**: with one owner it would block every pull request. The playbook never enables it and never edits the GitHub setting. + **Dependency updates (HYG-09).** When HYG-09 has an item, ask for `deps.update_policy` in the same batch; the + default is `security-only`. It is the owner's choice: the audit never marks a policy down. 3. **SECURITY (HYG-06).** Ask for the mailbox and the response time. If the owner has not supplied them, write the document with the placeholder `TODO(owner)` where each value goes. The disclosure path by repository kind: - **Public repository:** GitHub private vulnerability reporting, plus the placeholder for a mailbox and a response @@ -144,12 +147,33 @@ the policy files, and for nothing else: rulesets), the page says **"recorded, not enforced"** at the top; it must not read as enforcement. Never copy a private runbook from another repository; describe the rules from the repository's own workflow files and the owner's answers. Never call the GitHub settings API. -8. **Dependency updates and licence (HYG-09, HYG-10).** Verify only. If the dependency-update file exists, record - `done` when the check has an item; when it has none (the audit passed it), name it in the pull request body and - record nothing. If a `LICENSE` is missing, do not pick one: raise it as an owner question and set the item - `blocked` (reason: the owner chooses a licence). +8. **Dependency updates (HYG-09).** Read what exists first; never replace one tool with the other. + - **Tool.** Keep the tool the repository already uses. With none: Dependabot on GitHub, Renovate on GitLab, + Azure DevOps or Bitbucket, where Dependabot does not run. + - **Coverage.** One entry per ecosystem the audit detected from a manifest, with the `package-ecosystem` the + stack profile names (Node `npm`, Python `pip` or `uv`, Java `maven` or `gradle`, Go `gomod`, Rust `cargo`, + .NET `nuget`, Ruby `bundler`, PHP `composer`), plus `github-actions` when the repository has GitHub + workflows. Set `directory` (or `directories`) to where each manifest lives. Renovate needs no entry per + ecosystem; remove a detected one from `enabledManagers` only when the owner asks. + - **The `target-branch` trap.** Dependabot drops an entry's `ignore` and `groups` for security updates when the + entry sets `target-branch`; security pull requests always go to the default branch. If `target-branch` names + the default branch, delete the line. If it names another branch, do not change it: ask the owner whether to + make that branch the default (a setting the playbook never edits) or to drop `ignore`/`groups` there. + - **Policy (`deps.update_policy`).** For Dependabot, `security-only` writes `open-pull-requests-limit: 0` + (version updates off; security updates are a repository setting and an owner action), an `ignore` for + `version-update:semver-major` (a major stays an advisory alert), a `security-fixes` group with + `applies-to: security-updates`, and `cooldown: default-days: 7`. `security-and-minor` drops the limit and + keeps the major `ignore`; `all` drops both. For Renovate, write the same policy with `vulnerabilityAlerts`, + a `packageRules` entry with `enabled: false` for other updates, and `minimumReleaseAge: "7 days"`; confirm + the exact form in Renovate's configuration documentation before writing it, and cite the page in the pull + request body. The cooldown matches zizmor's `dependabot-cooldown` rule (CI-05). + - **Owner action.** Record "turn on Dependabot security updates in the repository settings" as a note on the + item: an agent can neither read nor set it. + - When HYG-09 passed and has no item, name it in the pull request body and change nothing. + - **Licence (HYG-10).** Verify only. If a `LICENSE` is missing, do not pick one: raise it as an owner question + and set the item `blocked` (reason: the owner chooses a licence). 9. **Commits:** one per concern: policy files (CODEOWNERS and SECURITY); contribution files (README, CONTRIBUTING - and the templates); release and protection pages. + and the templates); release and protection pages; the dependency-update file. ### Gradle and Quarkus @@ -190,7 +214,8 @@ and CONTRIBUTING are the root scripts, or per-app commands with the directory na | HYG-06 | `ls SECURITY*` at the root, `.github/` or `docs/` | Present; `PASS` | | HYG-07 | Launcher run | `PASS` once the release tool has committed the file; until then `FAIL` and the item is `partial` | | HYG-08 | `ls` for a document with `release` in its name | Present as prose, not only `release.yml`; `PASS` | -| HYG-09, HYG-10 | `ls .github/dependabot.yml`, `ls LICENSE*` | Present; `PASS` (verified, not added) | +| HYG-09 | Launcher run; read each `updates:` entry | An entry per detected ecosystem, no `target-branch` beside `ignore` or `groups`; `PASS` | +| HYG-10 | `ls LICENSE*` | Present; `PASS` (verified, not added) | | HYG-11 | `ls docs/*branch-protection*` | Present, says "recorded, not enforced" where that is true; `PASS` | ## Rollback @@ -207,6 +232,7 @@ vulnerability reporting, branch protection, required reviews) are never touched, - Do not copy `RELEASE_PROCESS.md` whole; write the short page. - Do not copy a private runbook, another repository's page, or its values into a repository's file. - Do not choose a licence. +- Do not replace Renovate with Dependabot or the reverse, and do not turn on security updates in the settings. - Do not create a file in a component directory expecting it to count. - Do not reformat existing pages (`format.mass_reformat`). From 095994f2b719bd75e855bfc2fcf28516e0fd0ed6 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 13:23:43 +0530 Subject: [PATCH 029/111] build(dev): build actionlint from a Go tool module, not from actionlint-py actionlint-py ships only a source package; building it downloads the actionlint binary from GitHub releases. On PR #77 three of the four `make check` jobs failed when that download returned HTTP 504. actionlint v1.7.12 (the same version) is now a Go `tool` in dev/tools/actionlint/go.mod. Go fetches it from the Go module proxy and go.sum checks it; CI's check job installs Go and caches the build. Dependabot's gomod entry watches both Go tool modules, and make security audits the new module too (golang.org/x/sys raised to 0.44.0 for GO-2026-5024). make check now needs Go, which make setup checks. Roadmap-Step: A6 Co-Authored-By: Claude Opus 5.5 --- .audit-baseline.json | 23 +++++----- .github/dependabot.yml | 8 ++-- .github/workflows/ci.yml | 5 +++ .secrets.baseline | 14 +++--- AGENTS.md | 6 +-- CONTRIBUTING.md | 2 +- Makefile | 14 +++--- RELEASE_PROCESS.md | 6 +-- .../examples/self-audit/2026-11-09.json | 23 +++++----- dev/tests/test_tool_pins.py | 43 ++++++++++++++++--- dev/tools/actionlint/go.mod | 20 +++++++++ dev/tools/actionlint/go.sum | 27 ++++++++++++ dev/tools/pyproject.toml | 1 - dev/tools/uv.lock | 8 ---- ...-dependabot-security-pull-requests-only.md | 5 ++- ...rb-audits-dependencies-with-osv-scanner.md | 7 ++- docs/reference/dev-tooling.md | 13 +++--- docs/reference/self-audit.md | 2 +- 18 files changed, 155 insertions(+), 72 deletions(-) create mode 100644 dev/tools/actionlint/go.mod create mode 100644 dev/tools/actionlint/go.sum diff --git a/.audit-baseline.json b/.audit-baseline.json index 14856e9d..e4404edb 100644 --- a/.audit-baseline.json +++ b/.audit-baseline.json @@ -161,7 +161,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "20 boundary rules reference real paths", + "summary": "23 boundary rules reference real paths", "tier": "B", "title": "Architectural boundaries declared", "verdict": "PASS" @@ -531,7 +531,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "github: local actionlint (.audit-baseline.json), actionlint (Makefile), actionlint (dev/tools/pyproject.toml), check-jsonschema (Makefile), zizmor (.audit-baseline.json), zizmor (Makefile), zizmor (dev/tools/pyproject.toml), CI zizmor (.github/workflows/ci.yml)", + "summary": "github: local actionlint (.audit-baseline.json), actionlint (Makefile), actionlint (dev/tools/actionlint/go.mod), check-jsonschema (Makefile), zizmor (.audit-baseline.json), zizmor (Makefile), zizmor (dev/tools/pyproject.toml), CI actionlint (.github/workflows/ci.yml), zizmor (.github/workflows/ci.yml)", "tier": "B", "title": "Workflow files are linted", "verdict": "PASS" @@ -600,6 +600,7 @@ "dev/tools/pyproject.toml", "dev/tools/package.json", "dev/tools/osv/go.mod", + "dev/tools/actionlint/go.mod", "lefthook.yml", "dev", ".github/", @@ -659,7 +660,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "the prohibitions section of AGENTS.md names paths that exist: Makefile, audit/engine/, AGENTS.md, _typos.toml, .python-version, dev/tools/pyproject.toml, dev/tools/package.json, dev/tools/osv/go.mod (+46 more)", + "summary": "the prohibitions section of AGENTS.md names paths that exist: Makefile, audit/engine/, AGENTS.md, _typos.toml, .python-version, dev/tools/pyproject.toml, dev/tools/package.json, dev/tools/osv/go.mod (+47 more)", "tier": "B", "title": "Allow/deny artifact present", "verdict": "PASS" @@ -733,7 +734,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "consistency unverified: dev/tools/osv/go.sum, dev/tools/package-lock.json, dev/tools/uv.lock", + "summary": "consistency unverified: dev/tools/actionlint/go.sum, dev/tools/package-lock.json, dev/tools/uv.lock", "tier": "B", "title": "Lockfile committed and consistent", "verdict": "PARTIAL" @@ -936,10 +937,10 @@ }, { "data": { - "denominator": 1188, + "denominator": 1189, "kind": "ratio", "method": "python: module-level def/class not starting with '_'", - "numerator": 476, + "numerator": 477, "unit": "public symbols" }, "id": "NAV-05", @@ -948,7 +949,7 @@ "scored": false, "severity": "COSMETIC", "status": "implemented", - "summary": "476/1188 public symbols documented (40%); method: python: module-level def/class not starting with '_'", + "summary": "477/1189 public symbols documented (40%); method: python: module-level def/class not starting with '_'", "tier": "B", "title": "Public-symbol documentation coverage", "verdict": "PARTIAL" @@ -1364,7 +1365,7 @@ } ], "inventory": { - "files_inspected": 350, + "files_inspected": 352, "files_set_aside": 463, "files_skipped": 7, "set_aside": [ @@ -1444,15 +1445,15 @@ } ], "provenance": { - "framework_revision": "65024fb26b6cdc04d071ee448257a38c8b2bda10", + "framework_revision": "d6ba4cb0af8e785f4096bbdf9e8d89eeeb6c6d95", "git_branch": "chore/v0.3.3-track-a-tooling", "git_dirty": true, - "git_sha": "65024fb26b6cdc04d071ee448257a38c8b2bda10", + "git_sha": "d6ba4cb0af8e785f4096bbdf9e8d89eeeb6c6d95", "phase": 1, "probes_enabled": false, "ruleset_hash": "04e5d0ff9063e012", "ruleset_revision": "2026-11-09.1", - "scanned_at": "2026-11-09T07:32:15+00:00", + "scanned_at": "2026-11-09T07:49:24+00:00", "target": ".", "tool": "python-agentic-audit", "user_config": { diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e0cc26c2..de904880 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -30,7 +30,7 @@ updates: commit-message: prefix: "ci" - # The Python dev tools (ruff, mypy, detect-secrets, coverage, jsonschema, skills-ref, typos, uv, actionlint-py, zizmor, + # The Python dev tools (ruff, mypy, detect-secrets, coverage, jsonschema, skills-ref, typos, uv, zizmor, # check-jsonschema), locked in uv.lock. The # Makefile, lefthook.yml and the workflows read each version from dev/tools/pyproject.toml. - package-ecosystem: uv @@ -69,10 +69,10 @@ updates: commit-message: prefix: "build" - # osv-scanner, the dependency audit in `make security`, pinned as a Go `tool` in dev/tools/osv/go.mod with its - # dependencies. The Makefile runs it with `go -C dev/tools/osv tool osv-scanner`. + # The Go dev tools, each pinned as a Go `tool` in its own module with its dependencies: osv-scanner (the dependency + # audit in `make security`) and actionlint (`make workflow-lint`). The Makefile runs them with `go -C tool`. - package-ecosystem: gomod - directory: /dev/tools/osv + directories: ["/dev/tools/osv", "/dev/tools/actionlint"] schedule: interval: weekly cooldown: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c183f32..b79a858a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,11 @@ jobs: - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version-file: dev/tools/pyproject.toml + # Go builds actionlint (`make workflow-lint`) from dev/tools/actionlint/go.mod, through the Go module proxy. + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: dev/tools/actionlint/go.mod + cache-dependency-path: dev/tools/actionlint/go.sum - run: make check security: diff --git a/.secrets.baseline b/.secrets.baseline index a7181689..71ae5c26 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -131,16 +131,16 @@ { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", - "hashed_secret": "3eff25ad652c322335d2e94c37143d30cde544cd", + "hashed_secret": "477409e5b5f2e1cc0244e71cfbaf82140ac26be1", "is_verified": false, - "line_number": 1447 + "line_number": 1448 }, { "type": "Hex High Entropy String", "filename": ".audit-baseline.json", "hashed_secret": "78d6ecebdb129eed66e4824377d8ca833d9b3932", "is_verified": false, - "line_number": 1453 + "line_number": 1454 } ], "audit/engine/audit/rules/checks/con.py": [ @@ -442,16 +442,16 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/self-audit/2026-11-09.json", - "hashed_secret": "3eff25ad652c322335d2e94c37143d30cde544cd", + "hashed_secret": "477409e5b5f2e1cc0244e71cfbaf82140ac26be1", "is_verified": false, - "line_number": 1447 + "line_number": 1448 }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/self-audit/2026-11-09.json", "hashed_secret": "78d6ecebdb129eed66e4824377d8ca833d9b3932", "is_verified": false, - "line_number": 1453 + "line_number": 1454 } ], "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json": [ @@ -1474,5 +1474,5 @@ } ] }, - "generated_at": "2026-11-09T07:32:37Z" + "generated_at": "2026-11-09T07:49:37Z" } diff --git a/AGENTS.md b/AGENTS.md index 331b1ebe..7ee4a835 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,9 +57,9 @@ no vendor-specific instruction file exists beside it. - When you change Markdown, also run `make docs-lint` (needs Node; lints the ratchet path list in the `Makefile`) and validate every Mermaid block you add (`make docs-mermaid FILES=`). - Python 3.11 or newer (pinned in `.python-version`), `uv` for the pinned dev tools (ruff, mypy, detect-secrets), - and Go 1.27 or newer for osv-scanner in `make security`. - Each tool version is written once, in `dev/tools/pyproject.toml`, `dev/tools/package.json` or - `dev/tools/osv/go.mod` (ADR 0003); never + and Go 1.27 or newer for actionlint (`make check`) and osv-scanner (`make security`). + Each tool version is written once, in `dev/tools/pyproject.toml`, `dev/tools/package.json` or a Go tool module + (`dev/tools/osv/go.mod`, `dev/tools/actionlint/go.mod`) (ADR 0003); never write a version in the `Makefile`, a hook or a workflow. Nothing is installed into the project. - Local hooks are `lefthook.yml` (a stop on commits to `dev` and `main`, format check on staged Python, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6360eca3..16aee46c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,7 +29,7 @@ the Makefile. The rules an agent follows are in [`AGENTS.md`](AGENTS.md); they a | Python 3.11 or newer | Everything | Pinned in `.python-version`; CI runs 3.11 and 3.14 | | `git` | Everything | | | [`uv`](https://docs.astral.sh/uv/) | `make check`, `make security`, `make coverage` | The dev tools (ruff, mypy, detect-secrets, coverage.py) run through `uvx` at the versions pinned in `dev/tools/`; nothing is installed into the project | -| [Go](https://go.dev/doc/install) 1.27 or newer | `make security` | Builds the pinned osv-scanner from `dev/tools/osv/go.mod` on first use. `make security` also needs the network | +| [Go](https://go.dev/doc/install) 1.27 or newer | `make check`, `make security` | Builds the pinned actionlint (`dev/tools/actionlint/go.mod`) and osv-scanner (`dev/tools/osv/go.mod`) on first use, through the Go module proxy. `make security` also needs the network | | `make` | Every command below | Linux and macOS. On Windows use WSL | | Node 22 | `make docs-lint` | The pinned `markdownlint-cli2` runs through `npx`; not needed for `make check` | | Chrome, through Puppeteer | `make docs-mermaid` | Only when you add or change a Mermaid block; the pinned mermaid-cli fetches it on first use | diff --git a/Makefile b/Makefile index a67cfca9..4324c0ef 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ # # The framework's command surface for this repository: seven verbs, the same names a human, an agent # and CI run. The runtime code is standard-library Python; the dev tools below are pinned in dev/tools/ and run -# through uvx, so nothing is installed into the project. Needs make, git, python3 (3.11+) and uv; `make security` -# also needs Go (1.27+) and the network. +# through uvx or `go tool`, so nothing is installed into the project. Needs make, git, python3 (3.11+), uv and Go +# (1.27+, for actionlint and osv-scanner); `make security` also needs the network. # # make format rewrite formatting in place # make format:check read-only: formatting and lint rules, and the ruff count against .quality-baseline.json @@ -47,12 +47,14 @@ MARKDOWNLINT := npx --yes markdownlint-cli2@$(call npm_pin,markdownlint-cli2) # The compare step of the quality baseline (.quality-baseline.json): fails when a tool's full-tree count goes up. QUALITY_BASELINE := $(PYTHON) dev/quality_baseline.py check # The lockfiles that `make security` audits: every dev tool, Python, Node and Go. -DEP_LOCKFILES := dev/tools/uv.lock dev/tools/package-lock.json dev/tools/osv/go.mod +DEP_LOCKFILES := dev/tools/uv.lock dev/tools/package-lock.json dev/tools/osv/go.mod dev/tools/actionlint/go.mod COVERAGE := uvx coverage@$(call pin,coverage) # Workflow and GitHub-config checks: actionlint (syntax, expressions, shellcheck on run: blocks), zizmor (workflow # and Dependabot security, offline, at the pedantic persona: its help-level findings fail too) and check-jsonschema # (the SchemaStore schemas it bundles, offline). -ACTIONLINT := uvx --from actionlint-py==$(call pin,actionlint-py) actionlint +# actionlint is a Go `tool` in dev/tools/actionlint/go.mod: the Go module proxy serves it, go.sum checks it, and +# Dependabot (gomod) sees the version. `go -C` runs it in that folder, so the workflows are passed by full path. +ACTIONLINT := go -C dev/tools/actionlint tool actionlint ZIZMOR := uvx zizmor@$(call pin,zizmor) # The spell check: typos reads _typos.toml (the word list and the excluded data paths) from the repository root, # and walks the tracked files that .gitignore does not hide. @@ -99,7 +101,7 @@ setup: @$(PYTHON) -c 'import sys; sys.exit(0 if sys.version_info >= (3, 11) else "setup: Python 3.11 or newer is required (see .python-version)")' @command -v uv >/dev/null || { echo "setup: uv is required (https://docs.astral.sh/uv/)" >&2; exit 1; } @command -v git >/dev/null || { echo "setup: git is required" >&2; exit 1; } - @command -v go >/dev/null || { echo "setup: Go 1.27 or newer is required for make security (https://go.dev/doc/install)" >&2; exit 1; } + @command -v go >/dev/null || { echo "setup: Go 1.27 or newer is required for make check and make security (https://go.dev/doc/install)" >&2; exit 1; } @echo "setup: prerequisites present (python, uv, git, go); nothing to install" format: @@ -175,7 +177,7 @@ spell-fix: $(TYPOS) --write-changes workflow-lint: - $(ACTIONLINT) + $(ACTIONLINT) $(CURDIR)/.github/workflows/*.yml $(ZIZMOR) --offline --persona pedantic .github $(CHECK_JSONSCHEMA) --builtin-schema vendor.github-workflows .github/workflows/*.yml $(CHECK_JSONSCHEMA) --builtin-schema vendor.dependabot .github/dependabot.yml diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 1e0b068c..1e09a36f 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -355,7 +355,7 @@ Each dev tool version is written once, in `dev/tools/`. The `Makefile`, `lefthoo the workflows read it from there. `dev/tests/test_tool_pins.py` fails when a version is written anywhere else. 1. Change the version in `dev/tools/pyproject.toml` (Python tools), `dev/tools/package.json` (Node tools) or - `dev/tools/osv/go.mod` (osv-scanner). + the Go tool module (`dev/tools/osv/go.mod` for osv-scanner, `dev/tools/actionlint/go.mod` for actionlint). 2. Make the lockfile again: ```bash @@ -363,8 +363,8 @@ the workflows read it from there. `dev/tests/test_tool_pins.py` fails when a ver (cd dev/tools && npm install --package-lock-only --ignore-scripts) ``` - For osv-scanner, or one of its Go dependencies, run - `go -C dev/tools/osv get @`, then `go -C dev/tools/osv mod tidy`. + For a Go tool, or one of its Go dependencies, run `go -C dev/tools/ get @`, then + `go -C dev/tools/ mod tidy`. `` is `osv` or `actionlint`. 3. Run `make check`. If ruff or markdownlint changed a rule, fix the new findings in the same branch. 4. Commit the manifest and its lockfile together, for example `build(dev): update ruff to 0.17.0`. diff --git a/audit/schema/examples/self-audit/2026-11-09.json b/audit/schema/examples/self-audit/2026-11-09.json index 14856e9d..e4404edb 100644 --- a/audit/schema/examples/self-audit/2026-11-09.json +++ b/audit/schema/examples/self-audit/2026-11-09.json @@ -161,7 +161,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "20 boundary rules reference real paths", + "summary": "23 boundary rules reference real paths", "tier": "B", "title": "Architectural boundaries declared", "verdict": "PASS" @@ -531,7 +531,7 @@ "scored": true, "severity": "DEGRADER", "status": "implemented", - "summary": "github: local actionlint (.audit-baseline.json), actionlint (Makefile), actionlint (dev/tools/pyproject.toml), check-jsonschema (Makefile), zizmor (.audit-baseline.json), zizmor (Makefile), zizmor (dev/tools/pyproject.toml), CI zizmor (.github/workflows/ci.yml)", + "summary": "github: local actionlint (.audit-baseline.json), actionlint (Makefile), actionlint (dev/tools/actionlint/go.mod), check-jsonschema (Makefile), zizmor (.audit-baseline.json), zizmor (Makefile), zizmor (dev/tools/pyproject.toml), CI actionlint (.github/workflows/ci.yml), zizmor (.github/workflows/ci.yml)", "tier": "B", "title": "Workflow files are linted", "verdict": "PASS" @@ -600,6 +600,7 @@ "dev/tools/pyproject.toml", "dev/tools/package.json", "dev/tools/osv/go.mod", + "dev/tools/actionlint/go.mod", "lefthook.yml", "dev", ".github/", @@ -659,7 +660,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "the prohibitions section of AGENTS.md names paths that exist: Makefile, audit/engine/, AGENTS.md, _typos.toml, .python-version, dev/tools/pyproject.toml, dev/tools/package.json, dev/tools/osv/go.mod (+46 more)", + "summary": "the prohibitions section of AGENTS.md names paths that exist: Makefile, audit/engine/, AGENTS.md, _typos.toml, .python-version, dev/tools/pyproject.toml, dev/tools/package.json, dev/tools/osv/go.mod (+47 more)", "tier": "B", "title": "Allow/deny artifact present", "verdict": "PASS" @@ -733,7 +734,7 @@ "scored": true, "severity": "BLOCKER", "status": "implemented", - "summary": "consistency unverified: dev/tools/osv/go.sum, dev/tools/package-lock.json, dev/tools/uv.lock", + "summary": "consistency unverified: dev/tools/actionlint/go.sum, dev/tools/package-lock.json, dev/tools/uv.lock", "tier": "B", "title": "Lockfile committed and consistent", "verdict": "PARTIAL" @@ -936,10 +937,10 @@ }, { "data": { - "denominator": 1188, + "denominator": 1189, "kind": "ratio", "method": "python: module-level def/class not starting with '_'", - "numerator": 476, + "numerator": 477, "unit": "public symbols" }, "id": "NAV-05", @@ -948,7 +949,7 @@ "scored": false, "severity": "COSMETIC", "status": "implemented", - "summary": "476/1188 public symbols documented (40%); method: python: module-level def/class not starting with '_'", + "summary": "477/1189 public symbols documented (40%); method: python: module-level def/class not starting with '_'", "tier": "B", "title": "Public-symbol documentation coverage", "verdict": "PARTIAL" @@ -1364,7 +1365,7 @@ } ], "inventory": { - "files_inspected": 350, + "files_inspected": 352, "files_set_aside": 463, "files_skipped": 7, "set_aside": [ @@ -1444,15 +1445,15 @@ } ], "provenance": { - "framework_revision": "65024fb26b6cdc04d071ee448257a38c8b2bda10", + "framework_revision": "d6ba4cb0af8e785f4096bbdf9e8d89eeeb6c6d95", "git_branch": "chore/v0.3.3-track-a-tooling", "git_dirty": true, - "git_sha": "65024fb26b6cdc04d071ee448257a38c8b2bda10", + "git_sha": "d6ba4cb0af8e785f4096bbdf9e8d89eeeb6c6d95", "phase": 1, "probes_enabled": false, "ruleset_hash": "04e5d0ff9063e012", "ruleset_revision": "2026-11-09.1", - "scanned_at": "2026-11-09T07:32:15+00:00", + "scanned_at": "2026-11-09T07:49:24+00:00", "target": ".", "tool": "python-agentic-audit", "user_config": { diff --git a/dev/tests/test_tool_pins.py b/dev/tests/test_tool_pins.py index 74718869..e3f2d80e 100644 --- a/dev/tests/test_tool_pins.py +++ b/dev/tests/test_tool_pins.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # Copyright (C) 2026 K. C. Ramakrishna -"""The dev tool versions live in three manifests only: ``dev/tools/pyproject.toml``, ``dev/tools/package.json`` and, -for osv-scanner, the Go module ``dev/tools/osv/go.mod``. +"""The dev tool versions live in the manifests in ``dev/tools/`` only: ``pyproject.toml``, ``package.json`` and one Go +module per Go tool (``osv/go.mod`` for osv-scanner, ``actionlint/go.mod`` for actionlint). Dependabot reads those files and their lockfiles (``.github/dependabot.yml``). A version that is also written somewhere else would not move with a Dependabot pull request, so these tests fail when a pin is not exact, when a @@ -12,6 +12,7 @@ from __future__ import annotations import datetime +import pathlib import json import re import shutil @@ -29,6 +30,11 @@ NODE_LOCK = TOOLS / "package-lock.json" GO_MOD = TOOLS / "osv" / "go.mod" GO_SUM = TOOLS / "osv" / "go.sum" +#: Each Go tool: its module folder, the module that provides it, and the command path of the `tool` directive. +GO_TOOLS = { + "osv-scanner": (TOOLS / "osv", "github.com/google/osv-scanner/v2", "cmd/osv-scanner"), + "actionlint": (TOOLS / "actionlint", "github.com/rhysd/actionlint", "cmd/actionlint"), +} OSV_CONFIG = TOOLS / "osv-scanner.toml" OSV_MODULE = "github.com/google/osv-scanner/v2" EXACT_GO = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+$") @@ -111,9 +117,9 @@ def test_package_lock_records_the_pins(self) -> None: self.assertEqual(lock["packages"][f"node_modules/{name}"]["version"], version) -def go_requirements() -> dict[str, str]: - """``module -> version`` for each ``require`` line of the osv-scanner tool module.""" - text = GO_MOD.read_text(encoding="utf-8") +def go_requirements(go_mod: pathlib.Path = GO_MOD) -> dict[str, str]: + """``module -> version`` for each ``require`` line of a Go tool module (default: osv-scanner's).""" + text = go_mod.read_text(encoding="utf-8") block = re.search(r"^require \((.*?)^\)", text, re.S | re.M) lines = block.group(1).splitlines() if block else [] lines += [m.group(1) for m in re.finditer(r"^require (\S+ \S+)", text, re.M)] @@ -142,6 +148,26 @@ def test_go_sum_holds_the_pinned_osv_scanner(self) -> None: self.assertIn(f"{OSV_MODULE} {version} h1:", GO_SUM.read_text(encoding="utf-8")) +class EachGoToolIsAPinnedToolRequirement(unittest.TestCase): + """Every Go tool has its own module: a `tool` directive, a tagged release, its checksum in go.sum.""" + + def test_each_go_tool_module(self) -> None: + for name, (folder, module, command) in GO_TOOLS.items(): + with self.subTest(tool=name): + go_mod = folder / "go.mod" + text = go_mod.read_text(encoding="utf-8") + self.assertRegex(text, rf"(?m)^tool {re.escape(module)}/{re.escape(command)}$") + version = go_requirements(go_mod).get(module, "") + self.assertRegex(version, EXACT_GO, f"{name} must be a tagged release, not a pseudo-version") + self.assertIn(f"{module} {version} h1:", (folder / "go.sum").read_text(encoding="utf-8")) + + def test_dependabot_watches_each_go_tool_module(self) -> None: + text = (REPO / ".github" / "dependabot.yml").read_text(encoding="utf-8") + for folder, _module, _command in GO_TOOLS.values(): + with self.subTest(folder=folder.name): + self.assertIn(f'"/{folder.relative_to(REPO).as_posix()}"', text) + + class AcceptedVulnerabilitiesAreReviewed(unittest.TestCase): """Each entry of dev/tools/osv-scanner.toml says why, and expires within six months.""" @@ -202,6 +228,11 @@ def test_tool_commands_carry_the_manifest_versions(self) -> None: self.assertIn(f"typos@{python['typos']}", output) self.assertIn(f"markdownlint-cli2@{node['markdownlint-cli2']}", output) + def test_workflow_lint_runs_actionlint_from_its_go_module(self) -> None: + output = self.dry_run("workflow-lint") + self.assertIn(f"go -C dev/tools/actionlint tool actionlint {REPO}/.github/workflows/", output) + self.assertNotIn("actionlint-py", output) + def test_check_runs_the_spell_check(self) -> None: self.assertIn(f"typos@{python_pins()['typos']}", self.dry_run("check")) @@ -209,7 +240,7 @@ def test_security_audits_every_dev_tool_lockfile(self) -> None: output = self.dry_run("security") self.assertIn("go -C dev/tools/osv tool osv-scanner scan source", output) self.assertIn(f"--config={REPO}/dev/tools/osv-scanner.toml", output) - for lockfile in (TOOL_LOCK, NODE_LOCK, GO_MOD): + for lockfile in (TOOL_LOCK, NODE_LOCK, *(folder / "go.mod" for folder, _m, _c in GO_TOOLS.values())): with self.subTest(lockfile=lockfile.name): self.assertIn(f"--lockfile={lockfile}", output) self.assertNotIn(go_requirements()[OSV_MODULE], output, "the Makefile must not write the osv-scanner version") diff --git a/dev/tools/actionlint/go.mod b/dev/tools/actionlint/go.mod new file mode 100644 index 00000000..dafae4b8 --- /dev/null +++ b/dev/tools/actionlint/go.mod @@ -0,0 +1,20 @@ +module github.com/Nistapp/codebase-agentic-readiness-framework/dev/tools/actionlint + +go 1.27.0 + +tool github.com/rhysd/actionlint/cmd/actionlint + +require ( + github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/fatih/color v1.19.0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.21 // indirect + github.com/mattn/go-shellwords v1.0.12 // indirect + github.com/rhysd/actionlint v1.7.12 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + go.yaml.in/yaml/v4 v4.0.0-rc.3 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.44.0 // indirect +) diff --git a/dev/tools/actionlint/go.sum b/dev/tools/actionlint/go.sum new file mode 100644 index 00000000..99ce4cbe --- /dev/null +++ b/dev/tools/actionlint/go.sum @@ -0,0 +1,27 @@ +github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= +github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.21 h1:jJKAZiQH+2mIinzCJIaIG9Be1+0NR+5sz/lYEEjdM8w= +github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= +github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/rhysd/actionlint v1.7.12 h1:vQ4GeJN86C0QH+gTUQcs8McmK62OLT3kmakPMtEWYnY= +github.com/rhysd/actionlint v1.7.12/go.mod h1:krOUhujIsJusovkaYzQ/VNH8PFexjNKqU0q5XI/4w+g= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go= +go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/dev/tools/pyproject.toml b/dev/tools/pyproject.toml index 3a49abc3..445f86b7 100644 --- a/dev/tools/pyproject.toml +++ b/dev/tools/pyproject.toml @@ -14,7 +14,6 @@ dependencies = [] [dependency-groups] tools = [ - "actionlint-py==1.7.12.25", "check-jsonschema==0.39.0", "coverage==7.17.2", "detect-secrets==1.5.0", diff --git a/dev/tools/uv.lock b/dev/tools/uv.lock index b7ddb81d..0fdf9936 100644 --- a/dev/tools/uv.lock +++ b/dev/tools/uv.lock @@ -6,12 +6,6 @@ resolution-markers = [ "python_full_version < '3.15'", ] -[[package]] -name = "actionlint-py" -version = "1.7.12.25" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/33/75/36ee89db39bb3620d1e46fe1f6ec77d6000d30ff45345699e51b4c850e83/actionlint_py-1.7.12.25.tar.gz", hash = "sha256:83f83be4cd317c79b15cbb7022787513a6994f00bfb2dd8da02fa4e62b4f640d", size = 12438, upload-time = "2026-09-22T16:24:42.477Z" } - [[package]] name = "ast-serialize" version = "0.13.1" @@ -273,7 +267,6 @@ source = { virtual = "." } [package.dev-dependencies] tools = [ - { name = "actionlint-py" }, { name = "check-jsonschema" }, { name = "coverage" }, { name = "detect-secrets" }, @@ -290,7 +283,6 @@ tools = [ [package.metadata.requires-dev] tools = [ - { name = "actionlint-py", specifier = "==1.7.12.25" }, { name = "check-jsonschema", specifier = "==0.39.0" }, { name = "coverage", specifier = "==7.17.2" }, { name = "detect-secrets", specifier = "==1.5.0" }, diff --git a/docs/architecture/adrs/0003-dependabot-security-pull-requests-only.md b/docs/architecture/adrs/0003-dependabot-security-pull-requests-only.md index a5480296..08dae07d 100644 --- a/docs/architecture/adrs/0003-dependabot-security-pull-requests-only.md +++ b/docs/architecture/adrs/0003-dependabot-security-pull-requests-only.md @@ -41,14 +41,15 @@ GitHub's documentation (the Dependabot options reference and "Configuring Depend 3. **One pull request per ecosystem.** A `security-fixes` group joins the minor and patch security fixes. 4. **No `target-branch`.** The default branch is `dev` ([ADR 0002](0002-dev-integrates-main-releases.md)), so every Dependabot pull request goes to `dev` without it. -5. **The dev tools are visible to Dependabot.** Their versions move to three manifests in `dev/tools/`, each with a +5. **The dev tools are visible to Dependabot.** Their versions move to manifests in `dev/tools/`, each with a lockfile: | Manifest | Lockfile | Tools | Dependabot ecosystem | |---|---|---|---| - | [`pyproject.toml`](../../../dev/tools/pyproject.toml) (group `tools`) | `uv.lock` | actionlint-py, check-jsonschema, coverage, detect-secrets, jsonschema, mypy, ruff, skills-ref, typos, uv, zizmor | `uv` | + | [`pyproject.toml`](../../../dev/tools/pyproject.toml) (group `tools`) | `uv.lock` | check-jsonschema, coverage, detect-secrets, jsonschema, mypy, ruff, skills-ref, typos, uv, zizmor | `uv` | | [`package.json`](../../../dev/tools/package.json) | `package-lock.json` | markdownlint-cli2, mermaid-cli | `npm` | | [`osv/go.mod`](../../../dev/tools/osv/go.mod) (a `tool` directive) | `osv/go.sum` | osv-scanner ([ADR 0007](0007-security-verb-audits-dependencies-with-osv-scanner.md)) | `gomod` | + | [`actionlint/go.mod`](../../../dev/tools/actionlint/go.mod) (a `tool` directive) | `actionlint/go.sum` | actionlint | `gomod` | Nothing installs these projects. The [`Makefile`](../../../Makefile), [`lefthook.yml`](../../../lefthook.yml), [`dev/mermaid_check.py`](../../../dev/mermaid_check.py) and the workflows (`setup-uv` `version-file`) read each diff --git a/docs/architecture/adrs/0007-security-verb-audits-dependencies-with-osv-scanner.md b/docs/architecture/adrs/0007-security-verb-audits-dependencies-with-osv-scanner.md index 8c20835f..24eb1028 100644 --- a/docs/architecture/adrs/0007-security-verb-audits-dependencies-with-osv-scanner.md +++ b/docs/architecture/adrs/0007-security-verb-audits-dependencies-with-osv-scanner.md @@ -27,7 +27,8 @@ version updates. Three have no fix that reaches this repository and are accepted 1. **osv-scanner runs inside `make security`.** After the secrets scan, the [`Makefile`](../../../Makefile) runs osv-scanner on every dev-tool lockfile: `dev/tools/uv.lock`, - `dev/tools/package-lock.json` and `dev/tools/osv/go.mod`. The tool name is written in the recipe, so `TOOL-04` + `dev/tools/package-lock.json` and the Go tool modules (`dev/tools/osv/go.mod`, `dev/tools/actionlint/go.mod`). + The tool name is written in the recipe, so `TOOL-04` sees it. 2. **osv-scanner is a Go `tool` requirement.** [`dev/tools/osv/go.mod`](../../../dev/tools/osv/go.mod) pins a tagged release; `go.sum` holds the checksums. `go -C dev/tools/osv tool osv-scanner` builds and runs it. The @@ -39,7 +40,9 @@ version updates. Three have no fix that reaches this repository and are accepted date passes, the scan fails again. 5. **No offline switch.** `make security` needs the network (osv.dev). CI has it; a contributor without it cannot run the gate. -6. **Contributors need Go 1.27 or newer.** `make setup` checks for it. CI installs it with `actions/setup-go` from +6. **Contributors need Go 1.27 or newer.** `make setup` checks for it. Since 2026-11-09 actionlint is a Go tool + too (`make check`): its PyPI wrapper downloaded the binary from GitHub releases on each install, and a GitHub + outage failed CI. CI installs it with `actions/setup-go` from `go.mod`; the release job does so without a cache. --- diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 087bbd68..b03cd3d3 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -17,7 +17,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | coverage.py | Test coverage below the floor in `.coveragerc` | `make coverage` | `coverage` | `dev/tools/pyproject.toml` | | jsonschema | The tools' schema validator disagrees with the reference package | `make crosscheck` | `check` | `dev/tools/pyproject.toml` | | `dev/recipe_lint.py` | Broken links, bad `SKILL.md` frontmatter, oversize files, unknown tools, check IDs and requirement IDs | `make recipe-lint` | `check` | This repository | -| actionlint | Workflow mistakes: syntax, expressions, action inputs, shellcheck on `run:` blocks | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` (`actionlint-py`) | +| actionlint | Workflow mistakes: syntax, expressions, action inputs, shellcheck on `run:` blocks | `make workflow-lint` | `check` | `dev/tools/actionlint/go.mod` | | zizmor | Workflow and Dependabot security: injection, broad permissions, unpinned actions, cooldown. Runs at the pedantic persona, so a permission without a comment or a missing concurrency limit fails too | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | | typos | Misspelled words in the tracked text, code identifiers included (word list: [`_typos.toml`](../../_typos.toml)) ([Spell check](#spell-check)) | `make spell`; `make spell-fix` writes the fixes | `check` | `dev/tools/pyproject.toml` | | check-jsonschema | `.github/workflows/*.yml` and `.github/dependabot.yml` that do not match their SchemaStore schemas | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | @@ -35,7 +35,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | release-please | Computes the next version from the commit subjects, writes `CHANGELOG.md`, tags and creates the GitHub release (`release-please-config.json`) | — | `release-please.yml`, on each push to `main` | `.github/workflows/release-please.yml` (action pinned by SHA) | | uv | Runs the Python tools through `uvx` and `uv run` | — | All jobs | `dev/tools/pyproject.toml` | | Node | Runs the Node tools through `npx` | — | `docs` | `.nvmrc` | -| Go | Builds and runs osv-scanner through `go tool` | — | `security` | `dev/tools/osv/go.mod` (`go` line) | +| Go | Builds and runs actionlint and osv-scanner through `go tool` | — | `check`, `security` | the `go` line of each Go tool module | `make check` runs `format:check`, `typecheck`, `test`, the engine's rule check, `crosscheck`, `recipe-lint`, `workflow-lint` and `spell`, in sequence. CI's `CI Gate` passes only when the `check`, `security`, `coverage`, `self-audit`, @@ -57,9 +57,9 @@ To fix a finding, do one of these: ## Dependency audit `make security` runs osv-scanner after the secrets scan -([ADR 0007](../architecture/adrs/0007-security-verb-audits-dependencies-with-osv-scanner.md)). It reads the three -dev-tool lockfiles and asks osv.dev for known vulnerabilities. It needs Go 1.27 or newer and the network. The first -run builds osv-scanner, which takes about 30 seconds. +([ADR 0007](../architecture/adrs/0007-security-verb-audits-dependencies-with-osv-scanner.md)). It reads the +dev-tool lockfiles (Python, Node and each Go tool module) and asks osv.dev for known vulnerabilities. It needs Go +1.27 or newer and the network. The first run builds osv-scanner, which takes about 30 seconds. The scan fails when it finds a vulnerability that [`dev/tools/osv-scanner.toml`](../../dev/tools/osv-scanner.toml) does not accept. The output has one table row per vulnerability: the OSV link, the package, its version, the fixed @@ -106,7 +106,8 @@ Each tool version is written once: - Python tools: `dev/tools/pyproject.toml`, group `tools`, locked in `dev/tools/uv.lock`. - Node tools: `dev/tools/package.json`, locked in `dev/tools/package-lock.json`. -- osv-scanner: `dev/tools/osv/go.mod`, a Go `tool` requirement, with checksums in `dev/tools/osv/go.sum`. +- Go tools: one module each, with a Go `tool` requirement and checksums in its `go.sum`: osv-scanner in + `dev/tools/osv/`, actionlint in `dev/tools/actionlint/`. The Go module proxy serves them. The `Makefile`, `lefthook.yml`, `dev/mermaid_check.py` and the workflows read each version from those files. Nothing is installed into the project: the tools run through `uvx`, `npx` and `go tool`. Dependabot watches these diff --git a/docs/reference/self-audit.md b/docs/reference/self-audit.md index e116e70b..3fe21bf2 100644 --- a/docs/reference/self-audit.md +++ b/docs/reference/self-audit.md @@ -57,7 +57,7 @@ Every check that is not `PASS`, with the reason it is open. | `EXEC-01` | FAIL | BLOCKER | **Owner exception** (2026-11-09): a false positive. osv-scanner's Go module (`dev/tools/osv/go.mod`) pins Go with its `go` directive, but the check reads that directive only from a root `go.mod`. An engine fix is planned (v0.3.3 Track C) | | `BASE-02` | PARTIAL | DEGRADER | The T5 playbook was applied (D32): `.quality-baseline.json` holds the counts (ruff 0, mypy 0, Markdown lint outside the ratchet 735) and `make format-check`, `make typecheck` and `make docs-lint` fail when one goes up. The check still reads PARTIAL by design: it credits only a numeric budget flag (`--max-warnings N`), and none of these tools has one. Engine issue [#45](https://github.com/Nistapp/codebase-agentic-readiness-framework/issues/45) tracks it; the coverage floor (91) is unchanged and `BASE-04` passes | | `EXEC-07` | FAIL | DEGRADER | **Owner exception** (2026-11-09): `mypy.ini` and `pyrightconfig.json` both configure types. `make typecheck` (mypy strict) is the gate; `pyrightconfig.json` only gives a language server the same import paths, so an agent gets definitions and type errors without hundreds of unresolved imports. `dev/tests/test_pyright_config.py` keeps the paths in step. The check cannot read that delegation ([Set up the agent tools](../how-to/set-up-agent-tools.md#optional-a-language-server)) | -| `EXEC-02` | PARTIAL | BLOCKER | No finding: the three lockfiles of the pinned dev tools (`dev/tools/uv.lock`, `dev/tools/package-lock.json`, `dev/tools/osv/go.sum`) cannot be checked against their manifests without running the package managers. `dev/tests/test_tool_pins.py` checks that they agree; the engine does not read that test | +| `EXEC-02` | PARTIAL | BLOCKER | No finding: the lockfiles of the pinned dev tools (`dev/tools/uv.lock`, `dev/tools/package-lock.json` and the `go.sum` of each Go tool module, `dev/tools/osv/` and `dev/tools/actionlint/`) cannot be checked against their manifests without running the package managers. `dev/tests/test_tool_pins.py` checks that they agree; the engine does not read that test | | `NAV-05` | PARTIAL | COSMETIC | No finding, informational and unscored (D34): 40% of public symbols documented (476 of 1188). Reviewed on 2026-11-09 and left as it is: a ruff `D1` ratchet would baseline about 700 violations for a cosmetic check, and mypy strict and the tests carry the meaning | | `IDX-02`, `IDX-03`, `TST-02`, `CON-04` | UNKNOWN | mixed | Need a probe, the user-level index store, or `--emit-baseline`; not run here | From 8b64cc18e65e980a7737f2078c44eb386a2049eb Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 13:42:10 +0530 Subject: [PATCH 030/111] fix(dev): keep each osv-scanner ignore file next to the lockfiles it covers OpenSSF Scorecard reads the osv-scanner.toml next to each lockfile. The single file in dev/tools/ covered uv.lock and package-lock.json, but not dev/tools/osv/go.mod, so Scorecard raised a Vulnerabilities alert for GO-2026-5932, which make security already accepts. The x/crypto entry moves to dev/tools/osv/osv-scanner.toml, and make security passes no --config, so osv-scanner reads the files the same way Scorecard does. The tests check every ignore file and that no --config comes back. Co-Authored-By: Claude Opus 5.5 --- Makefile | 5 +++-- dev/tests/test_tool_pins.py | 20 +++++++++++++------ dev/tools/osv-scanner.toml | 9 +++------ dev/tools/osv/osv-scanner.toml | 10 ++++++++++ ...rb-audits-dependencies-with-osv-scanner.md | 8 +++++--- docs/reference/dev-tooling.md | 13 ++++++++---- 6 files changed, 44 insertions(+), 21 deletions(-) create mode 100644 dev/tools/osv/osv-scanner.toml diff --git a/Makefile b/Makefile index 4324c0ef..1c74c15a 100644 --- a/Makefile +++ b/Makefile @@ -193,12 +193,13 @@ check: format-check typecheck test verify-rules crosscheck recipe-lint workflow- # No third-party runtime dependencies exist to audit, so the gate is a secrets scan. The baseline # lists the deliberate key-shaped strings in the fixtures and tests; a new hit fails the scan. # The dependency audit: osv-scanner, pinned in dev/tools/osv/go.mod (a Go `tool` directive, so Dependabot sees it), -# reads the dev tools' lockfiles and fails on a known vulnerability that dev/tools/osv-scanner.toml does not accept. +# reads the dev tools' lockfiles and fails on a known vulnerability that no osv-scanner.toml accepts. Without +# --config, osv-scanner reads the osv-scanner.toml next to each lockfile, the same file OpenSSF Scorecard reads. # --no-call-analysis=go: the module holds no Go code of its own, so govulncheck has nothing to analyse. # The audit engine's TOOL-04 reads this recipe and needs the tool name written out, not hidden in a variable. security: git ls-files -z | xargs -0 $(DETECT_SECRETS_HOOK) --baseline .secrets.baseline - go -C dev/tools/osv tool osv-scanner scan source --no-call-analysis=go --config=$(CURDIR)/dev/tools/osv-scanner.toml $(foreach f,$(DEP_LOCKFILES),--lockfile=$(CURDIR)/$(f)) + go -C dev/tools/osv tool osv-scanner scan source --no-call-analysis=go $(foreach f,$(DEP_LOCKFILES),--lockfile=$(CURDIR)/$(f)) # Run this after a deliberate change to a fixture or test that holds a key-shaped string, then review the diff. update-secrets-baseline: diff --git a/dev/tests/test_tool_pins.py b/dev/tests/test_tool_pins.py index e3f2d80e..d65aa55b 100644 --- a/dev/tests/test_tool_pins.py +++ b/dev/tests/test_tool_pins.py @@ -35,7 +35,8 @@ "osv-scanner": (TOOLS / "osv", "github.com/google/osv-scanner/v2", "cmd/osv-scanner"), "actionlint": (TOOLS / "actionlint", "github.com/rhysd/actionlint", "cmd/actionlint"), } -OSV_CONFIG = TOOLS / "osv-scanner.toml" +#: The ignore files: one next to the lockfiles it covers, as osv-scanner and OpenSSF Scorecard look for them. +OSV_CONFIGS = (TOOLS / "osv-scanner.toml", TOOLS / "osv" / "osv-scanner.toml") OSV_MODULE = "github.com/google/osv-scanner/v2" EXACT_GO = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+$") #: The longest an accepted vulnerability may stay accepted before someone looks again. @@ -169,14 +170,20 @@ def test_dependabot_watches_each_go_tool_module(self) -> None: class AcceptedVulnerabilitiesAreReviewed(unittest.TestCase): - """Each entry of dev/tools/osv-scanner.toml says why, and expires within six months.""" + """Each entry of each osv-scanner.toml says why, and expires within six months.""" def entries(self) -> list[dict[str, object]]: - data = tomllib.loads(OSV_CONFIG.read_text(encoding="utf-8")) - self.assertEqual(set(data) - {"IgnoredVulns"}, set(), "only IgnoredVulns belongs in the config") - entries: list[dict[str, object]] = data.get("IgnoredVulns", []) + entries: list[dict[str, object]] = [] + for config in OSV_CONFIGS: + data = tomllib.loads(config.read_text(encoding="utf-8")) + self.assertEqual(set(data) - {"IgnoredVulns"}, set(), f"{config}: only IgnoredVulns belongs here") + entries.extend(data.get("IgnoredVulns", [])) return entries + def test_every_ignore_file_sits_next_to_an_audited_lockfile(self) -> None: + found = sorted(p.relative_to(REPO).as_posix() for p in TOOLS.rglob("osv-scanner.toml")) + self.assertEqual(found, sorted(c.relative_to(REPO).as_posix() for c in OSV_CONFIGS)) + def test_each_entry_has_an_id_a_reason_and_a_date(self) -> None: for entry in self.entries(): with self.subTest(entry=entry.get("id")): @@ -239,7 +246,8 @@ def test_check_runs_the_spell_check(self) -> None: def test_security_audits_every_dev_tool_lockfile(self) -> None: output = self.dry_run("security") self.assertIn("go -C dev/tools/osv tool osv-scanner scan source", output) - self.assertIn(f"--config={REPO}/dev/tools/osv-scanner.toml", output) + # No --config: osv-scanner reads the osv-scanner.toml next to each lockfile, as Scorecard does. + self.assertNotIn("--config", output) for lockfile in (TOOL_LOCK, NODE_LOCK, *(folder / "go.mod" for folder, _m, _c in GO_TOOLS.values())): with self.subTest(lockfile=lockfile.name): self.assertIn(f"--lockfile={lockfile}", output) diff --git a/dev/tools/osv-scanner.toml b/dev/tools/osv-scanner.toml index 55314ee5..2012095b 100644 --- a/dev/tools/osv-scanner.toml +++ b/dev/tools/osv-scanner.toml @@ -1,7 +1,9 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # Copyright (C) 2026 K. C. Ramakrishna # -# Known vulnerabilities that `make security` (osv-scanner) accepts for now. Every other one fails the gate. +# Known vulnerabilities that `make security` (osv-scanner) accepts for now, for the lockfiles in this folder +# (uv.lock, package-lock.json). Every other one fails the gate. osv-scanner and OpenSSF Scorecard both read the +# osv-scanner.toml next to a lockfile, so each Go tool module has its own (dev/tools//osv-scanner.toml). # Each entry needs an `id`, an `ignoreUntil` date at most six months ahead and a `reason` that says why the # vulnerability cannot reach this repository or why no fix exists yet. dev/tests/test_tool_pins.py checks this. # When `ignoreUntil` passes, the scan fails again: fix it or renew the entry with a new reason. @@ -16,8 +18,3 @@ reason = "re2js 2.8.6: quadratic-time lookbehind matching. markdownlint 0.42.0 p id = "GHSA-238p-pmpm-9mq7" ignoreUntil = 2027-02-09 reason = "katex 0.16.47: prototype pollution bypasses trust restrictions when KaTeX renders. Reached only through markdownlint 0.42.0 (micromark-extension-math), which parses Markdown and renders no HTML. No markdownlint release raises it yet." - -[[IgnoredVulns]] -id = "GO-2026-5932" -ignoreUntil = 2027-02-09 -reason = "golang.org/x/crypto: the advisory covers the deprecated openpgp package in every version; no tagged release fixes it. The osv-scanner binary does not link golang.org/x/crypto/openpgp (checked with go list -deps on 2026-11-09)." diff --git a/dev/tools/osv/osv-scanner.toml b/dev/tools/osv/osv-scanner.toml new file mode 100644 index 00000000..45a38c34 --- /dev/null +++ b/dev/tools/osv/osv-scanner.toml @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna +# +# Known vulnerabilities that `make security` (osv-scanner) accepts for now, for this folder's go.mod (osv-scanner's +# own dependencies). The rules for an entry are in dev/tools/osv-scanner.toml; dev/tests/test_tool_pins.py checks them. + +[[IgnoredVulns]] +id = "GO-2026-5932" +ignoreUntil = 2027-02-09 +reason = "golang.org/x/crypto: the advisory covers the deprecated openpgp package in every version; no tagged release fixes it. The osv-scanner binary does not link golang.org/x/crypto/openpgp (checked with go list -deps on 2026-11-09)." diff --git a/docs/architecture/adrs/0007-security-verb-audits-dependencies-with-osv-scanner.md b/docs/architecture/adrs/0007-security-verb-audits-dependencies-with-osv-scanner.md index 24eb1028..bc015f2a 100644 --- a/docs/architecture/adrs/0007-security-verb-audits-dependencies-with-osv-scanner.md +++ b/docs/architecture/adrs/0007-security-verb-audits-dependencies-with-osv-scanner.md @@ -34,10 +34,12 @@ version updates. Three have no fix that reaches this repository and are accepted tagged release; `go.sum` holds the checksums. `go -C dev/tools/osv tool osv-scanner` builds and runs it. The version is written once, and Dependabot's `gomod` entry sees it, under the policy of ADR 0003. 3. **A known vulnerability fails the gate.** osv-scanner exits non-zero, so `make security` and `CI Gate` fail. -4. **Exceptions are reviewed and expire.** [`dev/tools/osv-scanner.toml`](../../../dev/tools/osv-scanner.toml) - lists each accepted vulnerability with an `id`, a `reason` and an `ignoreUntil` date at most six months ahead. +4. **Exceptions are reviewed and expire.** An `osv-scanner.toml` next to the lockfiles it covers + ([`dev/tools/osv-scanner.toml`](../../../dev/tools/osv-scanner.toml), `dev/tools/osv/osv-scanner.toml`) lists each + accepted vulnerability with an `id`, a `reason` and an `ignoreUntil` date at most six months ahead. [`test_tool_pins.py`](../../../dev/tests/test_tool_pins.py) checks the format and the six-month limit. When the - date passes, the scan fails again. + date passes, the scan fails again. `make security` passes no `--config`, so osv-scanner reads these files the way + OpenSSF Scorecard does. 5. **No offline switch.** `make security` needs the network (osv.dev). CI has it; a contributor without it cannot run the gate. 6. **Contributors need Go 1.27 or newer.** `make setup` checks for it. Since 2026-11-09 actionlint is a Go tool diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index b03cd3d3..d0d2501d 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -61,13 +61,18 @@ To fix a finding, do one of these: dev-tool lockfiles (Python, Node and each Go tool module) and asks osv.dev for known vulnerabilities. It needs Go 1.27 or newer and the network. The first run builds osv-scanner, which takes about 30 seconds. -The scan fails when it finds a vulnerability that [`dev/tools/osv-scanner.toml`](../../dev/tools/osv-scanner.toml) -does not accept. The output has one table row per vulnerability: the OSV link, the package, its version, the fixed -version and the lockfile. To fix a finding, do one of these: +The scan fails when it finds a vulnerability that no `osv-scanner.toml` accepts. Each ignore file covers the +lockfiles in its own folder: [`dev/tools/osv-scanner.toml`](../../dev/tools/osv-scanner.toml) for `uv.lock` and +`package-lock.json`, and `dev/tools//osv-scanner.toml` for a Go tool module. OpenSSF Scorecard reads the same +files, so an accepted vulnerability raises no Scorecard alert either. + +The output has one table row per vulnerability: the OSV link, the package, its version, the fixed version and the +lockfile. To fix a finding, do one of these: 1. Raise the tool or the dependency to the fixed version. Follow [Change a tool version](../../RELEASE_PROCESS.md#change-a-tool-version). -2. If no fix reaches this repository, add an entry to `osv-scanner.toml`. Give the `id` and a `reason`. +2. If no fix reaches this repository, add an entry to the `osv-scanner.toml` next to the lockfile. Give the `id` and + a `reason`. Set `ignoreUntil` to a date at most six months ahead. `dev/tests/test_tool_pins.py` checks the entry. When an `ignoreUntil` date passes, the scan fails again. Look at the advisory again, then fix or renew it. From 818676d17fb7819e21b68adc1e22a61a7884ee69 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 13:46:28 +0530 Subject: [PATCH 031/111] ci: publish the Scorecard results and record how each alert is handled publish_results is now true (the maintainer's decision): the workflow signs each result and sends it to the public OpenSSF API, which backs the score page on scorecard.dev and the new badge in README.md. The signing needs id-token: write, on the Scorecard job only. branch-protection.md gains "Scorecard alerts": each of the 7 checks that scored below 10 on the first run, why, and how it is handled (fixed, dismissed with a reason, or left open until it clears). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/scorecard.yml | 8 +++++--- README.md | 2 ++ docs/reference/branch-protection.md | 24 ++++++++++++++++++++++-- docs/reference/dev-tooling.md | 2 +- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 1530b8ca..9fed58f2 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -4,8 +4,9 @@ # OpenSSF Scorecard: scores this repository's security practices (branch protection, pinned actions, token # permissions, signed releases, ...) and uploads the result to the Security tab as code-scanning alerts. Advisory: # no ruleset requires it (docs/reference/branch-protection.md, "Code scanning"). -# `publish_results: false`: nothing is sent to the public OpenSSF API, so there is no public score page or badge. -# Turning it on is the maintainer's decision; it also needs `id-token: write` on the job. +# `publish_results: true` (the maintainer's decision, 2026-11-09): the action signs the result and sends it to the +# public OpenSSF API, which backs the score page on scorecard.dev and the badge in README.md. The signing needs +# `id-token: write`, on this job only; the job runs nothing but checkout, Scorecard and the SARIF upload. # Scorecard runs on the default branch only (`dev`). name: Scorecard @@ -28,6 +29,7 @@ jobs: permissions: contents: read # check out the code and read the repository's settings through the API security-events: write # upload the result to the Security tab + id-token: write # sign the published result (publish_results), through GitHub's OIDC token steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -36,7 +38,7 @@ jobs: with: results_file: results.sarif results_format: sarif - publish_results: false + publish_results: true - uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: sarif_file: results.sarif diff --git a/README.md b/README.md index 7be6c535..e596289d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Codebase Agentic Readiness Framework +[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/Nistapp/codebase-agentic-readiness-framework/badge)](https://scorecard.dev/viewer/?uri=github.com/Nistapp/codebase-agentic-readiness-framework) + > *"The more you sweat in peace, the less you bleed in war."* Preparing a codebase for AI coding agents is the sweat; > agents changing it from user prompts is the war. diff --git a/docs/reference/branch-protection.md b/docs/reference/branch-protection.md index 56c55d52..f60b86cc 100644 --- a/docs/reference/branch-protection.md +++ b/docs/reference/branch-protection.md @@ -109,8 +109,10 @@ not scan JavaScript or TypeScript, because the only such files are those fixture [`.github/workflows/scorecard.yml`](../../.github/workflows/scorecard.yml) runs OpenSSF Scorecard on each push to `dev` and every Monday. It scores the repository's security practices, such as pinned actions, token permissions and signed releases. Each low score shows as an alert in the Security tab, in the `scorecard` category. The results are -not published: `publish_results` is `false`, so there is no public score page and no badge. Publishing is the -maintainer's decision. +also published (`publish_results: true`, the maintainer's decision on 2026-11-09). The workflow signs each result and +sends it to the public OpenSSF API. The latest score is on +[scorecard.dev](https://scorecard.dev/viewer/?uri=github.com/Nistapp/codebase-agentic-readiness-framework) and in the +badge at the top of [`README.md`](../../README.md). How each alert is handled: [Scorecard alerts](#scorecard-alerts). Code scanning is advisory. No ruleset requires the CodeQL or the Scorecard workflow, and an alert does not block a merge. The @@ -122,6 +124,24 @@ maintainer handles an alert in the Security tab: A later change may add the ruleset rule "Require code scanning results" to `dev-protection.json`. That change needs the maintainer. +### Scorecard alerts + +The first Scorecard run on `dev` (2026-11-09) scored 7 checks below 10. Each one raised an alert. The table gives +each check, what it measures, why the score is low, and how it is handled. Scorecard rates each check 0 to 10. + +| Check | Score | What it measures | Why the score is low | Handling | +| --- | --: | --- | --- | --- | +| Branch-Protection | 3 | The rules on `dev` and `main` | No approving review is required. "Dismiss stale reviews", "require review from code owners" and "require approval of the most recent push" are off; each only acts when a review is required. `main` does not require an up-to-date branch, on purpose (see [Rules](#rules)) | Open. A solo maintainer cannot approve their own pull request, so a required review would block every merge. Revisit when a second maintainer joins | +| Code-Review | 0 | Whether recent changes had an approving review | 0 of the last 4 merged changes had one: the maintainer merges their own pull requests | Dismissed as "won't fix", for the same reason | +| Maintained | 0 | Recent commits and issue activity | The repository is younger than 90 days. Scorecard scores every new repository 0 | Open. It clears by itself after 90 days | +| SAST | 8 | Whether a code scanner runs on every commit | Only 13 of the last 30 commits were scanned, because CodeQL started on 2026-11-09 | Open. It rises as CodeQL scans new commits | +| Vulnerabilities | 9 | Known vulnerabilities in the dependencies, from OSV | `GO-2026-5932` in osv-scanner's Go module. `make security` accepts it, but its ignore file was not next to that `go.mod` | Fixed: each `osv-scanner.toml` now sits next to its lockfiles ([Dependency audit](dev-tooling.md#dependency-audit)) | +| Fuzzing | 0 | Whether a fuzzer runs on the code | No fuzzer | Dismissed as "won't fix". The engine and tools are standard-library Python that reads files; their tests and fixtures cover the inputs | +| CII-Best-Practices | 0 | An OpenSSF Best Practices badge | None applied for | Open. The badge is a self-assessment on [bestpractices.dev](https://www.bestpractices.dev/); optional | + +The other checks score 10, or do not apply. The full, current list is on the +[score page](https://scorecard.dev/viewer/?uri=github.com/Nistapp/codebase-agentic-readiness-framework). + ## Current state | Ruleset | Target | State on GitHub | diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index d0d2501d..11b98731 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -31,7 +31,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | The audit engine | A readiness finding that `.audit-baseline.json` does not have | `make self-audit` | `self-audit` | This repository | | actions/attest | Signs build provenance for the release files; users check it with `gh attestation verify` ([Install the skills](../how-to/install-the-skills.md#step-2b-optional-check-the-build-provenance)) | — | `release.yml`, release only | `.github/workflows/release.yml` (action pinned by SHA) | | CodeQL | Security flaws in the Python code and the workflows. Advisory: alerts go to the Security tab ([Code scanning](branch-protection.md#code-scanning)) | — | `codeql.yml`, outside `CI Gate` | `.github/workflows/codeql.yml` (action pinned by SHA) | -| OpenSSF Scorecard | Weak security practices of the repository (Branch-Protection, Pinned-Dependencies, Token-Permissions, Signed-Releases, ...). Advisory: alerts go to the Security tab; results are not published ([Code scanning](branch-protection.md#code-scanning)) | — | `scorecard.yml`, on push to `dev` and weekly | `.github/workflows/scorecard.yml` (action pinned by SHA) | +| OpenSSF Scorecard | Weak security practices of the repository (Branch-Protection, Pinned-Dependencies, Token-Permissions, Signed-Releases, ...). Advisory: alerts go to the Security tab; the score is published on scorecard.dev and in the README badge ([Scorecard alerts](branch-protection.md#scorecard-alerts)) | — | `scorecard.yml`, on push to `dev` and weekly | `.github/workflows/scorecard.yml` (action pinned by SHA) | | release-please | Computes the next version from the commit subjects, writes `CHANGELOG.md`, tags and creates the GitHub release (`release-please-config.json`) | — | `release-please.yml`, on each push to `main` | `.github/workflows/release-please.yml` (action pinned by SHA) | | uv | Runs the Python tools through `uvx` and `uv run` | — | All jobs | `dev/tools/pyproject.toml` | | Node | Runs the Node tools through `npx` | — | `docs` | `.nvmrc` | From 7172596bef8ea5df8674b3105e0cdf03bc2d4a14 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 13:58:14 +0530 Subject: [PATCH 032/111] feat(engine): add EXEC-08, tool versions visible to dependency automation Dependabot and Renovate read manifests, not commands. A tool version typed into a Makefile, a workflow or a hook gets no security alert and no update. EXEC-08 (Phase 1, DEGRADER, REQ-SH-058) reads the command runners, the CI workflows and the hook files for a literal version after a launcher (npx, pnpm dlx, uvx, uv run --with, pipx, pip install, go install, cargo install) or in a tool-installer action's version input. Any literal no manifest holds is FAIL; every literal also in a manifest is PARTIAL, as the copy drifts; none is PASS. A file a Renovate custom manager reads is visible. No such file is UNKNOWN. Ruleset 2026-11-09.3, catalogue 0.4.0 with since_ruleset on EXEC-08, so an older engine's report imports it as UNKNOWN. Two one-defect fixtures with eval expectations; implement theme T2 owns EXEC-08. The realistic drafts, the examples and the example implement state are regenerated, and the tools' pinned counts move by one. Self-audit: EXEC-08 PASS. Roadmap-Step: C3 Co-Authored-By: Claude Opus 5.5 --- .secrets.baseline | 33 +- audit/catalogue.json | 22 +- audit/checks/09-execution-determinism.md | 32 ++ audit/engine/audit/__init__.py | 2 +- audit/engine/audit/rules/checks/execution.py | 317 +++++++++++++++++- audit/engine/audit/rules/registry.py | 1 + .../02-check-catalogue.md | 7 +- .../user-overview/03-why-these-checks.md | 1 + audit/engine/docs/reference/changelog.md | 1 + audit/engine/tests/fixtures/README.md | 6 +- .../exec-08-duplicate-pin/package.json | 11 + .../exec-08-inline-version/package.json | 8 + .../tests/test_exec08_inline_versions.py | 126 +++++++ audit/engine/tests/test_fixture_matrix.py | 2 + audit/engine/tests/test_hyg_pack.py | 4 +- audit/engine/tests/test_registry_status.py | 6 +- audit/engine/tests/test_smoke.py | 4 +- ...service.audit-report.20261010T094207Z.json | 2 +- ...e-service.audit-report.20261010T094207Z.md | 2 +- audit/schema/examples/minimal.json | 2 +- audit/schema/examples/minimal.md | 2 +- ...ess-state.example-service.in-progress.json | 4 +- ...iness-state.example-service.in-progress.md | 2 +- .../readiness-state.example-service.json | 4 +- .../readiness-state.example-service.md | 2 +- ...ntic-tdd.audit-draft.20261010T065738Z.json | 20 +- ...tant-sdk.audit-draft.20261010T065739Z.json | 20 +- ...t-server.audit-draft.20261010T065739Z.json | 20 +- audit/tools/tests/test_import_engine.py | 43 ++- .../tools/tests/test_import_fixture_matrix.py | 4 +- audit/tools/tests/test_render_report.py | 6 +- audit/tools/tests/test_report.py | 26 +- audit/tools/tests/test_validate_report.py | 2 +- .../fixtures/exec-08-duplicate-pin.json | 14 + .../fixtures/exec-08-inline-version.json | 14 + .../how-the-checks-map-to-the-framework.md | 5 +- implement/playbooks/README.md | 2 +- implement/tools/progress.py | 2 +- spec/readiness-requirements.md | 10 + spec/traceability.md | 1 + 40 files changed, 715 insertions(+), 77 deletions(-) create mode 100644 audit/engine/tests/fixtures/exec-08-duplicate-pin/package.json create mode 100644 audit/engine/tests/fixtures/exec-08-inline-version/package.json create mode 100644 audit/engine/tests/test_exec08_inline_versions.py create mode 100644 dev/eval/expected/fixtures/exec-08-duplicate-pin.json create mode 100644 dev/eval/expected/fixtures/exec-08-inline-version.json diff --git a/.secrets.baseline b/.secrets.baseline index 262edb05..e77ef7fe 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -143,6 +143,15 @@ "line_number": 1540 } ], + "audit/catalogue.json": [ + { + "type": "Hex High Entropy String", + "filename": "audit/catalogue.json", + "hashed_secret": "b25e08dbffeee64044e9425caff21cd9087c55d1", + "is_verified": false, + "line_number": 3 + } + ], "audit/engine/audit/rules/checks/con.py": [ { "type": "Secret Keyword", @@ -270,7 +279,7 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/readiness-state.example-service.in-progress.json", - "hashed_secret": "9ba66d1f640d58e98891d9e7f69511c089f914e4", + "hashed_secret": "1396ee91052c2457767d76ce883d3c222ef52651", "is_verified": false, "line_number": 17 } @@ -279,7 +288,7 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/readiness-state.example-service.json", - "hashed_secret": "9ba66d1f640d58e98891d9e7f69511c089f914e4", + "hashed_secret": "1396ee91052c2457767d76ce883d3c222ef52651", "is_verified": false, "line_number": 17 } @@ -1132,6 +1141,24 @@ "line_number": 4 } ], + "dev/eval/expected/fixtures/exec-08-duplicate-pin.json": [ + { + "type": "Hex High Entropy String", + "filename": "dev/eval/expected/fixtures/exec-08-duplicate-pin.json", + "hashed_secret": "21ef00fd108e084674f8969c8924d40c5b7af083", + "is_verified": false, + "line_number": 4 + } + ], + "dev/eval/expected/fixtures/exec-08-inline-version.json": [ + { + "type": "Hex High Entropy String", + "filename": "dev/eval/expected/fixtures/exec-08-inline-version.json", + "hashed_secret": "a29d0abf2f01a822c45f6505f11b20bc8fc0b5d2", + "is_verified": false, + "line_number": 4 + } + ], "dev/eval/expected/fixtures/exec-ok.json": [ { "type": "Hex High Entropy String", @@ -1476,5 +1503,5 @@ } ] }, - "generated_at": "2026-11-09T07:42:18Z" + "generated_at": "2026-11-09T08:21:47Z" } diff --git a/audit/catalogue.json b/audit/catalogue.json index d7446283..58f77d82 100644 --- a/audit/catalogue.json +++ b/audit/catalogue.json @@ -1,6 +1,6 @@ { - "catalogue_version": "0.3.0", - "fingerprint": "214aa1d1a94addb2", + "catalogue_version": "0.4.0", + "fingerprint": "c1e37dbeb5030428", "checks": [ { "id": "IDX-01", @@ -815,6 +815,24 @@ "untrack-generated" ] }, + { + "id": "EXEC-08", + "family": "EXEC", + "title": "Tool versions visible to dependency automation", + "phase": 1, + "severity": "DEGRADER", + "scored": true, + "method": "automated", + "status": "implemented", + "req_ids": [ + "REQ-SH-058" + ], + "engine_id": "EXEC-08", + "since_ruleset": "2026-11-09.3", + "playbooks": [ + "pin-execution-environment" + ] + }, { "id": "TST-01", "family": "TST", diff --git a/audit/checks/09-execution-determinism.md b/audit/checks/09-execution-determinism.md index 9496bb48..ae930d35 100644 --- a/audit/checks/09-execution-determinism.md +++ b/audit/checks/09-execution-determinism.md @@ -291,3 +291,35 @@ Remediation `untrack-generated` (detection and documentation only: keep one Pitfalls `biome.json` matches both the `lint` and `format` globs by design (Biome itself combines the two); a repository with only `biome.json` for both concerns is not competition — only a second file matching the same single concern is. + +## EXEC-08 — Tool versions visible to dependency automation + +Why it matters Dependabot and Renovate read manifests, not commands. A tool version typed into a + `Makefile`, a workflow or a hook gets no security alert and no update, and an agent asked + to upgrade the tool must find every copy by hand. +Severity DEGRADER +Engine `EXEC-08`. Reads, line by line, the command runners (`Makefile`, `*.mk`, `justfile`, + `Taskfile`, `package.json`), the CI workflows (`.github/workflows/*`, `.gitlab-ci.yml`) + and the hook files (`lefthook.yml`, `.husky/*`) for a literal version after a launcher: + Node `npx`, `bunx`, `pnpm dlx`, `yarn dlx`, `npm exec`, `npm install -g`; Python `uvx`, + `uv tool`, `uv run --with`, `pipx`, `pip install`; Go `go run`/`go install`; Rust + `cargo install`; and the `version` input of `setup-uv`, `pnpm/action-setup`, + `setup-bun` and `golangci-lint-action`. Each literal is looked up in every + `package.json`, `pyproject.toml`, `requirements*.txt`, `go.mod` and `Cargo.toml`. +Method automated +Look for `name@1.2.3` or `name==1.2.3` after a launcher, where the version starts with a digit. + A version read from a variable (`ruff@$(call pin,ruff)`) is not a literal. +Rubric PASS: no literal tool version in any file read, or the file is matched by a Renovate + `customManagers` (or `regexManagers`) file pattern. + PARTIAL: every literal version also appears, same name and version, in a manifest; the + inline copy drifts when the bot updates the manifest. + FAIL: at least one literal version that no manifest holds. + NOT_APPLICABLE: never; the engine has no such verdict. + UNKNOWN when: there is no command runner, workflow or hook file to read. +Evidence to cite A `file` evidence item per literal, with its line and the tool and version. +Absence proof A PASS states how many command files were read and that none holds a literal version. +Remediation `pin-execution-environment` +Pitfalls Runtime versions (`node-version`, `python-version`, `go-version`) belong to `EXEC-01`, + not here. `uses:` action pins are visible to Dependabot's `github-actions` ecosystem. A + versionless `uvx ruff` floats; it is a reproducibility concern, not a visibility one, and + passes this check. Scripts a verb calls and `Dockerfile` `RUN` lines are not read yet. diff --git a/audit/engine/audit/__init__.py b/audit/engine/audit/__init__.py index c1690ecc..ff1f2955 100644 --- a/audit/engine/audit/__init__.py +++ b/audit/engine/audit/__init__.py @@ -24,7 +24,7 @@ #: Bumped whenever a rule pack, a check, or the instruction-variant table changes. #: Recorded in every report's provenance block so a report can be attributed to an #: exact catalogue revision (docs: Scan Engine & Tiering, § 7 Determinism). -__ruleset_revision__ = "2026-11-09.2" +__ruleset_revision__ = "2026-11-09.3" #: Highest framework phase this build evaluates. Phase 1 only in v1. __phase_ceiling__ = 1 diff --git a/audit/engine/audit/rules/checks/execution.py b/audit/engine/audit/rules/checks/execution.py index ea071bdb..4a19fded 100644 --- a/audit/engine/audit/rules/checks/execution.py +++ b/audit/engine/audit/rules/checks/execution.py @@ -5,7 +5,7 @@ Anchor: `docs/framework/phased-approach.md` § Activities. -Implemented here: `EXEC-01` … `EXEC-07`. +Implemented here: `EXEC-01` … `EXEC-08`. What each check reads and what makes it ``UNKNOWN`` rather than ``FAIL``: @@ -38,6 +38,16 @@ * **EXEC-07** — at most one authoritative config per concern (lint, format, test, types, security). Only repository-root configs are counted, so a monorepo's per-component configs are not mistaken for competition. +* **EXEC-08** — a tool version written as a literal into a command (``uvx ruff@0.5.0``, ``npx x@1.2.3``, + ``pip install x==1.2``, ``go install pkg@v1.2``, ``cargo install x --version 1.2``, or the ``version`` input of a + tool-installer action) is invisible to Dependabot and Renovate. The surfaces read are the command runners + (``Makefile``, ``*.mk``, ``justfile``, ``Taskfile``, ``package.json``), the CI workflows and the hook files, line by + line; a version that is a variable (``$(call pin,ruff)``) is not a literal. Each literal is looked up in the + manifests at any depth (:func:`_manifest_holds`). Any literal no manifest holds ⇒ ``FAIL``; every literal also + held by a manifest ⇒ ``PARTIAL`` (the inline copy drifts when the bot updates the manifest); none ⇒ ``PASS``. A + file that a Renovate ``customManagers``/``regexManagers`` pattern matches is visible and is not read. No surface + file at all ⇒ ``UNKNOWN``. Runtime versions (``node-version``) are ``EXEC-01``'s; ``uses:`` pins are visible to + Dependabot already; a versionless ``uvx ruff`` is not a visibility defect. Known limitation, recorded rather than hidden: manifest↔lockfile consistency parsing for ecosystems beyond node/python/rust/go is not attempted, so those ecosystems report the tracked fact and @@ -51,6 +61,7 @@ import json import re import shlex +from dataclasses import dataclass from audit.evaluate import CheckOutcome from audit.findings import Evidence, Finding, Verdict, statement @@ -1332,6 +1343,309 @@ def check_exec07(*, spec, target, inventory, stack, components, session) -> Chec ) +# =========================================================================== +# EXEC-08 — tool versions visible to dependency automation +# =========================================================================== + +#: Basenames of the command-runner and hook files whose lines are read. +_INLINE_SURFACE_NAMES = frozenset( + { + "Makefile", + "makefile", + "GNUmakefile", + "justfile", + "Justfile", + ".justfile", + "Taskfile.yml", + "Taskfile.yaml", + "package.json", + "lefthook.yml", + "lefthook.yaml", + ".lefthook.yml", + ".lefthook.yaml", + ".gitlab-ci.yml", + } +) + +_LITERAL_VERSION = r"v?\d[\w.+-]*" +_SEGMENT_SPLIT = re.compile(r"\s*(?:&&|\|\||;|\|)\s*") + +#: ``(ecosystem, launcher pattern)``. The tokens after a launcher are read for ``name@version`` or +#: ``name==version``; a version is a literal only when it starts with a digit (or ``v`` and a digit). +_INLINE_LAUNCHERS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ("node", re.compile(r"(?:^|[\s(\"'])(?:npx|bunx|pnpx|pnpm\s+dlx|yarn\s+dlx|npm\s+exec)\s")), + ("node", re.compile(r"(?:^|[\s(\"'])npm\s+(?:install|i)\s+(?:\S+\s+)*?(?:-g|--global)\b")), + ("python", re.compile(r"(?:^|[\s(\"'])(?:uvx|uv\s+tool\s+(?:run|install)|pipx\s+(?:run|install))\s")), + ("python", re.compile(r"(?:^|[\s(\"'])uv\s+run\s")), + ("python", re.compile(r"(?:^|[\s(\"'])(?:pip3?|uv\s+pip|python3?\s+-m\s+pip)\s+install\s")), + ("go", re.compile(r"(?:^|[\s(\"'])go\s+(?:run|install)\s")), + ("rust", re.compile(r"(?:^|[\s(\"'])cargo\s+(?:install|binstall)\s")), +) + +_NODE_TOKEN = re.compile(rf"^(@?[A-Za-z0-9][\w./-]*)@({_LITERAL_VERSION})$") +_PYTHON_TOKEN = re.compile(rf"^([A-Za-z0-9][\w.-]*)(?:\[[^\]]*\])?(?:==|@)({_LITERAL_VERSION})$") +_GO_TOKEN = re.compile(r"^([\w.-]+(?:/[\w.-]+)+)@(v\d[\w.+-]*)$") +_RUST_TOKEN = re.compile(rf"^([A-Za-z0-9][\w-]*)@({_LITERAL_VERSION})$") +_TOKEN_PATTERNS = {"node": _NODE_TOKEN, "python": _PYTHON_TOKEN, "go": _GO_TOKEN, "rust": _RUST_TOKEN} + +#: Tool-installer actions whose input names a tool version: ``action -> (input, ecosystem, tool name)``. +_INSTALLER_ACTIONS: dict[str, tuple[str, str, str]] = { + "astral-sh/setup-uv": ("version", "python", "uv"), + "pnpm/action-setup": ("version", "node", "pnpm"), + "oven-sh/setup-bun": ("bun-version", "node", "bun"), + "golangci/golangci-lint-action": ("version", "go", "github.com/golangci/golangci-lint"), +} +_USES_RE = re.compile(r"^(\s*)(?:-\s+)?uses:\s*['\"]?([\w.-]+/[\w.-]+)(?:/[\w./-]*)?@") + +_RENOVATE_FILES = ("renovate.json", ".renovaterc", ".renovaterc.json", ".github/renovate.json") + + +@dataclass(frozen=True) +class _InlineVersion: + path: str + line: int + ecosystem: str + tool: str + version: str + + +def _inline_surfaces(inv: Inventory) -> list[str]: + out = [] + for rel in inv.paths(): + base = rel.rsplit("/", 1)[-1] + if ( + base in _INLINE_SURFACE_NAMES + or base.endswith(".mk") + or (rel.startswith(".github/workflows/") and base.endswith((".yml", ".yaml"))) + or (rel.startswith(".husky/") and rel.count("/") == 1) + ): + out.append(rel) + return sorted(out) + + +def _token(raw: str) -> str: + token = raw.strip().strip("\"'`,)") + for prefix in ("--from=", "--with=", "--spec=", "--package="): + if token.startswith(prefix): + return token[len(prefix) :] + return token + + +def _versions_in_segment(segment: str) -> list[tuple[str, str, str]]: + """``(ecosystem, tool, version)`` for each literal-versioned token after a launcher in one command.""" + found: list[tuple[str, str, str]] = [] + for ecosystem, launcher in _INLINE_LAUNCHERS: + match = launcher.search(segment) + if not match: + continue + tokens = [_token(raw) for raw in segment[match.end() :].split()] + pattern = _TOKEN_PATTERNS[ecosystem] + for index, token in enumerate(tokens): + hit = pattern.match(token) + if hit: + found.append((ecosystem, hit.group(1), hit.group(2))) + elif ecosystem == "rust" and token in ("--version", "--vers") and index + 1 < len(tokens): + name = next((t for t in tokens[:index] if not t.startswith("-")), None) + if name and re.match(rf"^{_LITERAL_VERSION}$", tokens[index + 1]): + found.append((ecosystem, name, tokens[index + 1])) + return sorted(set(found)) + + +def _installer_versions(lines: list[str]) -> list[tuple[int, str, str, str]]: + """``(line, ecosystem, tool, version)`` for a literal ``version`` input of a known tool-installer action.""" + found: list[tuple[int, str, str, str]] = [] + for index, line in enumerate(lines): + uses = _USES_RE.match(line) + if not uses or uses.group(2) not in _INSTALLER_ACTIONS: + continue + key, ecosystem, tool = _INSTALLER_ACTIONS[uses.group(2)] + step_indent = len(uses.group(1)) + for number in range(index + 1, len(lines)): + text = lines[number] + if text.strip() and len(text) - len(text.lstrip()) <= step_indent: + break + value = re.match(rf"^\s*{re.escape(key)}:\s*['\"]?({_LITERAL_VERSION})['\"]?\s*(#.*)?$", text) + if value: + found.append((number + 1, ecosystem, tool, value.group(1))) + return found + + +def _inline_versions(inv: Inventory, rel: str) -> list[_InlineVersion]: + text = inv.read(rel) or "" + lines = text.splitlines() + out: list[_InlineVersion] = [] + for number, line in enumerate(lines, start=1): + if line.lstrip().startswith("#"): + continue + for segment in _SEGMENT_SPLIT.split(line): + for ecosystem, tool, version in _versions_in_segment(segment): + out.append(_InlineVersion(rel, number, ecosystem, tool, version)) + if rel.startswith(".github/workflows/"): + out.extend(_InlineVersion(rel, n, eco, tool, ver) for n, eco, tool, ver in _installer_versions(lines)) + return sorted(set(out), key=lambda v: (v.path, v.line, v.tool, v.version)) + + +def _renovate_patterns(inv: Inventory) -> list[re.Pattern[str]]: + """The file patterns of every Renovate custom manager: files Renovate reads versions from by regex.""" + patterns: list[re.Pattern[str]] = [] + for rel in _RENOVATE_FILES: + try: + config = json.loads(inv.read(rel) or "") + except ValueError: + continue + if not isinstance(config, dict): + continue + for key in ("customManagers", "regexManagers"): + managers = config.get(key) + for manager in managers if isinstance(managers, list) else []: + if not isinstance(manager, dict): + continue + for field_name in ("fileMatch", "managerFilePatterns"): + raw = manager.get(field_name) + for item in raw if isinstance(raw, list) else []: + text = str(item) + if len(text) > 1 and text.startswith("/") and text.endswith("/"): + text = text[1:-1] + try: + patterns.append(re.compile(text)) + except re.error: + continue + return patterns + + +def _normalise(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def _bare(version: str) -> str: + return version[1:] if version.startswith("v") else version + + +def _node_manifest_holds(inv: Inventory, tool: str, version: str) -> bool: + for rel in inv.paths(): + if rel.rsplit("/", 1)[-1] != "package.json": + continue + try: + manifest = json.loads(inv.read(rel) or "") + except ValueError: + continue + if not isinstance(manifest, dict): + continue + if manifest.get("packageManager") == f"{tool}@{_bare(version)}": + return True + for key in ("dependencies", "devDependencies", "optionalDependencies"): + deps = manifest.get(key) + if isinstance(deps, dict) and str(deps.get(tool, "")).lstrip("^~=v") == _bare(version): + return True + return False + + +def _python_manifest_holds(inv: Inventory, tool: str, version: str) -> bool: + wanted = _normalise(tool) + pin = re.compile(r"([A-Za-z0-9][\w.-]*)(?:\[[^\]]*\])?\s*==\s*([\w.+-]+)") + for rel in inv.paths(): + base = rel.rsplit("/", 1)[-1] + if base != "pyproject.toml" and not (base.startswith("requirements") and base.endswith(".txt")): + continue + for name, pinned in pin.findall(inv.read(rel) or ""): + if _normalise(name) == wanted and pinned == version: + return True + return False + + +def _go_manifest_holds(inv: Inventory, tool: str, version: str) -> bool: + for rel in inv.paths(): + if rel.rsplit("/", 1)[-1] != "go.mod": + continue + for line in (inv.read(rel) or "").splitlines(): + parts = line.split("//", 1)[0].split() + if parts and parts[0] == "require": + parts = parts[1:] + if len(parts) >= 2 and (tool == parts[0] or tool.startswith(parts[0] + "/")): + if _bare(parts[1]) == _bare(version): + return True + return False + + +def _rust_manifest_holds(inv: Inventory, tool: str, version: str) -> bool: + pin = re.compile(rf"^\s*{re.escape(tool)}\s*=\s*(?:\{{[^}}]*version\s*=\s*)?\"[=^~]?{re.escape(version)}\"", re.M) + return any(pin.search(inv.read(rel) or "") for rel in inv.paths() if rel.rsplit("/", 1)[-1] == "Cargo.toml") + + +def _manifest_holds(inv: Inventory, found: _InlineVersion) -> bool: + """Whether a manifest that dependency automation reads names the same tool at the same version.""" + lookup = { + "node": _node_manifest_holds, + "python": _python_manifest_holds, + "go": _go_manifest_holds, + "rust": _rust_manifest_holds, + }[found.ecosystem] + return lookup(inv, found.tool, found.version) + + +def check_exec08(*, spec, target, inventory, stack, components, session) -> CheckOutcome: + surfaces = _inline_surfaces(inventory) + if not surfaces: + return _unknown(spec, "no command runner, workflow or hook file to read for tool versions") + patterns = _renovate_patterns(inventory) + visible = [rel for rel in surfaces if any(pattern.search(rel) for pattern in patterns)] + inline = [v for rel in surfaces if rel not in visible for v in _inline_versions(inventory, rel)] + if not inline: + note = f"; {len(visible)} file(s) read by a Renovate custom manager" if visible else "" + return _outcome(spec, Verdict.PASS, f"no literal tool version in {len(surfaces)} command file(s){note}") + + unheld = [v for v in inline if not _manifest_holds(inventory, v)] + held = [v for v in inline if v not in unheld] + + def evidence(items: list[_InlineVersion]) -> list[Evidence]: + return [Evidence(v.path, line=v.line, note=f"{v.tool} {v.version}") for v in items] + + findings: list[Finding] = [] + if unheld: + findings.append( + Finding( + check=spec.id, + severity=spec.severity, + phase=spec.phase, + verdict=Verdict.FAIL, + statement=statement( + "rely on dependency automation to patch the tools the gate runs", + f"{len(unheld)} tool version(s) are written into commands and no manifest holds them " + f"({', '.join(sorted({v.tool for v in unheld}))})", + ), + evidence=evidence(unheld), + path=unheld[0].path, + remediation="Move each version into a manifest the dependency-update tool reads (a dev-tools " + "package.json or pyproject.toml with a lockfile, a go.mod tool directive) and have the " + "command read it from there.", + ) + ) + if held: + findings.append( + Finding( + check=spec.id, + severity=spec.severity, + phase=spec.phase, + verdict=Verdict.PARTIAL, + statement=statement( + "trust that one update changes every copy of a tool version", + f"{len(held)} tool version(s) written into commands repeat a manifest pin " + f"({', '.join(sorted({v.tool for v in held}))}), so the copy drifts when the manifest is updated", + ), + evidence=evidence(held), + path=f"{held[0].path}:duplicate", + remediation="Have the command read the version from the manifest instead of repeating it.", + ) + ) + verdict = Verdict.FAIL if unheld else Verdict.PARTIAL + summary = ( + f"{len(unheld)} literal tool version(s) no manifest holds" + if unheld + else f"{len(held)} literal tool version(s) repeat a manifest pin" + ) + return _outcome(spec, verdict, summary, findings) + + IMPLEMENTATIONS = { "EXEC-01": check_exec01, "EXEC-02": check_exec02, @@ -1340,4 +1654,5 @@ def check_exec07(*, spec, target, inventory, stack, components, session) -> Chec "EXEC-05": check_exec05, "EXEC-06": check_exec06, "EXEC-07": check_exec07, + "EXEC-08": check_exec08, } diff --git a/audit/engine/audit/rules/registry.py b/audit/engine/audit/rules/registry.py index 4ae3b7c9..c3682e9e 100644 --- a/audit/engine/audit/rules/registry.py +++ b/audit/engine/audit/rules/registry.py @@ -208,6 +208,7 @@ def _c( _c("EXEC-05", "No generated artifacts committed in-tree", "B", _S.DEGRADER), _c("EXEC-06", "Documented commands resolve", "B", _S.DEGRADER), _c("EXEC-07", "No competing configs for one concern", "B", _S.DEGRADER), + _c("EXEC-08", "Tool versions visible to dependency automation", "B", _S.DEGRADER), # -- TST ------------------------------------------------------------ _c("TST-01", "Test verb exists, suite non-empty", "A", _S.BLOCKER), _c("TST-02", "Suite status known", "C", _S.DEGRADER), diff --git a/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md b/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md index 3b5410a9..2baef3a2 100644 --- a/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md +++ b/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md @@ -28,15 +28,15 @@ | `CI` CI | 4 | — | 4 | — | | `BASE` Baselines & ratchet | 4 | 2 | 2 | — | | `CON` Constraints | 4 | — | 3 | — | -| `EXEC` Execution determinism | 7 | 2 | 5 | — | +| `EXEC` Execution determinism | 8 | 2 | 6 | — | | `TST` Tests | 5 | 2 | 2 | 1 | | `NAV` Navigation | 3 | 1 | 2 | — | | `SEC` Security hygiene | 3 | — | 3 | — | -| **Total catalogued** | **57** | **17** | **37** | **2** | +| **Total catalogued** | **58** | **17** | **38** | **2** | Plus **13 informational checks** (`NAV-04`, `NAV-05`, `HYG-01`–`HYG-11`), reported and never scored. -Of the 57, `CON-04` is an emitter rather than a check, leaving **56 scoreable checks** in v1. (`CON-01` carried weight 0 +Of the 58, `CON-04` is an emitter rather than a check, leaving **57 scoreable checks** in v1. (`CON-01` carried weight 0 until ruleset `2026-10-11.1`, because the engine did not read the artifact that the spec defines; it reads the prohibitions section now.) @@ -171,6 +171,7 @@ pattern is *named* when it stands in a code span of the section. The audit also | `EXEC-05` | No generated artifacts committed in-tree | B | DEGRADER | Generated or vendored paths are either untracked or carry a generated-file header | Implemented | | `EXEC-06` | Documented commands resolve | B | DEGRADER | Every fenced command in README and `AGENTS.md` resolves to a real script, verb, or file | Implemented | | `EXEC-07` | No competing configs for one concern | B | DEGRADER | At most one authoritative config per concern (lint, format, test, types) | Implemented | +| `EXEC-08` | Tool versions visible to dependency automation | B | DEGRADER | No literal tool version in a command, workflow or hook (`FAIL`), or only copies of a manifest pin (`PARTIAL`) | Implemented | --- diff --git a/audit/engine/docs/architecture/user-overview/03-why-these-checks.md b/audit/engine/docs/architecture/user-overview/03-why-these-checks.md index 6714bc3c..cd2de9d8 100644 --- a/audit/engine/docs/architecture/user-overview/03-why-these-checks.md +++ b/audit/engine/docs/architecture/user-overview/03-why-these-checks.md @@ -137,6 +137,7 @@ Where brownfield repositories fail hardest, and where the framework's Phase 1 is | `EXEC-05` No generated artifacts committed in-tree | Agents edit files that are regenerated | Work silently discarded after review | | `EXEC-06` Documented commands resolve | Agents obey written instructions literally | The most expensive brownfield trap there is | | `EXEC-07` No competing configs for one concern | One question, one authority | Agent picks the config that happens to match the first result | +| `EXEC-08` Tool versions visible to dependency automation | One place per tool version, where the update bot looks | A vulnerable tool stays in the gate with no alert, and an upgrade misses the copies | --- diff --git a/audit/engine/docs/reference/changelog.md b/audit/engine/docs/reference/changelog.md index 4832257a..8e6c8ceb 100644 --- a/audit/engine/docs/reference/changelog.md +++ b/audit/engine/docs/reference/changelog.md @@ -13,6 +13,7 @@ the record of how; git has the rest. | Revision | Date | Scoreable checks | What it is | |---|---|---|---| +| `2026-11-09.3` | 2026-11-09 | 57 | v0.3.3 C3: new check `EXEC-08` (tool versions visible to dependency automation, DEGRADER). It reads the command runners, the CI workflows and the hook files for a tool version written as a literal (`uvx ruff@0.5.0`, `npx x@1.2.3`, `pip install x==1.2`, `go install pkg@v1.2`, `cargo install --version`, a tool-installer action's `version` input): none ⇒ `PASS`, every literal also in a manifest ⇒ `PARTIAL`, any literal no manifest holds ⇒ `FAIL`, no such file ⇒ `UNKNOWN`. A file a Renovate custom manager reads is visible. Every repository with an inline tool version gains an `EXEC-08` finding | | `2026-11-09.2` | 2026-11-09 | 56 | v0.3.3 C2: `HYG-09` (informational, unscored) reads the Dependabot or Renovate file. `PARTIAL` when a detected ecosystem (from a manifest, plus `github-actions` with GitHub CI) has no entry, or when a Dependabot entry sets `target-branch` together with `ignore` or `groups`; `UNKNOWN` when the file cannot be read. The update policy itself is the owner's choice and never a verdict. No score moves | | `2026-11-09.1` | 2026-11-09 | 56 | v0.3.3 C1: new check `CI-05` (workflow files are linted, DEGRADER). It reads the linters each CI provider supports (actionlint, zizmor, check-jsonschema schemas, `glab ci lint`, `circleci config validate`): local ⇒ `PASS`, CI only ⇒ `PARTIAL`, none ⇒ `FAIL`, no workflow or Jenkins only ⇒ `UNKNOWN`. Every repository with workflow files and no linter gains a `CI-05` finding | | `2026-11-06.1` | 2026-11-06 | 55 | MR06.4: `TST-04` accepts `python3 -m unittest -k ` as a single-test form; `AGT-08` takes a line of the root `AGENTS.md` that says "read by" or "supported harnesses" and names a known harness as the answer to the reach gap (#26) | diff --git a/audit/engine/tests/fixtures/README.md b/audit/engine/tests/fixtures/README.md index ffaf7ed5..90065bb9 100644 --- a/audit/engine/tests/fixtures/README.md +++ b/audit/engine/tests/fixtures/README.md @@ -7,8 +7,8 @@ Current state: the fixture matrix is complete and consolidated by `../test_fixtu carries one explicit expected verdict per fixture and asserts the positive fixtures collectively pass every scoreable check. The `AGT` content-contract checks (`AGT-03`, `04`, `06`, `07`, `09`) and `AGT-05` are exercised by `../test_agt_pack.py` and `../test_cmd_pack.py`; the earlier `AGT` checks -(`01`, `02`, `08`, `10`) now have their own fixtures below. The `EXEC` checks (`EXEC-01`…`07`) are -exercised by `../test_exec_pack.py`; the `CMD` checks (`CMD-01`…`04`) by `../test_cmd_pack.py`; and +(`01`, `02`, `08`, `10`) now have their own fixtures below. The `EXEC` checks (`EXEC-01`…`08`) are +exercised by `../test_exec_pack.py` (and `EXEC-08` by `../test_exec08_inline_versions.py`); the `CMD` checks (`CMD-01`…`04`) by `../test_cmd_pack.py`; and the `SEC` checks (`SEC-01`…`03`) by `../test_sec_pack.py`, against the fixtures below. The `DOC` and `NAV` checks (`DOC-01`…`04`, `NAV-01`…`05`) are exercised by `../test_doc_nav_pack.py`. The `TOOL` checks (`TOOL-01`…`06`) are exercised by `../test_tool_pack.py`; the `CI` checks (`CI-01`…`03`) by @@ -72,6 +72,8 @@ fixtures/ exec-05-generated/ tracked generated file, no header (needs git) -> EXEC-05 FAIL exec-06-broken-command/ README runs an npm script that is not declared -> EXEC-06 FAIL exec-07-dual-lint/ .eslintrc.json + biome.json compete -> EXEC-07 FAIL + exec-08-inline-version/ an npm script runs `npx eslint@9.0.0`, no manifest holds it -> EXEC-08 FAIL + exec-08-duplicate-pin/ the same script, eslint 9.0.0 also in devDependencies -> EXEC-08 PARTIAL exec-ok/ the smallest node repository passing every EXEC check (needs git) doc-01-empty/ docs/ present, no markdown page -> DOC-01 PARTIAL doc-02-no-style-guide/ docs/ present, no STYLE_GUIDE.md -> DOC-02 FAIL diff --git a/audit/engine/tests/fixtures/exec-08-duplicate-pin/package.json b/audit/engine/tests/fixtures/exec-08-duplicate-pin/package.json new file mode 100644 index 00000000..6cdec0db --- /dev/null +++ b/audit/engine/tests/fixtures/exec-08-duplicate-pin/package.json @@ -0,0 +1,11 @@ +{ + "name": "exec-08-duplicate-pin", + "version": "0.1.0", + "private": true, + "scripts": { + "lint": "npx --yes eslint@9.0.0 ." + }, + "devDependencies": { + "eslint": "9.0.0" + } +} diff --git a/audit/engine/tests/fixtures/exec-08-inline-version/package.json b/audit/engine/tests/fixtures/exec-08-inline-version/package.json new file mode 100644 index 00000000..041342ff --- /dev/null +++ b/audit/engine/tests/fixtures/exec-08-inline-version/package.json @@ -0,0 +1,8 @@ +{ + "name": "exec-08-inline-version", + "version": "0.1.0", + "private": true, + "scripts": { + "lint": "npx --yes eslint@9.0.0 ." + } +} diff --git a/audit/engine/tests/test_exec08_inline_versions.py b/audit/engine/tests/test_exec08_inline_versions.py new file mode 100644 index 00000000..1f92eaef --- /dev/null +++ b/audit/engine/tests/test_exec08_inline_versions.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna + +"""`EXEC-08`: a tool version written into a command is invisible to Dependabot and Renovate. + +Any literal version no manifest holds is `FAIL`; every literal also held by a manifest is `PARTIAL`, because the +inline copy drifts; none is `PASS`. A version read from a variable is not a literal. +""" + +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import support # noqa: E402 + +MAKE_HEAD = "check:\n" + + +def make(*recipe: str) -> str: + return MAKE_HEAD + "".join(f"\t{line}\n" for line in recipe) + + +class Exec08Tests(unittest.TestCase): + def check(self, files: dict[str, str]) -> dict: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for rel, text in files.items(): + support.write(root, rel, text) + report = support.run_audit(root) + check = next(c for c in report["checks"] if c["id"] == "EXEC-08") + check["findings"] = [f for f in report.get("findings", []) if f["check"] == "EXEC-08"] + return check + + def tools(self, check: dict) -> list[str]: + return sorted(e["note"] for f in check["findings"] for e in f["evidence"]) + + def test_python_launchers_with_literal_versions_fail(self): + check = self.check( + { + "Makefile": make( + "uvx ruff@0.5.0 check .", + "uvx --from detect-secrets==1.5.0 detect-secrets scan", + "uv run --with jsonschema==4.0.0 python x.py", + "pip install black==24.1.0 isort==5.13.2", + "pipx run mypy==1.10.0 .", + ) + } + ) + self.assertEqual(check["verdict"], "FAIL") + self.assertEqual( + self.tools(check), + ["black 24.1.0", "detect-secrets 1.5.0", "isort 5.13.2", "jsonschema 4.0.0", "mypy 1.10.0", "ruff 0.5.0"], + ) + + def test_go_rust_and_scoped_node_launchers_are_read(self): + check = self.check( + { + "Makefile": make( + "go install golang.org/x/tools/cmd/goimports@v0.20.0", + "cargo install cargo-deny --version 0.14.0", + "npx -y @biomejs/biome@1.8.0 check .", + ) + } + ) + self.assertEqual( + self.tools(check), ["@biomejs/biome 1.8.0", "cargo-deny 0.14.0", "golang.org/x/tools/cmd/goimports v0.20.0"] + ) + + def test_a_version_read_from_a_variable_comments_and_versionless_commands_pass(self): + check = self.check( + { + "Makefile": make("uvx ruff@$(call pin,ruff) check .", "uvx mypy .", "go install x/y@latest"), + "lefthook.yml": "# npx eslint@9.0.0 is not run here\npre-commit:\n commands:\n lint:\n run: make check\n", + } + ) + self.assertEqual(check["verdict"], "PASS") + + def test_workflow_run_steps_and_installer_inputs_are_read(self): + workflow = ( + "on: [pull_request]\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n" + ' - uses: astral-sh/setup-uv@v6\n with:\n version: "0.5.1"\n' + " - run: pip install pre-commit==3.7.0\n" + ) + check = self.check({".github/workflows/ci.yml": workflow}) + self.assertEqual(check["verdict"], "FAIL") + self.assertEqual(self.tools(check), ["pre-commit 3.7.0", "uv 0.5.1"]) + lines = sorted((e["line"], e["note"]) for e in check["findings"][0]["evidence"]) + self.assertEqual(lines, [(8, "uv 0.5.1"), (9, "pre-commit 3.7.0")]) + + def test_every_literal_held_by_a_manifest_is_partial(self): + check = self.check( + { + "Makefile": make("uvx ruff@0.5.0 check .", "go run golang.org/x/vuln/cmd/govulncheck@v1.1.0 ./..."), + "tools/pyproject.toml": '[dependency-groups]\ntools = ["Ruff==0.5.0"]\n', + "tools/go.mod": "module tools\n\ngo 1.24\n\nrequire golang.org/x/vuln v1.1.0\n", + } + ) + self.assertEqual(check["verdict"], "PARTIAL") + + def test_a_different_manifest_version_does_not_hold_it(self): + check = self.check( + { + "Makefile": make("uvx ruff@0.5.0 check ."), + "pyproject.toml": '[project]\ndependencies = ["ruff==0.6.0"]\n', + } + ) + self.assertEqual(check["verdict"], "FAIL") + + def test_a_file_a_renovate_custom_manager_reads_is_visible(self): + renovate = '{"customManagers": [{"customType": "regex", "managerFilePatterns": ["/(^|/)Makefile$/"]}]}' + check = self.check({"Makefile": make("uvx ruff@0.5.0 check ."), "renovate.json": renovate}) + self.assertEqual(check["verdict"], "PASS") + + def test_no_command_surface_is_unknown(self): + check = self.check({"src/app.py": "print('x')\n"}) + self.assertEqual(check["verdict"], "UNKNOWN") + + +if __name__ == "__main__": + unittest.main() diff --git a/audit/engine/tests/test_fixture_matrix.py b/audit/engine/tests/test_fixture_matrix.py index 267c14df..94e8d395 100644 --- a/audit/engine/tests/test_fixture_matrix.py +++ b/audit/engine/tests/test_fixture_matrix.py @@ -142,6 +142,8 @@ "exec-05-generated": {"EXEC-05": "FAIL"}, "exec-06-broken-command": {"EXEC-06": "FAIL"}, "exec-07-dual-lint": {"EXEC-07": "FAIL"}, + "exec-08-inline-version": {"EXEC-08": "FAIL"}, + "exec-08-duplicate-pin": {"EXEC-08": "PARTIAL"}, "exec-ok": { "EXEC-01": "PASS", "EXEC-02": "PASS", diff --git a/audit/engine/tests/test_hyg_pack.py b/audit/engine/tests/test_hyg_pack.py index b50ebc74..4161f6d2 100644 --- a/audit/engine/tests/test_hyg_pack.py +++ b/audit/engine/tests/test_hyg_pack.py @@ -92,8 +92,8 @@ def test_report_lists_hyg_checks_but_excludes_them_from_the_scoreable_count(self reported = {c["id"] for c in report["checks"]} for check in HYG_IDS: self.assertIn(check, reported) - # 70 catalogued, 14 informational or emitter-only ⇒ 56 scoreable (CON-01 is scored since ruleset 2026-10-11.1). - self.assertEqual(report["summary"]["checks_scoreable"], 56) + # 71 catalogued, 14 informational or emitter-only ⇒ 57 scoreable (CON-01 is scored since ruleset 2026-10-11.1). + self.assertEqual(report["summary"]["checks_scoreable"], 57) class ScoreIsolationTests(unittest.TestCase): diff --git a/audit/engine/tests/test_registry_status.py b/audit/engine/tests/test_registry_status.py index 8b7b1f5c..37987ca3 100644 --- a/audit/engine/tests/test_registry_status.py +++ b/audit/engine/tests/test_registry_status.py @@ -63,9 +63,9 @@ class RegistryTests(unittest.TestCase): def test_every_catalogued_check_is_implemented_and_none_is_blocked(self): self.assertEqual({s.status for s in REGISTRY}, {"implemented"}) - def test_there_are_fifty_six_scoreable_checks_and_con_01_is_one_of_them(self): - self.assertEqual(len(scoreable()), 56) - self.assertEqual(len(REGISTRY), 70) + def test_there_are_fifty_seven_scoreable_checks_and_con_01_is_one_of_them(self): + self.assertEqual(len(scoreable()), 57) + self.assertEqual(len(REGISTRY), 71) con_01 = REGISTRY_BY_ID["CON-01"] self.assertEqual( (con_01.scored, con_01.status, con_01.severity, con_01.tier), (True, "implemented", Severity.BLOCKER, "B") diff --git a/audit/engine/tests/test_smoke.py b/audit/engine/tests/test_smoke.py index 596dabd6..b03aa208 100644 --- a/audit/engine/tests/test_smoke.py +++ b/audit/engine/tests/test_smoke.py @@ -39,8 +39,8 @@ def test_ids_are_unique(self): def test_catalogue_size_matches_the_documented_counts(self): from audit.rules.registry import informational, scoreable - self.assertEqual(len(REGISTRY), 70, "catalogue drifted from the documented 70 rows") - self.assertEqual(len(scoreable()), 56, "scoreable count drifted from the documented 56") + self.assertEqual(len(REGISTRY), 71, "catalogue drifted from the documented 71 rows") + self.assertEqual(len(scoreable()), 57, "scoreable count drifted from the documented 57") self.assertEqual(len(informational()), 14, "informational count drifted from 14") def test_every_spec_has_a_tier(self): diff --git a/audit/schema/examples/example-service.audit-report.20261010T094207Z.json b/audit/schema/examples/example-service.audit-report.20261010T094207Z.json index c8586205..3b299726 100644 --- a/audit/schema/examples/example-service.audit-report.20261010T094207Z.json +++ b/audit/schema/examples/example-service.audit-report.20261010T094207Z.json @@ -7,7 +7,7 @@ "version": "0.1.0" }, "framework_revision": "cafecafecafecafecafecafecafecafecafecafe", - "catalogue_version": "0.3.0", + "catalogue_version": "0.4.0", "engine": { "name": "python-agentic-audit", "version": "0.1.0", diff --git a/audit/schema/examples/example-service.audit-report.20261010T094207Z.md b/audit/schema/examples/example-service.audit-report.20261010T094207Z.md index f46cb165..b8b06cd6 100644 --- a/audit/schema/examples/example-service.audit-report.20261010T094207Z.md +++ b/audit/schema/examples/example-service.audit-report.20261010T094207Z.md @@ -12,7 +12,7 @@ | Status | final | | Started | `2026-10-10T09:00:00+00:00` | | Finalised | `2026-10-10T09:42:07+00:00` | -| Recipe | `brownfield-audit` 0.1.0, catalogue `0.3.0` | +| Recipe | `brownfield-audit` 0.1.0, catalogue `0.4.0` | | Engine | `python-agentic-audit` 0.1.0, ruleset `2026-09-28.13`, ruleset hash `a1b2a1b2a1b2a1b2`, scanned `2026-10-10T09:01:30+00:00` | | Framework revision | `cafecafecafe` | | Agent (self-reported) | `claude-code` with `claude-sonnet-5-5` | diff --git a/audit/schema/examples/minimal.json b/audit/schema/examples/minimal.json index cd13f4bb..ac9b0c28 100644 --- a/audit/schema/examples/minimal.json +++ b/audit/schema/examples/minimal.json @@ -7,7 +7,7 @@ "version": "0.1.0" }, "framework_revision": null, - "catalogue_version": "0.3.0", + "catalogue_version": "0.4.0", "agent": { "harness": "claude-code", "model": "claude-sonnet-5-5" diff --git a/audit/schema/examples/minimal.md b/audit/schema/examples/minimal.md index 9da4c43d..6881108c 100644 --- a/audit/schema/examples/minimal.md +++ b/audit/schema/examples/minimal.md @@ -15,7 +15,7 @@ | Status | draft | | Started | `2026-10-10T09:00:00+00:00` | | Finalised | — | -| Recipe | `brownfield-audit` 0.1.0, catalogue `0.3.0` | +| Recipe | `brownfield-audit` 0.1.0, catalogue `0.4.0` | | Engine | No engine report is part of this report. | | Framework revision | — | | Agent (self-reported) | `claude-code` with `claude-sonnet-5-5` | diff --git a/audit/schema/examples/readiness-state.example-service.in-progress.json b/audit/schema/examples/readiness-state.example-service.in-progress.json index 3565b232..0c7997a9 100644 --- a/audit/schema/examples/readiness-state.example-service.in-progress.json +++ b/audit/schema/examples/readiness-state.example-service.in-progress.json @@ -13,8 +13,8 @@ { "path": "docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json", "git_sha": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", - "catalogue_version": "0.3.0", - "sha256": "443653e7d98e395b4b20e2f2a0f0020db50ce11ee7bdc1c63331ac61572ad1d9" + "catalogue_version": "0.4.0", + "sha256": "7144f9fe6f688dd9b72315249a632ff6101fd9a24b5ef6ff715d7c9c8a8162c9" } ], "preferences": [ diff --git a/audit/schema/examples/readiness-state.example-service.in-progress.md b/audit/schema/examples/readiness-state.example-service.in-progress.md index 3d3a8af7..5bb381cd 100644 --- a/audit/schema/examples/readiness-state.example-service.in-progress.md +++ b/audit/schema/examples/readiness-state.example-service.in-progress.md @@ -10,7 +10,7 @@ A skipped finding is final until the owner unskips it, and every preference bind |---|---| | Default branch | `main` | | Recipe | brownfield-implement 0.2.0-dev, framework `cafecafecafecafecafecafecafecafecafecafe` | -| Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdeadbeefdeadbeefdeadbeefdeadbeef`, catalogue 0.3.0 | +| Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdeadbeefdeadbeefdeadbeefdeadbeef`, catalogue 0.4.0 | ## Summary diff --git a/audit/schema/examples/readiness-state.example-service.json b/audit/schema/examples/readiness-state.example-service.json index 467aa5e9..fc81735a 100644 --- a/audit/schema/examples/readiness-state.example-service.json +++ b/audit/schema/examples/readiness-state.example-service.json @@ -13,8 +13,8 @@ { "path": "docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json", "git_sha": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", - "catalogue_version": "0.3.0", - "sha256": "443653e7d98e395b4b20e2f2a0f0020db50ce11ee7bdc1c63331ac61572ad1d9" + "catalogue_version": "0.4.0", + "sha256": "7144f9fe6f688dd9b72315249a632ff6101fd9a24b5ef6ff715d7c9c8a8162c9" } ], "preferences": [], diff --git a/audit/schema/examples/readiness-state.example-service.md b/audit/schema/examples/readiness-state.example-service.md index aaf9e71d..7495f494 100644 --- a/audit/schema/examples/readiness-state.example-service.md +++ b/audit/schema/examples/readiness-state.example-service.md @@ -10,7 +10,7 @@ A skipped finding is final until the owner unskips it, and every preference bind |---|---| | Default branch | `main` | | Recipe | brownfield-implement 0.2.0-dev, framework `cafecafecafecafecafecafecafecafecafecafe` | -| Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdeadbeefdeadbeefdeadbeefdeadbeef`, catalogue 0.3.0 | +| Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdeadbeefdeadbeefdeadbeefdeadbeef`, catalogue 0.4.0 | ## Summary diff --git a/audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json b/audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json index 120cbeb8..e5aed3f0 100644 --- a/audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json +++ b/audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json @@ -7,7 +7,7 @@ "version": "0.1.0" }, "framework_revision": "7eedf23c99c39f94cad435e8a52eb28642308a92", - "catalogue_version": "0.3.0", + "catalogue_version": "0.4.0", "engine": { "name": "python-agentic-audit", "version": "0.1.0", @@ -1540,6 +1540,18 @@ "summary": "The engine did not run this check: its ruleset is older than the catalogue. Scan again with a newer engine.", "evidence": [], "searches": [] + }, + { + "id": "EXEC-08", + "title": "Tool versions visible to dependency automation", + "phase": 1, + "severity": "DEGRADER", + "scored": true, + "method": "automated", + "verdict": "UNKNOWN", + "summary": "The engine did not run this check: its ruleset is older than the catalogue. Scan again with a newer engine.", + "evidence": [], + "searches": [] } ], "findings": [ @@ -2123,15 +2135,15 @@ } ], "summary": { - "phase1_score": 0.7016, - "scored_checks": 56, + "phase1_score": 0.6905, + "scored_checks": 57, "blockers_remaining": { "total": 6, "fail": 2, "partial": 2, "unknown": 2 }, - "unknown_scored_checks": 5, + "unknown_scored_checks": 6, "unconfirmed_agent_findings": 0 }, "unattested": [ diff --git a/audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json b/audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json index 48d31fe2..4b1ec644 100644 --- a/audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json +++ b/audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json @@ -7,7 +7,7 @@ "version": "0.1.0" }, "framework_revision": "7eedf23c99c39f94cad435e8a52eb28642308a92", - "catalogue_version": "0.3.0", + "catalogue_version": "0.4.0", "engine": { "name": "python-agentic-audit", "version": "0.1.0", @@ -1766,6 +1766,18 @@ "summary": "The engine did not run this check: its ruleset is older than the catalogue. Scan again with a newer engine.", "evidence": [], "searches": [] + }, + { + "id": "EXEC-08", + "title": "Tool versions visible to dependency automation", + "phase": 1, + "severity": "DEGRADER", + "scored": true, + "method": "automated", + "verdict": "UNKNOWN", + "summary": "The engine did not run this check: its ruleset is older than the catalogue. Scan again with a newer engine.", + "evidence": [], + "searches": [] } ], "findings": [ @@ -2698,15 +2710,15 @@ } ], "summary": { - "phase1_score": 0.625, - "scored_checks": 56, + "phase1_score": 0.6151, + "scored_checks": 57, "blockers_remaining": { "total": 6, "fail": 2, "partial": 2, "unknown": 2 }, - "unknown_scored_checks": 5, + "unknown_scored_checks": 6, "unconfirmed_agent_findings": 0 }, "unattested": [ diff --git a/audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json b/audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json index 95cf2e1b..8fbfbaa8 100644 --- a/audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json +++ b/audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json @@ -7,7 +7,7 @@ "version": "0.1.0" }, "framework_revision": "7eedf23c99c39f94cad435e8a52eb28642308a92", - "catalogue_version": "0.3.0", + "catalogue_version": "0.4.0", "engine": { "name": "python-agentic-audit", "version": "0.1.0", @@ -1518,6 +1518,18 @@ "summary": "The engine did not run this check: its ruleset is older than the catalogue. Scan again with a newer engine.", "evidence": [], "searches": [] + }, + { + "id": "EXEC-08", + "title": "Tool versions visible to dependency automation", + "phase": 1, + "severity": "DEGRADER", + "scored": true, + "method": "automated", + "verdict": "UNKNOWN", + "summary": "The engine did not run this check: its ruleset is older than the catalogue. Scan again with a newer engine.", + "evidence": [], + "searches": [] } ], "findings": [ @@ -2293,15 +2305,15 @@ } ], "summary": { - "phase1_score": 0.4073, - "scored_checks": 56, + "phase1_score": 0.4008, + "scored_checks": 57, "blockers_remaining": { "total": 9, "fail": 5, "partial": 1, "unknown": 3 }, - "unknown_scored_checks": 7, + "unknown_scored_checks": 8, "unconfirmed_agent_findings": 0 }, "unattested": [ diff --git a/audit/tools/tests/test_import_engine.py b/audit/tools/tests/test_import_engine.py index 747bd789..abd7664e 100644 --- a/audit/tools/tests/test_import_engine.py +++ b/audit/tools/tests/test_import_engine.py @@ -475,7 +475,7 @@ def test_the_envelope_counts_what_came_in(self) -> None: self.assertEqual(data["by_verdict"], {"FAIL": 2, "NOT_APPLICABLE": 1, "PARTIAL": 1, "PASS": 1, "UNKNOWN": 1}) self.assertEqual(list(data["by_verdict"]), sorted(data["by_verdict"])) self.assertEqual((data["findings_imported"], data["findings"], data["findings_pending"]), (3, 3, 3)) - self.assertEqual((data["checks_recorded"], data["checks_expected"]), (6, 72)) + self.assertEqual((data["checks_recorded"], data["checks_expected"]), (6, 73)) self.assertEqual(data["evidence_not_carried"], 1) self.assertEqual( envelope["warnings"], @@ -726,37 +726,48 @@ def test_a_check_missing_from_the_engines_report_is_still_missing(self) -> None: #: What the adapter says on importing each raw report: the engine's score and the report's differ because the report -#: counts CON-01, which the engine's ruleset (before `2026-10-11.1`) had blocked, and CI-05, which it did not have. +#: counts CON-01, which the engine's ruleset (before `2026-10-11.1`) had blocked, and CI-05 and EXEC-08, which it did +#: not have. BLOCKED_CON_01 = ( "CON-01: the engine's ruleset did not evaluate it, so it is recorded as UNKNOWN and is scored as the catalogue " "scores it" ) -#: The raw reports come from ruleset `2026-09-28.13`, older than `CI-05` (`since_ruleset` in the catalogue), so the -#: adapter records `CI-05` as UNKNOWN and says so. It counts in the report's score and not in the engine's. +#: The raw reports come from ruleset `2026-09-28.13`, older than `CI-05` and `EXEC-08` (`since_ruleset` in the +#: catalogue), so the adapter records each as UNKNOWN and says so. They count in the report's score and not in the +#: engine's. NEWER_CI_05 = ( "CI-05: the engine's ruleset 2026-09-28.13 is older than the check (2026-11-09.1), so it is recorded as " "UNKNOWN; scan again with a newer engine" ) +NEWER_EXEC_08 = ( + "EXEC-08: the engine's ruleset 2026-09-28.13 is older than the check (2026-11-09.3), so it is recorded as " + "UNKNOWN; scan again with a newer engine" +) +#: The checks newer than the raw reports' ruleset. +NEWER_CHECKS = ("CI-05", "EXEC-08") PAIRS = { "agentic-tdd": [ - "the engine's phase-1 score is 0.7311 and the report's is 0.7016; the report's follows " + "the engine's phase-1 score is 0.7311 and the report's is 0.6905; the report's follows " "spec/readiness-summary.md and is the one kept", "IDX-02: the evidence path '/home/kc/.cache/codebase-memory-mcp' is not carried: " "the path is absolute, so it is outside the repository", BLOCKED_CON_01, NEWER_CI_05, + NEWER_EXEC_08, ], "digital-assistant-sdk": [ - "the engine's phase-1 score is 0.6513 and the report's is 0.625; the report's follows " + "the engine's phase-1 score is 0.6513 and the report's is 0.6151; the report's follows " "spec/readiness-summary.md and is the one kept", BLOCKED_CON_01, NEWER_CI_05, + NEWER_EXEC_08, ], "digital-assistant-server": [ - "the engine's phase-1 score is 0.4244 and the report's is 0.4073; the report's follows " + "the engine's phase-1 score is 0.4244 and the report's is 0.4008; the report's follows " "spec/readiness-summary.md and is the one kept", BLOCKED_CON_01, NEWER_CI_05, + NEWER_EXEC_08, ], } @@ -833,7 +844,7 @@ def test_the_adapters_own_warnings_are_the_ones_expected(self) -> None: adapter_warnings = [w for w in envelope["warnings"] if not w.startswith("/")] self.assertEqual(adapter_warnings, expected) - def test_every_engine_verdict_and_finding_comes_through_and_the_score_is_the_engines_but_for_con_01_and_ci_05( + def test_every_engine_verdict_and_finding_comes_through_and_the_score_is_the_engines_but_for_newer_checks( self, ) -> None: for name in PAIRS: @@ -844,9 +855,13 @@ def test_every_engine_verdict_and_finding_comes_through_and_the_score_is_the_eng {c["id"]: (c["verdict"], c["engine_verdict"]) for c in document["checks"] if "engine_verdict" in c}, {c["id"]: (c["verdict"], c["verdict"]) for c in engine["checks"]}, ) - # CI-05 is newer than the engine's ruleset: recorded as UNKNOWN, with no verdict of the engine's. - ci_05 = next(c for c in document["checks"] if c["id"] == "CI-05") - self.assertEqual((ci_05["verdict"], "engine_verdict" in ci_05), ("UNKNOWN", False)) + # CI-05 and EXEC-08 are newer than the engine's ruleset: recorded as UNKNOWN, with no verdict of the + # engine's. + newer = [c for c in document["checks"] if c["id"] in NEWER_CHECKS] + self.assertEqual( + [(c["id"], c["verdict"], "engine_verdict" in c) for c in newer], + [(check, "UNKNOWN", False) for check in NEWER_CHECKS], + ) self.assertEqual([f["check"] for f in document["findings"]], [f["check"] for f in engine["findings"]]) # The engine's ruleset had blocked CON-01 and left it out of its score; the report counts it, as the # catalogue does. With it left out again, the report's score is the engine's. @@ -856,7 +871,9 @@ def test_every_engine_verdict_and_finding_comes_through_and_the_score_is_the_eng (True, "UNKNOWN", "The engine did not evaluate this check: its ruleset had blocked it."), ) self.assertLess(document["summary"]["phase1_score"], engine["summary"]["phase1_score"]) - con_01["scored"] = ci_05["scored"] = False + con_01["scored"] = False + for check in newer: + check["scored"] = False self.assertEqual(_summary.compute_summary(document)["phase1_score"], engine["summary"]["phase1_score"]) self.assertEqual({c["method"] for c in document["checks"]}, {"automated"}) self.assertEqual({f["disposition"] for f in document["findings"]}, {"pending"}) @@ -935,7 +952,7 @@ def test_the_whole_audit_goes_on_from_the_imported_draft_to_a_final_report(self) code, envelope, _ = self.finalise(draft) self.assertEqual(code, 0, envelope) final = read_json(envelope["data"]["report"]) - self.assertEqual((final["status"], len(final["checks"]), vr.validate_document(final).errors), ("final", 72, [])) + self.assertEqual((final["status"], len(final["checks"]), vr.validate_document(final).errors), ("final", 73, [])) self.assertTrue(Path(envelope["data"]["markdown"]).read_text(encoding="utf-8")) diff --git a/audit/tools/tests/test_import_fixture_matrix.py b/audit/tools/tests/test_import_fixture_matrix.py index 67c6083c..97fe4a52 100644 --- a/audit/tools/tests/test_import_fixture_matrix.py +++ b/audit/tools/tests/test_import_fixture_matrix.py @@ -106,7 +106,7 @@ def test_every_fixture_imports_to_a_valid_draft_that_has_the_verdict_the_matrix_ document, envelope = self.imported(name) self.assertEqual(vr.validate_document(document).errors, []) self.assertTrue(render_report.render(document)) - self.assertEqual(len(document["checks"]), 70) + self.assertEqual(len(document["checks"]), 71) checks = {c["id"]: c for c in document["checks"]} for check_id, expected in self.matrix.MATRIX[name].items(): self.assertEqual(checks[check_id]["verdict"], expected, f"{name}: {check_id}") @@ -115,7 +115,7 @@ def test_every_fixture_imports_to_a_valid_draft_that_has_the_verdict_the_matrix_ if expected != "PASS": defects += 1 self.assertNotEqual(checks[check_id]["verdict"], "PASS") - self.assertEqual(envelope["data"]["checks_imported"], 70) + self.assertEqual(envelope["data"]["checks_imported"], 71) self.assertGreater(defects, 40) diff --git a/audit/tools/tests/test_render_report.py b/audit/tools/tests/test_render_report.py index d1b98642..d2221737 100644 --- a/audit/tools/tests/test_render_report.py +++ b/audit/tools/tests/test_render_report.py @@ -508,7 +508,7 @@ def test_the_worked_example_says_what_was_audited_with_what_and_when(self) -> No self.assertEqual(values["Status"], "final") self.assertEqual(values["Started"], "`2026-10-10T09:00:00+00:00`") self.assertEqual(values["Finalised"], "`2026-10-10T09:42:07+00:00`") - self.assertEqual(values["Recipe"], "`brownfield-audit` 0.1.0, catalogue `0.3.0`") + self.assertEqual(values["Recipe"], "`brownfield-audit` 0.1.0, catalogue `0.4.0`") self.assertEqual( values["Engine"], "`python-agentic-audit` 0.1.0, ruleset `2026-09-28.13`, ruleset hash `a1b2a1b2a1b2a1b2`, " @@ -1499,7 +1499,7 @@ def test_every_check_of_every_example_has_a_row_in_report_order(self) -> None: order.setdefault(check["id"].split("-")[0], len(order)) expected = sorted(report["checks"], key=lambda c: order[c["id"].split("-")[0]]) self.assertEqual(found, [f"`{c['id']}`" for c in expected]) - self.assertEqual(len(rows_of(section(self.render(load(drafts()[0])), "Appendix — all checks"))), 70) + self.assertEqual(len(rows_of(section(self.render(load(drafts()[0])), "Appendix — all checks"))), 71) def test_a_row_gives_the_severity_method_scoring_both_verdicts_and_the_summary(self) -> None: text = section(self.render(service()), "Appendix — all checks") @@ -1879,7 +1879,7 @@ def test_every_realistic_draft_renders_with_every_check_and_every_finding(self) with self.subTest(path=path.name): report = load(path) markdown = self.render(report) - self.assertEqual(len(rows_of(section(markdown, "Appendix — all checks"))), 70) + self.assertEqual(len(rows_of(section(markdown, "Appendix — all checks"))), 71) for finding in report["findings"]: self.assertEqual(markdown.count(f"{BR}finding `{finding['id']}`"), 1) self.assertIn("> [!NOTE]\n> **Draft.**", markdown) diff --git a/audit/tools/tests/test_report.py b/audit/tools/tests/test_report.py index 3aba239f..68cbf095 100644 --- a/audit/tools/tests/test_report.py +++ b/audit/tools/tests/test_report.py @@ -104,7 +104,7 @@ def drafts(self) -> list[str]: return sorted(p.name for p in self.out.iterdir()) if self.out.is_dir() else [] def engine_draft(self) -> Path: - """A copy of the engine's real report on agentic-tdd, converted: 70 engine checks (CI-05 newer, so UNKNOWN), 24 + """A copy of the engine's real report on agentic-tdd, converted: 71 engine checks (CI-05 and EXEC-08 newer, so UNKNOWN), 24 pending findings. The copy points at the temporary target, with no commit, so that nothing depends on this machine. @@ -566,7 +566,7 @@ def test_it_records_an_agent_check_and_fills_in_what_the_catalogue_knows(self) - "action": "added", "verdict": "PASS", "checks_recorded": 1, - "checks_expected": 72, + "checks_expected": 73, }, ) self.assertEqual(stderr, "report: AGT-11 added as PASS\n") @@ -918,7 +918,7 @@ def test_only_phase_1_checks_count_toward_what_a_draft_is_expected_to_hold(self) def test_an_agent_only_check_is_added_beside_the_engines(self) -> None: code, envelope, _ = self.add_check(self.draft, agent_check()) self.assertEqual(code, 0) - self.assertEqual(envelope["data"]["checks_recorded"], 71) + self.assertEqual(envelope["data"]["checks_recorded"], 72) self.assertEqual(self.check("AGT-11")["method"], "agent") @@ -1549,7 +1549,7 @@ def test_a_new_draft_has_every_family_remaining(self) -> None: self.assertEqual( (data["status"], data["eval_mode"], data["confirmation_policy"]), ("draft", False, "interactive") ) - self.assertEqual((data["checks_recorded"], data["checks_expected"]), (0, 72)) + self.assertEqual((data["checks_recorded"], data["checks_expected"]), (0, 73)) self.assertEqual(data["families_done"], "") self.assertEqual( list(data["families_remaining"]), @@ -1557,12 +1557,12 @@ def test_a_new_draft_has_every_family_remaining(self) -> None: ) self.assertEqual(data["families_remaining"]["CI"], "CI-01 CI-02 CI-03 CI-04 CI-05") self.assertEqual((data["findings"], data["findings_pending"], data["blockers_pending"]), (0, 0, 0)) - self.assertEqual((data["ready_to_finalise"], data["blocking"]), (False, "72 of 72 checks are not recorded")) + self.assertEqual((data["ready_to_finalise"], data["blocking"]), (False, "73 of 73 checks are not recorded")) self.assertEqual(data["pending"], []) self.assertNotIn("pending_omitted", data) self.assertFalse(envelope["truncated"]) self.assertEqual( - stderr, "report: 0 of 72 checks recorded; not ready to finalise: 72 of 72 checks are not recorded\n" + stderr, "report: 0 of 73 checks recorded; not ready to finalise: 73 of 73 checks are not recorded\n" ) def test_recording_a_check_takes_it_off_the_remaining_list(self) -> None: @@ -1576,11 +1576,11 @@ def test_recording_a_check_takes_it_off_the_remaining_list(self) -> None: def test_an_engine_draft_lists_what_is_left_and_the_pending_findings_blockers_first(self) -> None: draft = self.engine_draft() data = self.data(draft) - self.assertEqual((data["checks_recorded"], data["checks_expected"]), (70, 72)) + self.assertEqual((data["checks_recorded"], data["checks_expected"]), (71, 73)) self.assertEqual(data["families_remaining"], {"AGT": "AGT-11", "CI": "CI-04"}) self.assertEqual(data["families_done"], "IDX DOC CMD TOOL BASE CON EXEC TST NAV SEC HYG") self.assertEqual((data["findings"], data["findings_pending"], data["blockers_pending"]), (24, 24, 4)) - self.assertEqual(data["blocking"], "2 of 72 checks are not recorded; 24 findings are pending") + self.assertEqual(data["blocking"], "2 of 73 checks are not recorded; 24 findings are pending") findings = read_json(draft)["findings"] blockers = [f for f in findings if f["severity"] == "BLOCKER"] self.assertEqual(data["pending"][: len(blockers)], [f"{f['id']} BLOCKER {f['key']}" for f in blockers]) @@ -1604,10 +1604,10 @@ def test_what_was_left_out_of_the_list_is_said_in_the_envelope(self) -> None: self.assertEqual(code, 0) self.assertIs(envelope["truncated"], True) - def test_a_report_of_71_checks_prints_40_lines_or_fewer(self) -> None: + def test_a_report_of_72_checks_prints_40_lines_or_fewer(self) -> None: draft = self.engine_draft() self.add_check(draft, agent_check()) - self.assertEqual(len(read_json(draft)["checks"]), 71) + self.assertEqual(len(read_json(draft)["checks"]), 72) self.assertLessEqual(self.stdout_lines("status", str(draft)), 40) def test_no_draft_can_make_it_print_more_than_40_lines(self) -> None: @@ -1627,11 +1627,11 @@ def test_no_draft_can_make_it_print_more_than_40_lines(self) -> None: def test_it_is_ready_when_every_check_is_recorded_and_every_finding_decided(self) -> None: draft = self.complete_engine_draft() data = self.data(draft) - self.assertEqual((data["checks_recorded"], data["checks_expected"]), (72, 72)) + self.assertEqual((data["checks_recorded"], data["checks_expected"]), (73, 73)) self.assertEqual((data["families_remaining"], data["findings_pending"], data["blockers_pending"]), ({}, 0, 0)) self.assertEqual((data["ready_to_finalise"], data["blocking"]), (True, "")) self.assertEqual(data["pending"], []) - self.assertEqual(self.status(draft)[2], "report: 72 of 72 checks recorded; ready to finalise\n") + self.assertEqual(self.status(draft)[2], "report: 73 of 73 checks recorded; ready to finalise\n") def test_one_pending_finding_is_enough_to_hold_it_back(self) -> None: draft = self.complete_engine_draft() @@ -1735,7 +1735,7 @@ def test_it_writes_the_final_report_and_its_markdown_as_new_files_and_leaves_the self.assertEqual(envelope["data"]["draft"], str(self.draft)) self.assertEqual(envelope["data"]["status"], "final") self.assertEqual(envelope["data"]["finalised_at"], "2026-10-10T11:45:30+00:00") - self.assertEqual((envelope["data"]["checks"], envelope["data"]["findings"]), (72, 24)) + self.assertEqual((envelope["data"]["checks"], envelope["data"]["findings"]), (73, 24)) self.assertEqual(envelope["data"]["summary"], read_json(final)["summary"]) self.assertRegex( stderr, diff --git a/audit/tools/tests/test_validate_report.py b/audit/tools/tests/test_validate_report.py index 0c0c75a8..49f741c4 100644 --- a/audit/tools/tests/test_validate_report.py +++ b/audit/tools/tests/test_validate_report.py @@ -227,7 +227,7 @@ def test_a_playbook_the_catalogue_names_needs_no_file_and_a_null_playbook_is_fin self.assertEqual(vr.validate_document(report).errors, []) def test_a_catalogue_of_another_major_or_minor_version_is_a_warning_only(self) -> None: - for version, warned in (("0.3.0", False), ("0.3.7", False), ("0.2.8", True), ("0.4.0", True), ("1.3.0", True)): + for version, warned in (("0.4.0", False), ("0.4.7", False), ("0.3.0", True), ("0.5.0", True), ("1.4.0", True)): with self.subTest(version=version): report = base() report["provenance"]["catalogue_version"] = version diff --git a/dev/eval/expected/fixtures/exec-08-duplicate-pin.json b/dev/eval/expected/fixtures/exec-08-duplicate-pin.json new file mode 100644 index 00000000..1d3a8989 --- /dev/null +++ b/dev/eval/expected/fixtures/exec-08-duplicate-pin.json @@ -0,0 +1,14 @@ +{ + "expected_version": "0.1", + "repo_label": "exec-08-duplicate-pin", + "repo_sha": "5439f014bd0bb8ea2eaaefef84c673c97685d8e9", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-08-duplicate-pin' (audit/engine/tests/fixtures/exec-08-duplicate-pin), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "checks": [ + { + "id": "EXEC-08", + "acceptable_verdicts": [ + "PARTIAL" + ] + } + ] +} diff --git a/dev/eval/expected/fixtures/exec-08-inline-version.json b/dev/eval/expected/fixtures/exec-08-inline-version.json new file mode 100644 index 00000000..18224e53 --- /dev/null +++ b/dev/eval/expected/fixtures/exec-08-inline-version.json @@ -0,0 +1,14 @@ +{ + "expected_version": "0.1", + "repo_label": "exec-08-inline-version", + "repo_sha": "b21b3e039c094238c38017b89781af73d0f77b75", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-08-inline-version' (audit/engine/tests/fixtures/exec-08-inline-version), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "checks": [ + { + "id": "EXEC-08", + "acceptable_verdicts": [ + "FAIL" + ] + } + ] +} diff --git a/docs/reference/how-the-checks-map-to-the-framework.md b/docs/reference/how-the-checks-map-to-the-framework.md index 56b09dad..42d27b12 100644 --- a/docs/reference/how-the-checks-map-to-the-framework.md +++ b/docs/reference/how-the-checks-map-to-the-framework.md @@ -31,7 +31,7 @@ To go the other way, start from a framework heading, find the requirements whose ## Every check, generated -All 72 catalogue checks, each with its requirement(s) and the framework heading(s) the requirement's +All 73 catalogue checks, each with its requirement(s) and the framework heading(s) the requirement's `Source` field links. Generated 2026-10-19 from `catalogue.json`'s `req_ids` and `readiness-requirements.md`'s `Source` fields — the same two steps as the worked example above, run once per check rather than by hand. 14 checks are informational or an emitter and trace to no requirement @@ -78,6 +78,7 @@ per check rather than by hand. 14 checks are informational or an emitter and tra | `EXEC-05` | No generated artifacts committed in-tree | DEGRADER | yes | [REQ-SH-043](../../spec/readiness-requirements.md#req-sh-043--generated-files-marked-or-untracked) | [Phased-Approach.md § Phase 1: Agentic Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-) | | `EXEC-06` | Documented commands resolve | DEGRADER | yes | [REQ-SH-025](../../spec/readiness-requirements.md#req-sh-025--documented-commands-resolve) | [Phased-Approach.md § Phase 1: Agentic Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-) | | `EXEC-07` | No competing configs for one concern | DEGRADER | yes | [REQ-SH-044](../../spec/readiness-requirements.md#req-sh-044--one-configuration-per-concern) | [Phased-Approach.md § Phase 1: Agentic Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-) | +| `EXEC-08` | Tool versions visible to dependency automation | DEGRADER | yes | [REQ-SH-058](../../spec/readiness-requirements.md#req-sh-058--tool-versions-visible-to-dependency-automation) | [Phased-Approach.md § Phase 1: Agentic Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-) | | `HYG-01` | README present and non-placeholder | COSMETIC | no | none | none | | `HYG-02` | CODEOWNERS present | COSMETIC | no | none | none | | `HYG-03` | Pull-request template present | COSMETIC | no | none | none | @@ -112,7 +113,7 @@ per check rather than by hand. 14 checks are informational or an emitter and tra | `TST-04` | Fast path documented | DEGRADER | yes | [REQ-SH-048](../../spec/readiness-requirements.md#req-sh-048--fast-path-documented) | [Phased-Approach.md § Phase 1: Agentic Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-) | | `TST-05` | Coverage configuration present | COSMETIC | yes | [REQ-SH-049](../../spec/readiness-requirements.md#req-sh-049--coverage-reporting) | [greenfield/README.md § 3.1 vitest.config.ts](../../greenfield/README.md#31-vitestconfigts--with-coverage) | -72 checks total. +73 checks total. ## Where the relation is held diff --git a/implement/playbooks/README.md b/implement/playbooks/README.md index 24374bc3..99d395e6 100644 --- a/implement/playbooks/README.md +++ b/implement/playbooks/README.md @@ -58,7 +58,7 @@ in IR04.1, `untrack-generated` in IR05.1, `establish-docs-contract` in IR09.1, |---|---|---|---|---| | [`define-command-surface`](define-command-surface.md) | T1 Command surface and quality tools | CMD-01, CMD-02, CMD-03, CMD-04, EXEC-03, TST-01 | yes | Drafted (IR04.1); applied in IR04.2, IR04.3 | | [`configure-quality-tools`](configure-quality-tools.md) | T1 Command surface and quality tools | TOOL-01, TOOL-02, TOOL-03, TOOL-04 | yes | Drafted (IR04.1); applied in IR04.2, IR04.3 | -| [`pin-execution-environment`](pin-execution-environment.md) | T2 Execution determinism | EXEC-01, EXEC-02 | yes | Drafted (IR05.1); applied and corrected in IR05.2, IR05.3 (T2 closed) | +| [`pin-execution-environment`](pin-execution-environment.md) | T2 Execution determinism | EXEC-01, EXEC-02, EXEC-08 | yes | Drafted (IR05.1); applied and corrected in IR05.2, IR05.3 (T2 closed) | | [`untrack-generated`](untrack-generated.md) | T2 Execution determinism | EXEC-05, EXEC-07 | yes | Drafted (IR05.1); applied and corrected in IR05.2, IR05.3 (T2 closed) | | [`env-inventory-and-secrets`](env-inventory-and-secrets.md) | T3 Secrets and env | EXEC-04, SEC-01, SEC-02, SEC-03 | yes | Applied and corrected (IR06.1-IR06.3) | | [`add-ci-parity`](add-ci-parity.md) | T4 CI parity, hooks, commits | CI-01, CI-02, CI-03, CI-04, CI-05 | yes | Drafted (IR07.1); applied and corrected in IR07.2, IR07.3 (T4 closed) | diff --git a/implement/tools/progress.py b/implement/tools/progress.py index 078b78a5..6ac9e06b 100644 --- a/implement/tools/progress.py +++ b/implement/tools/progress.py @@ -75,7 +75,7 @@ def _span(prefix: str, first: int, last: int, *, skip: Sequence[int] = ()) -> tu # a test holds the two together. CON-04 (unscored, no playbook) sits with CON-01..03: add-deny-rules reads its draft. THEMES: tuple[tuple[str, str, tuple[str, ...]], ...] = ( ("T1", "Command surface and quality tools", _span("CMD", 1, 4) + _span("TOOL", 1, 4)), - ("T2", "Execution determinism", _span("EXEC", 1, 7, skip=(4,))), + ("T2", "Execution determinism", _span("EXEC", 1, 8, skip=(4,))), ("T3", "Secrets and env", _span("SEC", 1, 3) + ("EXEC-04",)), ("T4", "CI parity, hooks, commits", _span("CI", 1, 5) + ("TOOL-05", "TOOL-06")), ("T5", "Tests and baselines", _span("TST", 1, 5) + _span("BASE", 1, 4)), diff --git a/spec/readiness-requirements.md b/spec/readiness-requirements.md index d5e8fc0e..344c7828 100644 --- a/spec/readiness-requirements.md +++ b/spec/readiness-requirements.md @@ -425,6 +425,16 @@ Each of lint, format, test, types and security MUST have at most one authoritati - **Source:** [Phased-Approach.md § Phase 1: Agentic Bootstrap](../docs/framework/phased-approach.md#phase-1-agentic-bootstrap-) - **Checks:** EXEC-07 +### REQ-SH-058 — Tool versions visible to dependency automation + +A tool version that a command, a workflow or a hook runs MUST be held in a manifest that dependency automation +reads, or in a file a configured custom manager reads. It MUST NOT be written only as a literal in the command. + +- **Phase:** 1 +- **Verification:** automated +- **Source:** [Phased-Approach.md § Phase 1: Agentic Bootstrap](../docs/framework/phased-approach.md#phase-1-agentic-bootstrap-) +- **Checks:** EXEC-08 + ## Tests ### REQ-SH-045 — Test verb and suite diff --git a/spec/traceability.md b/spec/traceability.md index ca2161c2..36543451 100644 --- a/spec/traceability.md +++ b/spec/traceability.md @@ -58,6 +58,7 @@ Check IDs and severities are those of the [check catalogue](../audit/catalogue.j | `EXEC-05` | No generated artifacts committed in-tree | DEGRADER | yes | [REQ-SH-043](readiness-requirements.md#req-sh-043--generated-files-marked-or-untracked) | | `EXEC-06` | Documented commands resolve | DEGRADER | yes | [REQ-SH-025](readiness-requirements.md#req-sh-025--documented-commands-resolve) | | `EXEC-07` | No competing configs for one concern | DEGRADER | yes | [REQ-SH-044](readiness-requirements.md#req-sh-044--one-configuration-per-concern) | +| `EXEC-08` | Tool versions visible to dependency automation | DEGRADER | yes | [REQ-SH-058](readiness-requirements.md#req-sh-058--tool-versions-visible-to-dependency-automation) | | `TST-01` | Test verb exists, suite non-empty | BLOCKER | yes | [REQ-SH-045](readiness-requirements.md#req-sh-045--test-verb-and-suite) | | `TST-02` | Suite status known | DEGRADER | yes | [REQ-SH-046](readiness-requirements.md#req-sh-046--suite-passes) | | `TST-03` | Skip / xfail census | DEGRADER | yes | [REQ-SH-047](readiness-requirements.md#req-sh-047--no-skipped-tests) | From c93011b617981b14a4eb4688c6973ebc49387eeb Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 14:02:22 +0530 Subject: [PATCH 033/111] docs(implement): move inline tool versions into a manifest for EXEC-08 pin-execution-environment now resolves EXEC-08. Inputs 7 lists each literal the audit cited and offers the stack's place for tool versions: a package.json's devDependencies (Node), a pyproject dependency group locked in uv.lock (Python), a go.mod tool directive (Go), or a Renovate custom manager when no manifest fits. Step 10 moves each version without upgrading it, locks it, makes the command read it and checks the Dependabot entry. The preference deps.tool_manifest records the choice; a Verification row and a do-not-do line are added. The Python, Node and Java profiles say where tool versions go. Roadmap-Step: C3 Co-Authored-By: Claude Opus 5.5 --- audit/stacks/java.md | 2 ++ audit/stacks/python.md | 2 ++ audit/stacks/typescript-node.md | 4 ++- .../playbooks/pin-execution-environment.md | 31 +++++++++++++++++-- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/audit/stacks/java.md b/audit/stacks/java.md index 83f0f9f9..3d416b19 100644 --- a/audit/stacks/java.md +++ b/audit/stacks/java.md @@ -58,6 +58,8 @@ version read by the build itself; its absence leaves the JDK to whatever `JAVA_H (`dependencyLocking { lockAllConfigurations() }`, writing `gradle.lockfile` files). `EXEC-02` recognises `gradle.lockfile`, at any depth, as the one lockfile name for `jvm`, so a Gradle build without it reads `FAIL` for that check (see Pitfalls). +Tool versions (`EXEC-08`) are usually Gradle or Maven plugins in the build file, which Dependabot reads, so the +check fires here only on an `npx` or `uvx` call with a typed version in a runner file or workflow. ## CI idiom diff --git a/audit/stacks/python.md b/audit/stacks/python.md index bba43175..a57261a2 100644 --- a/audit/stacks/python.md +++ b/audit/stacks/python.md @@ -44,6 +44,8 @@ poetry both honour `.python-version` directly, pip does not read it at all — s actually acts on, not just which file exists. The matching lockfile (`poetry.lock`/`uv.lock`/ `Pipfile.lock`) must be git-tracked at the root; `uv sync --frozen`/`poetry install --no-update`/ `pipenv sync` is the non-interactive install path, never a bare `install` that can resolve new versions. +Tool versions (`EXEC-08`) go in a dependency group of a `pyproject.toml`, locked in `uv.lock`, never inline in +`uvx ruff@0.5.0` or `pip install black==24.1.0`; the runner reads the pin or runs `uv run`. ## CI idiom diff --git a/audit/stacks/typescript-node.md b/audit/stacks/typescript-node.md index 9f1fbc1c..647b6cb3 100644 --- a/audit/stacks/typescript-node.md +++ b/audit/stacks/typescript-node.md @@ -53,7 +53,9 @@ but a `.nvmrc`/`.tool-versions` exact version is stronger and preferred for a fr repository) — none of the three profile repos committed an `.nvmrc`, all three used `engines.node` alone. The lockfile matching the declared package manager (`package-lock.json` for npm, etc.) must be git-tracked at the repository root; `npm ci`/`pnpm install --frozen-lockfile`/`yarn install ---immutable` is the non-interactive setup path (EXEC-03), never a bare `install`. +--immutable` is the non-interactive setup path (EXEC-03), never a bare `install`. Tool versions (`EXEC-08`) go +in `devDependencies`, so `npx ` resolves the locked one; `npx @1.2.3` in a script or workflow is +invisible to Dependabot. ## CI idiom diff --git a/implement/playbooks/pin-execution-environment.md b/implement/playbooks/pin-execution-environment.md index 70e5166b..c25e6e80 100644 --- a/implement/playbooks/pin-execution-environment.md +++ b/implement/playbooks/pin-execution-environment.md @@ -6,8 +6,9 @@ ## Goal Every run of the repository gets the same toolchain and the same dependency versions: each ecosystem has a -committed pin (JDK, Node, package manager), each component has a tracked lockfile that matches its manifest, and -the pins in CI, Dockerfiles and version files agree with each other. +committed pin (JDK, Node, package manager), each component has a tracked lockfile that matches its manifest, the +pins in CI, Dockerfiles and version files agree with each other, and every tool version the gate runs sits in a +manifest that dependency automation reads. ## Checks resolved @@ -16,6 +17,7 @@ the pins in CI, Dockerfiles and version files agree with each other. | EXEC-01 | [09-execution-determinism.md](../../audit/checks/09-execution-determinism.md#exec-01--toolchain-pinned--wrapper-committed) | Commits a pin for every detected ecosystem and makes every other place that names a version agree with it | | EXEC-02 | [09-execution-determinism.md](../../audit/checks/09-execution-determinism.md#exec-02--lockfile-committed-and-consistent) | Commits a lockfile that resolves exactly what the manifest resolves today | | EXEC-03 | [09-execution-determinism.md](../../audit/checks/09-execution-determinism.md#exec-03--non-interactive-setup-path) | Check only (step 9): [`define-command-surface`](define-command-surface.md) owns the fix | +| EXEC-08 | [09-execution-determinism.md](../../audit/checks/09-execution-determinism.md#exec-08--tool-versions-visible-to-dependency-automation) | Moves each tool version written into a command into a manifest with a lockfile, and has the command read it (step 10) | ## Preconditions @@ -76,6 +78,21 @@ the pins in CI, Dockerfiles and version files agree with each other. `partial` with the reason; it does not add a root workspace manifest unless the owner asks, because that changes install paths and image builds. The engine reads only the root manifest, so a nested lockfile reads `PARTIAL` by design ([check pitfall](../../audit/checks/09-execution-determinism.md#exec-02--lockfile-committed-and-consistent)). +7. **Where tool versions go (EXEC-08).** List each literal the audit cited (tool, version, file, line) and confirm + the versions stay as they are: this step moves a version, it never upgrades one. The place follows the stack + profile; offer the owner the matching option: + + | Stack | Manifest for tool versions | How the command reads it | + |---|---|---| + | Node | `devDependencies` of the root (or a `tools/`) `package.json`, with its lockfile | `npx ` with no version, or an npm script | + | Python | A dependency group (`[dependency-groups] tools`) in a `pyproject.toml`, locked in `uv.lock`; `requirements-dev.txt` with `pip-tools` on a pip project | `uv run `, or a runner variable that reads the pin from the manifest | + | Go | A `tool` directive in `go.mod` (Go 1.24 or newer), or a `tools/go.mod` | `go tool ` | + | Rust | No manifest holds a `cargo install` tool: ask the owner; a Renovate custom manager is the fallback | — | + | Any, with Renovate | A `customManagers` regex entry whose file pattern matches the runner file | The command keeps its version; Renovate updates it in place | + + A tool-installer action's `version` input (`setup-uv`, `pnpm/action-setup`) moves to the file the action reads + (`version-file:` pointing at the manifest, or `packageManager` in `package.json`). Record the choice as the + preference `deps.tool_manifest`. ## Steps @@ -108,7 +125,13 @@ the pins in CI, Dockerfiles and version files agree with each other. component. If [`define-command-surface`](define-command-surface.md) has not added it, add it there; this playbook adds no setup script. Record the engine's limit when the documented command is a project script it does not recognise. -10. Commits: toolchain pin, lockfile, version alignment, doc claim. +10. **Move tool versions (EXEC-08).** With the place chosen in Inputs 7: add each tool, at the same version, to the + manifest, lock it with the tool's own command (`npm install --save-dev --save-exact`, `uv lock`, + `go get -tool`), then change each command to read it. Run the gate: it must pass with the same tool versions. + Make sure `dependabot.yml` (or the Renovate config) has an entry for the manifest's ecosystem and directory + ([`add-repo-hygiene`](add-repo-hygiene.md), HYG-09). Edit a workflow only when `ci.edits_authorised` allows; + otherwise list its literals as debt. A repeated copy of a pin (EXEC-08 `PARTIAL`) is removed the same way. +11. Commits: toolchain pin, lockfile, version alignment, doc claim, tool versions. ### Gradle @@ -169,6 +192,7 @@ Profile: [`stacks/typescript-node.md`](../../audit/stacks/typescript-node.md#exe | EXEC-01 | Read the pin file per ecosystem; `grep` the version in every workflow and `Dockerfile*` | One pin per ecosystem and component; every other place names the same version | | EXEC-02 | Gradle: `./gradlew dependencies` compared to the step 1 report, read `gradle.lockfile`. Node: `npm ci` per component, then `git status --porcelain` | Gradle: identical resolved versions, lockfile tracked. Node: `npm ci` succeeds and leaves the tree clean | | EXEC-03 | Read the instruction file | One non-interactive setup command per component (owned by `define-command-surface`) | +| EXEC-08 | Launcher run; `grep` each cited line | No literal tool version in a command, workflow or hook; each tool is in a manifest with a lockfile; `PASS` | Also run the repository's own gate (`./gradlew build`, `npm run build` per component) once on the pinned toolchain, and read the CI run on the branch: it must be green. The engine's re-audit can still read EXEC-02 as @@ -192,6 +216,7 @@ runtime change needs undoing. - Do not edit workflow files without `ci.edits_authorised`. - Do not build images or start containers by hand: use the repository's scripts. - Do not install a version manager the owner did not approve. +- Do not upgrade a tool while moving its version into a manifest. ## State updates From 5d4ff904ea9893fe98e97e80c7d4daae6634a3e7 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 14:45:57 +0530 Subject: [PATCH 034/111] feat: stamp the release version in reports, state and skill bundles The audit report said recipe version 0.1.0 and the implement state 0.2.0-dev, whatever the release. Both RECIPE_VERSION lines now hold the release version, carry the x-release-please-version marker and are release-please extra-files. build_skills.py writes the manifest's version into each bundle's SKILL.md as metadata.version; skills-ref accepts that key and rejects a top-level version. test_release_config.py fails when any stamp differs from the manifest, lacks the marker or is not an extra-files entry. The progress golden test pins the recipe version the example was made with, as it pins the date, so a release bump does not break it. Roadmap-Step: V1 Co-Authored-By: Claude Opus 5.5 --- RELEASE_PROCESS.md | 15 ++++--- audit/tools/report.py | 2 +- audit/tools/tests/test_report.py | 2 +- dev/build_skills.py | 56 +++++++++++++++++++------- dev/tests/test_build_skills.py | 31 ++++++++++++++ dev/tests/test_release_config.py | 40 ++++++++++++++++-- implement/tools/progress.py | 2 +- implement/tools/tests/test_progress.py | 12 +++++- release-please-config.json | 2 +- 9 files changed, 133 insertions(+), 29 deletions(-) diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 1e09a36f..5f02aa86 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -64,14 +64,18 @@ since the last release: | A `feat:` | PATCH | MINOR | | Only `fix:`, `docs:`, `ci:` and other types | PATCH | PATCH | -The version is written in three places, and release-please changes all three in the release pull request: +The version is written in five places, and release-please changes all five in the release pull request: - [`.release-please-manifest.json`](.release-please-manifest.json): the last released version. - `__version__` in [`audit/engine/audit/__init__.py`](audit/engine/audit/__init__.py): what `audit --version` and - each report's provenance print. + the report's `provenance.engine.version` print. +- `RECIPE_VERSION` in [`audit/tools/report.py`](audit/tools/report.py): the report's `provenance.recipe.version`. +- `RECIPE_VERSION` in [`implement/tools/progress.py`](implement/tools/progress.py): the state's `recipe.version`. - The new entry in [`CHANGELOG.md`](CHANGELOG.md). -[`dev/tests/test_release_config.py`](dev/tests/test_release_config.py) fails when the manifest and `__version__` +Each of the three code lines carries the `x-release-please-version` marker and is an `extra-files` entry. +`dev/build_skills.py` copies the manifest's version into each skill bundle's `SKILL.md` as `metadata.version`. +[`dev/tests/test_release_config.py`](dev/tests/test_release_config.py) fails when a stamp and the manifest disagree. The configuration is [`release-please-config.json`](release-please-config.json). ## One-time setup @@ -188,8 +192,9 @@ The push to `main` starts `release-please.yml`. A minute later the release pull gh pr list --base main --head release-please--branches--main ``` -2. Read its diff. It changes `.release-please-manifest.json`, `__version__` in the engine and `CHANGELOG.md`. A second - commit, `style(changelog): ...`, holds the markdownlint fixes. +2. Read its diff. It changes `.release-please-manifest.json`, `__version__` in the engine, the two + `RECIPE_VERSION` lines and `CHANGELOG.md`. A second commit, `style(changelog): ...`, holds the markdownlint + fixes. 3. Check the version. To force a different one, see [Choose the version](#choose-the-version). 4. Without the App token, close and reopen the pull request to start CI: diff --git a/audit/tools/report.py b/audit/tools/report.py index f4887318..e538c4c2 100644 --- a/audit/tools/report.py +++ b/audit/tools/report.py @@ -84,7 +84,7 @@ ) RECIPE_NAME = "brownfield-audit" -RECIPE_VERSION = "0.1.0" +RECIPE_VERSION = "0.3.2" # x-release-please-version SCHEMA_VERSION = "0.1" DRAFT_KIND = "audit-draft" DEFAULT_PROBE_TIMEOUT_S = 120 diff --git a/audit/tools/tests/test_report.py b/audit/tools/tests/test_report.py index 3aba239f..da6814b6 100644 --- a/audit/tools/tests/test_report.py +++ b/audit/tools/tests/test_report.py @@ -290,7 +290,7 @@ def test_a_label_that_is_not_allowed_is_an_intake_error_and_writes_nothing(self) def test_the_provenance_is_what_the_intake_and_the_catalogue_say(self) -> None: document = read_json(self.start()) provenance = document["provenance"] - self.assertEqual(provenance["recipe"], {"name": "brownfield-audit", "version": "0.1.0"}) + self.assertEqual(provenance["recipe"], {"name": "brownfield-audit", "version": rb.RECIPE_VERSION}) self.assertEqual(provenance["agent"], {"harness": "claude-code", "model": "claude-sonnet-5-5"}) self.assertEqual(provenance["catalogue_version"], vr.load_catalogue()["catalogue_version"]) self.assertEqual(provenance["started_at"], "2026-10-10T09:30:00+00:00") diff --git a/dev/build_skills.py b/dev/build_skills.py index 337535f3..b0767c44 100644 --- a/dev/build_skills.py +++ b/dev/build_skills.py @@ -17,7 +17,9 @@ - ``SKILL.md``: the recipe's own file (``audit/SKILL.md``), with its frontmatter ``name`` rewritten from ``audit`` (the source folder's name, which `recipe_lint.py` requires there) to the bundle's own name, - ``agentic-readiness-audit`` (the name `skills-ref` checks against the bundle's folder name). + ``agentic-readiness-audit`` (the name `skills-ref` checks against the bundle's folder name), and a + ``metadata.version`` key added: the release version from ``.release-please-manifest.json`` + (:func:`stamp_frontmatter`), so an installed bundle says which release it is. - ``references/``: the check files (``audit/checks/``), the stack profiles (``audit/stacks/``) and the schema docs (``audit/schema/*.schema.json``, ``SCHEMA.md``, ``LICENSING.md``) — read by the model, not run. - ``scripts/``: the five tools the recipe's Tools table calls (``facts.py``, ``engine.py``, ``report.py``, @@ -35,7 +37,8 @@ **agentic-readiness-implement** carries the implement recipe, a ``LICENSE`` like the audit bundle's, and resolves every link it keeps: -- ``SKILL.md``: ``implement/SKILL.md``, its ``name`` rewritten from ``implement`` to the bundle's name. +- ``SKILL.md``: ``implement/SKILL.md``, its ``name`` rewritten from ``implement`` to the bundle's name, and the + same ``metadata.version`` added. - ``references/``: the state contract, the playbooks, the kickoff prompt, the check files, the stack profiles, ADR 0003, ``spec/command-surface.md`` and the spec decisions, the state schema and its docs, and the parts of the Tools CLI reference that cover the carried tools (:data:`TOOLS_CLI_SECTIONS`). @@ -61,12 +64,13 @@ Failure behaviour: a target problem (the engine's ``build.py`` is missing, or exits nonzero) is exit 1 with the subprocess's stderr in ``data.errors``; so is a source file whose frontmatter ``name`` line or Tools CLI -section is missing. Nothing else in this tool can fail the way a user-facing argument can, since it takes none +section is missing, or whose frontmatter already has a ``metadata`` key. Nothing else in this tool can fail the way a user-facing argument can, since it takes none but ``--out-dir``. """ from __future__ import annotations +import json import os import re import shutil @@ -83,6 +87,7 @@ IMPLEMENT = REPO / "implement" IMPLEMENT_TOOLS = IMPLEMENT / "tools" LICENSE_CODE = REPO / "LICENSE-CODE" +MANIFEST = REPO / ".release-please-manifest.json" # `_common` (and `recipe_lint`'s own `_common` import) live in audit/tools/, not beside this script. if str(AUDIT_TOOLS) not in sys.path: @@ -143,14 +148,38 @@ def _write_license(bundle_dir: Path) -> None: shutil.copy2(LICENSE_CODE, bundle_dir / "LICENSE") -def _write_skill_md(bundle_dir: Path) -> None: - text = (AUDIT / "SKILL.md").read_text(encoding="utf-8") - if SKILL_MD_SOURCE_NAME not in text: +def release_version() -> str: + """The release version: the root package's entry in `.release-please-manifest.json` (ADR 0005).""" + version = json.loads(MANIFEST.read_text(encoding="utf-8"))["."] + if not isinstance(version, str): + raise _common.ToolError(f".release-please-manifest.json: the '.' entry is not a string: {version!r}") + return version + + +def stamp_frontmatter(text: str, source: str, source_name: str, bundle_name: str, version: str) -> str: + """Rewrite a `SKILL.md` for its bundle: the `name` line, and a `metadata.version` key before the closing `---`. + + The Agent Skills spec allows `metadata` (a string map) and rejects a top-level `version`, so the release + version goes under `metadata`. `source` names the file in an error. + """ + if source_name not in text: raise _common.ToolError( - f"audit/SKILL.md's frontmatter does not contain {SKILL_MD_SOURCE_NAME!r}; " - "it may have been renamed without updating this tool" + f"{source}'s frontmatter does not contain {source_name!r}; it may have been renamed without updating this tool" ) - (bundle_dir / "SKILL.md").write_text(text.replace(SKILL_MD_SOURCE_NAME, SKILL_MD_BUNDLE_NAME), encoding="utf-8") + if not text.startswith("---\n") or "\n---\n" not in text[3:]: + raise _common.ToolError(f"{source} does not start with a frontmatter block between two '---' lines") + end = text.index("\n---\n", 3) + 1 + frontmatter = text[:end] + if "\nmetadata:" in frontmatter: + raise _common.ToolError(f"{source}'s frontmatter already has a metadata key; merge the version into it here") + frontmatter = frontmatter.replace(source_name, bundle_name, 1) + f'metadata:\n version: "{version}"\n' + return frontmatter + text[end:] + + +def _write_skill_md(bundle_dir: Path) -> None: + text = (AUDIT / "SKILL.md").read_text(encoding="utf-8") + stamped = stamp_frontmatter(text, "audit/SKILL.md", SKILL_MD_SOURCE_NAME, SKILL_MD_BUNDLE_NAME, release_version()) + (bundle_dir / "SKILL.md").write_text(stamped, encoding="utf-8") def _write_references(bundle_dir: Path) -> None: @@ -429,12 +458,9 @@ def build_implement_bundle(out_dir: Path) -> Path: for source, dest in markdown.items(): text = source.read_text(encoding="utf-8") if dest == "SKILL.md": - if IMPLEMENT_SKILL_SOURCE_NAME not in text: - raise _common.ToolError( - f"implement/SKILL.md's frontmatter does not contain {IMPLEMENT_SKILL_SOURCE_NAME!r}; " - "it may have been renamed without updating this tool" - ) - text = text.replace(IMPLEMENT_SKILL_SOURCE_NAME, IMPLEMENT_SKILL_BUNDLE_NAME, 1) + text = stamp_frontmatter( + text, "implement/SKILL.md", IMPLEMENT_SKILL_SOURCE_NAME, IMPLEMENT_SKILL_BUNDLE_NAME, release_version() + ) elif dest == "references/tools-cli.md": text = extract_sections(text, TOOLS_CLI_SECTIONS) texts[dest] = (source, text) diff --git a/dev/tests/test_build_skills.py b/dev/tests/test_build_skills.py index b1dc3363..913ef97b 100644 --- a/dev/tests/test_build_skills.py +++ b/dev/tests/test_build_skills.py @@ -37,6 +37,19 @@ FENCE = re.compile(r"^\s*(```+|~~~+)") +def manifest_version() -> str: + version = json.loads((bs.REPO / ".release-please-manifest.json").read_text(encoding="utf-8"))["."] + assert isinstance(version, str) + return version + + +def frontmatter_version(text: str) -> str | None: + """The `metadata.version` value inside a `SKILL.md`'s frontmatter, or None.""" + frontmatter = text[: text.index("\n---\n", 3)] + match = re.search(r'^metadata:\n version: "([^"]+)"$', frontmatter, re.MULTILINE) + return match[1] if match else None + + def call(argv: Sequence[str]) -> tuple[int, dict[str, Any], str]: out, err = io.StringIO(), io.StringIO() code = _common.run_tool(bs.INFO, bs.handler, list(argv), stdout=out, stderr=err) @@ -91,6 +104,23 @@ def test_skill_md_frontmatter_is_rewritten_for_the_bundle(self) -> None: text = (bundle_dir / "SKILL.md").read_text(encoding="utf-8") self.assertIn(f"\nname: {bs.BUNDLE_NAME}\n", text) self.assertNotIn("\nname: audit\n", text) + self.assertEqual(frontmatter_version(text), manifest_version()) + + def test_stamp_frontmatter_puts_the_version_inside_the_frontmatter(self) -> None: + text = "---\nname: demo\ndescription: A demo.\n---\n\n# Demo\n" + stamped = bs.stamp_frontmatter(text, "demo/SKILL.md", "name: demo\n", "name: bundle\n", "9.8.7") + self.assertEqual( + stamped, '---\nname: bundle\ndescription: A demo.\nmetadata:\n version: "9.8.7"\n---\n\n# Demo\n' + ) + + def test_stamp_frontmatter_refuses_a_metadata_key_it_would_duplicate(self) -> None: + text = "---\nname: demo\nmetadata:\n owner: x\n---\n# Demo\n" + with self.assertRaisesRegex(_common.ToolError, "already has a metadata key"): + bs.stamp_frontmatter(text, "demo/SKILL.md", "name: demo\n", "name: bundle\n", "9.8.7") + + def test_stamp_frontmatter_refuses_a_missing_name_line(self) -> None: + with self.assertRaisesRegex(_common.ToolError, "does not contain 'name: demo"): + bs.stamp_frontmatter("---\nname: other\n---\n", "demo/SKILL.md", "name: demo\n", "name: b\n", "1.0.0") def test_handler_reports_the_bundle_path_and_file_count(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -200,6 +230,7 @@ def test_skill_md_frontmatter_is_rewritten_for_the_bundle(self) -> None: text = (self.bundle / "SKILL.md").read_text(encoding="utf-8") self.assertIn(f"\nname: {bs.IMPLEMENT_BUNDLE_NAME}\n", text) self.assertNotIn("\nname: implement\n", text) + self.assertEqual(frontmatter_version(text), manifest_version()) def test_no_framework_tool_command_is_left(self) -> None: for file in self.markdown_files(): diff --git a/dev/tests/test_release_config.py b/dev/tests/test_release_config.py index acb81646..619ec054 100644 --- a/dev/tests/test_release_config.py +++ b/dev/tests/test_release_config.py @@ -1,12 +1,17 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # Copyright (C) 2026 K. C. Ramakrishna -"""The release-please setup stays consistent with the engine and the changelog (ADR 0005, RELEASE_PROCESS.md). +"""The release-please setup stays consistent with the version stamps and the changelog (ADR 0005, RELEASE_PROCESS.md). release-please bumps the version in ``.release-please-manifest.json`` and in each of its ``extra-files``. These tests -fail when the engine's ``__version__`` drifts from the manifest (as it did before release-please: the engine said -0.1.0 while v0.3.0 was released), when the engine's version line loses the marker that release-please looks for, or -when a hand-written ``[Unreleased]`` heading comes back into the changelog that release-please writes. +fail when a version stamp drifts from the manifest (as the engine's did before release-please: it said 0.1.0 while +v0.3.0 was released, and the audit report's recipe version said 0.1.0 through v0.3.2), when a stamp line loses the +marker that release-please looks for or is not an ``extra-files`` entry, or when a hand-written ``[Unreleased]`` +heading comes back into the changelog that release-please writes. + +The stamps are the engine's ``__version__`` and the two recipes' ``RECIPE_VERSION`` (the audit report's +``provenance.recipe.version`` and the implement state's ``recipe.version``). The skill bundles' ``metadata.version`` +is copied from the manifest at build time; ``test_build_skills`` checks it. """ from __future__ import annotations @@ -23,6 +28,12 @@ ENGINE_INIT = "audit/engine/audit/__init__.py" MARKER = "x-release-please-version" VERSION_LINE = re.compile(r'^__version__ = "(?P[^"]+)"(?P.*)$', re.MULTILINE) +#: Every file that holds a copy of the release version, and the pattern of the one line that holds it. +STAMPS = { + ENGINE_INIT: VERSION_LINE, + "audit/tools/report.py": re.compile(r'^RECIPE_VERSION = "(?P[^"]+)"(?P.*)$', re.MULTILINE), + "implement/tools/progress.py": re.compile(r'^RECIPE_VERSION = "(?P[^"]+)"(?P.*)$', re.MULTILINE), +} class ReleaseConfigTest(unittest.TestCase): @@ -42,6 +53,27 @@ def test_engine_version_line_carries_the_marker(self) -> None: def test_engine_init_is_an_extra_file(self) -> None: self.assertIn(ENGINE_INIT, self.config["extra-files"]) + def stamp_lines(self) -> dict[str, re.Match[str]]: + lines = {} + for path, pattern in STAMPS.items(): + matches = list(pattern.finditer((REPO / path).read_text(encoding="utf-8"))) + self.assertEqual(len(matches), 1, f"{path}: expected one version line, found {len(matches)}") + lines[path] = matches[0] + return lines + + def test_every_stamp_matches_the_manifest(self) -> None: + for path, line in self.stamp_lines().items(): + with self.subTest(path=path): + self.assertEqual(line["version"], self.manifest["."]) + + def test_every_stamp_line_carries_the_marker(self) -> None: + for path, line in self.stamp_lines().items(): + with self.subTest(path=path): + self.assertIn(MARKER, line["rest"]) + + def test_every_stamp_file_is_an_extra_file(self) -> None: + self.assertEqual(sorted(self.config["extra-files"]), sorted(STAMPS)) + def test_one_root_package_with_plain_v_tags(self) -> None: self.assertEqual(list(self.config["packages"]), ["."]) self.assertEqual(list(self.manifest), ["."]) diff --git a/implement/tools/progress.py b/implement/tools/progress.py index 078b78a5..322e59e9 100644 --- a/implement/tools/progress.py +++ b/implement/tools/progress.py @@ -48,7 +48,7 @@ import validate_report RECIPE_NAME = "brownfield-implement" -RECIPE_VERSION = "0.2.0-dev" +RECIPE_VERSION = "0.3.2" # x-release-please-version SCHEMA_VERSION = "0.2" SUPPORTED_SCHEMA_VERSIONS = ("0.1", "0.2") diff --git a/implement/tools/tests/test_progress.py b/implement/tools/tests/test_progress.py index 227d27c8..0973865f 100644 --- a/implement/tools/tests/test_progress.py +++ b/implement/tools/tests/test_progress.py @@ -40,6 +40,9 @@ MD_NAME = "example-service-agentic-readiness-STATE.md" # The values the checked-in example was made with. EXAMPLE_ARGS = ("--date", "2026-10-29", "--framework-rev", "cafecafecafecafecafecafecafecafecafecafe") +# The recipe version it was made with. release-please bumps `progress.RECIPE_VERSION` each release; the checked-in +# example keeps the version that made it, the same way it keeps its date. +EXAMPLE_RECIPE_VERSION = "0.2.0-dev" def call(argv: Sequence[str]) -> tuple[int, dict[str, Any], str]: @@ -118,8 +121,15 @@ def test_no_subcommand_is_a_usage_error(self) -> None: class InitTests(TargetCase): + def test_init_stamps_the_release_version(self) -> None: + self.init_ok() + state = json.loads(self.json_path.read_text(encoding="utf-8")) + self.assertEqual(state["recipe"]["version"], progress.RECIPE_VERSION) + self.assertIn(f"brownfield-implement {progress.RECIPE_VERSION},", self.md_path.read_text(encoding="utf-8")) + def test_init_writes_the_example_state_and_its_markdown(self) -> None: - envelope = self.init_ok() + with mock.patch.object(progress, "RECIPE_VERSION", EXAMPLE_RECIPE_VERSION): + envelope = self.init_ok() self.assertEqual(self.json_path.read_bytes(), EXAMPLE_STATE.read_bytes()) self.assertEqual(self.md_path.read_bytes(), EXAMPLE_MD.read_bytes()) data = envelope["data"] diff --git a/release-please-config.json b/release-please-config.json index 39b56389..485717e8 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -20,7 +20,7 @@ { "type": "style", "section": "Style", "hidden": true }, { "type": "chore", "section": "Chores", "hidden": true } ], - "extra-files": ["audit/engine/audit/__init__.py"], + "extra-files": ["audit/engine/audit/__init__.py", "audit/tools/report.py", "implement/tools/progress.py"], "packages": { ".": {} } From 01be98d19cbc6a214f6b0b9fcf68d07ba206bab4 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 14:44:22 +0530 Subject: [PATCH 035/111] docs: add generated audit and implement master lists Two reader-facing pages in docs/reference/ list every audit check by area and every implement playbook by theme. dev/build_masters.py generates them from catalogue.json, the check files and the playbooks; make check runs masters-check so a stale page fails the gate. make update-masters rewrites them. Linked from README.md, docs/README.md, AGENTS.md, audit/AGENTS.md, implement/AGENTS.md and the dev-tooling reference. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 8 +- Makefile | 19 +- README.md | 1 + audit/AGENTS.md | 3 + dev/build_masters.py | 475 +++++++++++++++++ dev/tests/test_build_masters.py | 123 +++++ docs/README.md | 2 + docs/reference/audit-master.md | 787 +++++++++++++++++++++++++++++ docs/reference/dev-tooling.md | 5 +- docs/reference/implement-master.md | 450 +++++++++++++++++ implement/AGENTS.md | 3 + 11 files changed, 1868 insertions(+), 8 deletions(-) create mode 100644 dev/build_masters.py create mode 100644 dev/tests/test_build_masters.py create mode 100644 docs/reference/audit-master.md create mode 100644 docs/reference/implement-master.md diff --git a/AGENTS.md b/AGENTS.md index 7ee4a835..c1c28d58 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,9 +30,10 @@ make lint # read-only lint make typecheck # mypy, strict, on audit/tools, implement/tools and dev, and the engine's build.py make test # the engine's suite, then the audit tools', the implement tools' and the dev checks' make check # THE gate: format:check, typecheck, test, the engine's --verify-rules, the schema - # cross-check, recipe-lint, workflow-lint (actionlint, zizmor, check-jsonschema), spell (typos) + # cross-check, recipe-lint, masters-check, workflow-lint (actionlint, zizmor, check-jsonschema), spell (typos) make security # secrets scan, then the dependency audit (osv-scanner; needs Go and the network) make commit-lint # Conventional Commits subjects on this branch (RANGE=origin/dev..HEAD); CI runs it on each PR +make update-masters # regenerate docs/reference/audit-master.md and implement-master.md (check runs masters-check) ``` One test, without the whole suite. Run the engine's tests from `audit/engine/`: @@ -75,6 +76,9 @@ no vendor-specific instruction file exists beside it. `make docs-lint` compare the live count with the baseline and fail when it went up (a ratchet). A baseline only goes down, a coverage floor only goes up; raising either needs the owner. When a count falls, lower the baseline in the same change. +- The audit and implement master lists (`docs/reference/audit-master.md`, `docs/reference/implement-master.md`) are + generated from `audit/catalogue.json`, `audit/checks/` and `implement/playbooks/`. After you change one of those, + run `make update-masters` and commit the pages in the same change. Never edit the pages by hand. - After a deliberate change to a fixture or test that holds a key-shaped string, run `make update-secrets-baseline` and review the diff. - Never weaken a gate to get green: no `.skip`, no rule turned off, no assertion relaxed. If you cannot pass a gate, @@ -156,6 +160,8 @@ A change is done when all of this holds: - Documentation index, style guide and decisions of record: `docs/README.md`, `docs/STYLE_GUIDE.md`, `docs/architecture/adrs/README.md`. - Requirements: `spec/README.md`, and `spec/readiness-requirements.md`. +- What an audit checks and what an implement run changes, for a reader: `docs/reference/audit-master.md` and + `docs/reference/implement-master.md`. - Terms: `docs/glossary.md` and `docs/framework/glossary.md`. - The engine's design, checks and ADRs: `audit/engine/docs/architecture/README.md`. - The tool contract and the tool catalogue: `audit/tools/README.md`, `audit/tools/CATALOGUE.md`. diff --git a/Makefile b/Makefile index 1c74c15a..96be310f 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,8 @@ # make typecheck mypy, strict, on the typed scope below # make test the engine's unittest suite, then the recipe tools' (the catalogue checks), then the dev checks' # make check THE gate: format:check -> typecheck -> test, plus the engine's rule check, the schema -# cross-check, the recipe linter, the workflow lint and the spell check +# cross-check, the recipe linter, the master-list freshness check, +# the workflow lint and the spell check # make security secrets scan against the committed baseline, then the dependency audit (osv-scanner) of # the dev tools' lockfiles; needs Go and the network # make coverage (extra) both unittest suites under coverage.py; fails below the floor in .coveragerc @@ -66,7 +67,7 @@ COVERAGE_DATA := $(CURDIR)/dev/dist/coverage/.coverage # The Markdown that `docs-lint` holds to zero violations. A ratchet: a path joins once it is clean, and the list # only grows. Everything else is reported by `docs-lint-all` and never fails. DOCS_LINT_PATHS := 'implement/SKILL.md' 'implement/state.md' 'implement/playbooks/*.md' 'implement/prompts/*.md' \ - 'docs/README.md' 'docs/STYLE_GUIDE.md' 'docs/architecture/**/*.md' 'docs/reference/branch-protection.md' 'docs/reference/dev-tooling.md' 'docs/how-to/contribute-a-change.md' 'docs/how-to/set-up-agent-tools.md' 'docs/COMMUNICATION_RULES.md' 'CONTRIBUTING.md' 'CHANGELOG.md' 'RELEASE_PROCESS.md' 'SECURITY.md' \ + 'docs/README.md' 'docs/STYLE_GUIDE.md' 'docs/architecture/**/*.md' 'docs/reference/branch-protection.md' 'docs/reference/dev-tooling.md' 'docs/reference/audit-master.md' 'docs/reference/implement-master.md' 'docs/how-to/contribute-a-change.md' 'docs/how-to/set-up-agent-tools.md' 'docs/COMMUNICATION_RULES.md' 'CONTRIBUTING.md' 'CHANGELOG.md' 'RELEASE_PROCESS.md' 'SECURITY.md' \ 'docs/explanation/user-overview/03-trust-and-safety.md' 'docs/explanation/contributor-deep-dive/04-stack-profiles.md' 'docs/explanation/contributor-deep-dive/05-evaluation.md' 'docs/how-to/add-a-stack-profile.md' 'docs/reference/harness-notes.md' \ 'audit/engine/docs/how-to/read-the-report.md' @@ -87,11 +88,11 @@ TYPED_SOURCES := $(shell find audit/tools implement/tools dev -name '*.py' -not .DEFAULT_GOAL := help .NOTPARALLEL: -.PHONY: help setup format format-check format\:check lint typecheck test coverage verify-rules crosscheck recipe-lint check security update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint spell spell-fix +.PHONY: help setup format format-check format\:check lint typecheck test coverage verify-rules crosscheck recipe-lint masters-check update-masters check security update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint spell spell-fix help: @echo "verbs: format format:check (format-check) lint typecheck test check security" - @echo "extra: setup (checks prerequisites, installs nothing) coverage self-audit update-audit-baseline verify-rules crosscheck recipe-lint workflow-lint commit-lint spell spell-fix update-secrets-baseline validate-skills" + @echo "extra: setup (checks prerequisites, installs nothing) coverage self-audit update-audit-baseline verify-rules crosscheck recipe-lint masters-check update-masters workflow-lint commit-lint spell spell-fix update-secrets-baseline validate-skills" @echo "docs: docs-lint docs-lint-all docs-prose [FILES=...] docs-mermaid FILES=... changelog-fix" # There is nothing to install into the project: the runtime code is standard-library Python and the dev tools run @@ -168,6 +169,14 @@ crosscheck: recipe-lint: $(PYTHON) dev/recipe_lint.py . +# The audit and implement master lists (docs/reference/*-master.md) are generated from the catalogue, the check files +# and the playbooks. masters-check fails when either page is stale; update-masters rewrites them. +masters-check: + $(PYTHON) dev/build_masters.py --check + +update-masters: + $(PYTHON) dev/build_masters.py --write + # Lints the GitHub Actions workflows and checks .github/dependabot.yml. All three tools run offline. # Read-only: report each misspelling with its file and line. `make spell-fix` writes the suggested fixes; review them. spell: @@ -188,7 +197,7 @@ RANGE ?= origin/dev..HEAD commit-lint: $(PYTHON) dev/commit_lint.py --range $(RANGE) -check: format-check typecheck test verify-rules crosscheck recipe-lint workflow-lint spell +check: format-check typecheck test verify-rules crosscheck recipe-lint masters-check workflow-lint spell # No third-party runtime dependencies exist to audit, so the gate is a secrets scan. The baseline # lists the deliberate key-shaped strings in the fixtures and tests; a new hit fails the scan. diff --git a/README.md b/README.md index e596289d..bb7a5b01 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ it pushes a branch only if you opt in. | Install the skills from the release zips, with no clone | [Install the skills](docs/how-to/install-the-skills.md) | | See a real run: one repository audited, fixed and audited again | [Worked example](audit/schema/examples/server-worked-example/README.md) | | Start a new repository that is agent-ready from the first commit | [Greenfield](greenfield/README.md) | +| See every check an audit runs, and what an implement run changes | [Audit master list](docs/reference/audit-master.md), [Implement master list](docs/reference/implement-master.md) | | Read the method: the seven phases and the requirements | [Phased approach](docs/framework/phased-approach.md), [`spec/`](spec/README.md) | | Find any other page | [Documentation index](docs/README.md) | diff --git a/audit/AGENTS.md b/audit/AGENTS.md index 24091000..758feec7 100644 --- a/audit/AGENTS.md +++ b/audit/AGENTS.md @@ -25,6 +25,9 @@ To audit a repository, follow [`SKILL.md`](SKILL.md). The rest of this file is f tool. - Keep `tools/CATALOGUE.md` the same as each tool's `--describe` output. Regenerate it with `python3 dev/recipe_lint.py --write-catalogue .` from the repository root. +- A change to `catalogue.json` or to a file in `checks/` changes the generated + [audit master list](../docs/reference/audit-master.md). Run `make update-masters` from the repository root and + commit the result in the same change. - Never run a target's own commands directly. Use the engine's probe runner. It is opt-in: `--run-gates` plus `--allow-probe `. - The model's judgment cites evidence. The tools do the counting. diff --git a/dev/build_masters.py b/dev/build_masters.py new file mode 100644 index 00000000..12a3b1d4 --- /dev/null +++ b/dev/build_masters.py @@ -0,0 +1,475 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna + +"""Build the two reader-facing master lists from the recipe sources, or check that they are current. + + build_masters.py --check exit 1 when a page differs from what the sources give (``make masters-check``) + build_masters.py --write rewrite both pages in place (``make update-masters``) + +The pages are ``docs/reference/audit-master.md`` (every audit check, grouped by area and family) and +``docs/reference/implement-master.md`` (every playbook, grouped by theme). Their sources: + +- ``audit/catalogue.json``: each check's id, title, severity, scoring, method and playbooks. +- ``audit/checks/NN-*.md``: each family's name (the H1) and each check's ``Why it matters`` and rubric ``PASS``. +- ``implement/playbooks/README.md``: each playbook's theme (the "Playbook list" table). +- ``implement/playbooks/.md``: each playbook's ``Goal``, ``Checks resolved`` and ``Do-not-do`` sections. + +A page is never edited by hand. ``make check`` runs ``--check``, so a change to a source that leaves a page stale +fails the gate, in CI too. Links copied from a source are rewritten relative to ``docs/reference/``; +``recipe_lint.py`` then checks that every link and anchor resolves. Exit 2 when a source is malformed (a family +with no area, a check with no section, a playbook with no theme). Standard library only. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import textwrap +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Sequence + +ROOT = Path(__file__).resolve().parent.parent +OUT_DIR = Path("docs/reference") +AUDIT_PAGE = OUT_DIR / "audit-master.md" +IMPLEMENT_PAGE = OUT_DIR / "implement-master.md" +CATALOGUE = Path("audit/catalogue.json") +CHECKS_DIR = Path("audit/checks") +PLAYBOOKS_DIR = Path("implement/playbooks") + +# The reader's grouping of the check families. Every family in the catalogue must appear here exactly once; a new +# family fails the build until it is placed. +AREAS: tuple[tuple[str, str, tuple[str, ...]], ...] = ( + ( + "Agent context", + "Can an agent find the code, the rules and the documents it needs before it edits anything?", + ("IDX", "AGT", "DOC", "NAV"), + ), + ( + "Command surface and quality gates", + "Can an agent run one command that formats, lints, type-checks and tests, and trust its answer?", + ("CMD", "TOOL", "TST", "BASE"), + ), + ( + "Enforcement in CI", + "Does the pipeline run the same gate the agent runs locally, and does it block the merge?", + ("CI",), + ), + ( + "Reproducible execution", + "Does the same command give the same result on a fresh machine, without a human in the loop?", + ("EXEC",), + ), + ( + "Safety boundaries and secrets", + "Is it written down, and enforced, what an agent must never touch or reveal?", + ("CON", "SEC"), + ), + ( + "Team hygiene (informational)", + "Is the repository run like a team project? Reported for the whole picture; never scored.", + ("HYG",), + ), +) + +SEVERITY_ORDER = ("BLOCKER", "DEGRADER", "COSMETIC") +WIDTH = 120 # markdownlint's MD013 line length for prose (.markdownlint-cli2.jsonc) +METHOD_LABEL = {"automated": "engine check", "agent": "agent judgement"} + +_LINK = re.compile(r"(? str: + """Rewrite each relative link in ``text`` (written in ``source``) so it resolves from :data:`OUT_DIR`.""" + + def fix(match: re.Match[str]) -> str: + label, target = match.group(1), match.group(2) + if _EXTERNAL.match(target): + return match.group(0) + path_part, hash_mark, anchor = target.partition("#") + # A same-page anchor in the source points at the source page, not at this one. + resolved = source if path_part == "" else source.parent / path_part + moved = os.path.relpath(os.path.normpath(resolved), OUT_DIR) + return f"{label}({Path(moved).as_posix()}{hash_mark}{anchor})" + + return _LINK.sub(fix, text) + + +def section(text: str, heading: str) -> str: + """The body of the ``## heading`` section of ``text``, stripped; empty when there is none.""" + found = re.search(rf"^## {re.escape(heading)}\s*$(.*?)(?=^## |\Z)", text, re.MULTILINE | re.DOTALL) + return found.group(1).strip() if found else "" + + +def field_text(body: str, name: str) -> str: + """One fixed-anatomy field of a check section, its continuation lines joined into one line.""" + found = re.search(rf"^{re.escape(name)}\s+(.*?)(?=^\S|\Z)", body, re.MULTILINE | re.DOTALL) + return " ".join(found.group(1).split()) if found else "" + + +def read_families(root: Path) -> tuple[dict[str, str], dict[str, tuple[Path, str]]]: + """Each family's name, and each check's ``(file, section body)``, from the check files.""" + names: dict[str, str] = {} + sections: dict[str, tuple[Path, str]] = {} + for path in sorted((root / CHECKS_DIR).glob("[0-9][0-9]-*.md")): + text = path.read_text(encoding="utf-8") + h1 = _FAMILY_H1.search(text) + if h1 is None: + raise SourceError(f"{path.relative_to(root)}: no '# (``)' heading") + names[h1.group(2)] = h1.group(1) + for found in _CHECK_SECTION.finditer(text): + sections[found.group(1)] = (path.relative_to(root), found.group(2)) + return names, sections + + +def read_checks(root: Path) -> tuple[str, list[Check], dict[str, str]]: + """The catalogue version, every check in catalogue order, and each family's name.""" + catalogue: dict[str, Any] = json.loads((root / CATALOGUE).read_text(encoding="utf-8")) + names, sections = read_families(root) + checks: list[Check] = [] + for entry in catalogue["checks"]: + if entry["id"] not in sections: + raise SourceError(f"{entry['id']} has no section in {CHECKS_DIR}/") + path, body = sections[entry["id"]] + rubric = field_text(body, "Rubric") + passes = re.search(r"PASS:(.*?)(?=PARTIAL:|FAIL:|NOT_APPLICABLE:|UNKNOWN when:|\Z)", rubric) + checks.append( + Check( + id=entry["id"], + title=entry["title"], + family=entry["family"], + severity=entry["severity"], + scored=bool(entry["scored"]), + method=entry["method"], + playbooks=tuple(entry["playbooks"]), + why=rebase_links(field_text(body, "Why it matters"), path), + passes=rebase_links(passes.group(1).strip() if passes else "", path), + ) + ) + placed = [family for _, _, families in AREAS for family in families] + unplaced = sorted({check.family for check in checks} - set(placed)) + if unplaced: + raise SourceError(f"family {', '.join(unplaced)} has no area in build_masters.AREAS") + missing = sorted({check.family for check in checks} - set(names)) + if missing: + raise SourceError(f"family {', '.join(missing)} has no check file in {CHECKS_DIR}/") + return str(catalogue["catalogue_version"]), checks, names + + +def read_playbooks(root: Path) -> list[Playbook]: + """Every playbook, in the order of the playbook list in ``implement/playbooks/README.md``.""" + index = (root / PLAYBOOKS_DIR / "README.md").read_text(encoding="utf-8") + rows = _THEME_ROW.findall(index) + if not rows: + raise SourceError(f"{PLAYBOOKS_DIR}/README.md: no playbook list rows") + listed = {name for name, _, _ in rows} + on_disk = {path.stem for path in (root / PLAYBOOKS_DIR).glob("*.md")} - {"README"} + if listed != on_disk: + raise SourceError(f"{PLAYBOOKS_DIR}/README.md lists {sorted(listed)}; the folder holds {sorted(on_disk)}") + playbooks: list[Playbook] = [] + for name, theme, theme_name in rows: + path = PLAYBOOKS_DIR / f"{name}.md" + text = (root / path).read_text(encoding="utf-8") + resolves = tuple( + (check_id, rebase_links(what.strip(), path)) + for check_id, what in _RESOLVED_ROW.findall(section(text, "Checks resolved")) + ) + goal = section(text, "Goal") + if not goal or not resolves: + raise SourceError(f"{path}: needs a Goal section and a Checks resolved table") + playbooks.append( + Playbook( + name=name, + theme=theme, + theme_name=theme_name.strip(), + goal=rebase_links(goal, path), + resolves=resolves, + never=rebase_links(section(text, "Do-not-do"), path), + ) + ) + return playbooks + + +# --- Rendering ---------------------------------------------------------------------------------------------- + + +def slug(heading: str) -> str: + """GitHub's anchor for a heading (the same rule as ``recipe_lint.slugify``).""" + return re.sub(r"[^\w\s-]", "", heading.lower()).replace(" ", "-") + + +def wrap(text: str, first: str = "", rest: str = "") -> list[str]: + """``text`` as prose lines of at most :data:`WIDTH` columns, never broken inside a word.""" + return textwrap.wrap( + text, WIDTH, initial_indent=first, subsequent_indent=rest, break_long_words=False, break_on_hyphens=False + ) + + +def check_heading(check: Check) -> str: + return f"`{check.id}` {check.title}" + + +def generated_note(sources: str) -> list[str]: + return wrap( + f"**Generated** by [`dev/build_masters.py`](../../dev/build_masters.py) from {sources}. Do not edit it by " + "hand: change a source, then run `make update-masters`. `make check` fails when this page is stale.", + "> ", + "> ", + ) + + +def count_line(checks: Sequence[Check]) -> str: + parts = [ + f"{sum(c.severity == s for c in checks)} {s}" for s in SEVERITY_ORDER if any(c.severity == s for c in checks) + ] + return ", ".join(parts) + + +def render_audit(version: str, checks: Sequence[Check], names: dict[str, str]) -> str: + by_family: dict[str, list[Check]] = {} + for check in checks: + by_family.setdefault(check.family, []).append(check) + scored = sum(c.scored for c in checks) + agent = sum(c.method == "agent" for c in checks) + out: list[str] = [ + "# Audit master list", + "", + "> **Audience:** CTOs, senior engineers and anyone who wants to know what an audit looks at.", + "> **Source of truth:** [`catalogue.json`](../../audit/catalogue.json) and the check files in", + "> [`audit/checks/`](../../audit/checks/README.md).", + ">", + *generated_note("the catalogue and the check files"), + "", + "---", + "", + *wrap( + f"The audit runs **{len(checks)} checks** in {len(by_family)} families (catalogue {version}). " + f"{scored} are scored; the rest are reported for the whole picture and earn no credit. " + f"The engine decides {len(checks) - agent} checks from the files alone; the agent judges {agent}. " + "Each check below says what passes and why it matters. To fix a finding, follow the playbook it names " + "in the [implement master list](implement-master.md)." + ), + "", + "| Severity | Meaning |", + "|---|---|", + "| `BLOCKER` | An agent cannot work safely until this is fixed |", + "| `DEGRADER` | An agent can work, but slower, with more mistakes or with less trust in its result |", + "| `COSMETIC` | Worth fixing; it does not change what an agent can do |", + "", + "## At a glance", + "", + "| Area | The question it answers | Families | Checks |", + "|---|---|---|---|", + ] + for area, question, families in AREAS: + in_area = [c for f in families for c in by_family.get(f, [])] + links = ", ".join(f"[`{f}`](#{slug(f'{names[f]} ({f})')})" for f in families if f in by_family) + out.append(f"| [{area}](#{slug(area)}) | {question} | {links} | {len(in_area)} ({count_line(in_area)}) |") + for area, question, families in AREAS: + out += ["", f"## {area}", "", question] + for family in families: + members = by_family.get(family, []) + if not members: + continue + out += ["", f"### {names[family]} ({family})", "", f"{len(members)} checks: {count_line(members)}."] + for check in members: + facts = [f"`{check.severity}`", "scored" if check.scored else "not scored", METHOD_LABEL[check.method]] + if check.playbooks: + fixes = ", ".join(f"[`{p}`](implement-master.md#{p})" for p in check.playbooks) + facts.append(f"fixed by {fixes}") + else: + facts.append("no playbook (the audit emits it as an input)") + out += [ + "", + f"#### {check_heading(check)}", + "", + *wrap(" · ".join(facts)), + "", + *wrap(f"**Passes when:** {check.passes}", "- ", " "), + *wrap(f"**Why it matters:** {check.why}", "- ", " "), + ] + out += [ + "", + "## Read more", + "", + "- The full rubric of each check (PARTIAL, FAIL, evidence, pitfalls):", + " [`audit/checks/`](../../audit/checks/README.md).", + "- Each check's requirement and framework section:", + " [Checks to framework map](how-the-checks-map-to-the-framework.md).", + "- How an audit runs: [Run an audit](../how-to/run-an-audit.md).", + ] + return "\n".join(out) + "\n" + + +IMPLEMENT_FLOW = """```mermaid +flowchart LR + R["Final audit report
(owner-confirmed)"] --> T["Triage: keep, skip
or defer each finding"] + T --> P["Plan the next theme"] + P --> A["Apply its playbooks
on a branch, one commit per item"] + A --> C["Re-check with the engine
and run the gate"] + C --> PR["Pull request
for the owner"] + PR -->|owner merges| P + PR -->|last theme| F["Full re-audit
and comparison"] +```""" + + +def render_implement(checks: Sequence[Check], playbooks: Sequence[Playbook]) -> str: + titles = {c.id: check_heading(c) for c in checks} + by_theme: dict[str, list[Playbook]] = {} + theme_names: dict[str, str] = {} + for playbook in playbooks: + by_theme.setdefault(playbook.theme, []).append(playbook) + theme_names.setdefault(playbook.theme, playbook.theme_name) + themes = sorted(by_theme, key=lambda t: int(t[1:])) + orphans = [c for c in checks if not c.playbooks] + out: list[str] = [ + "# Implement master list", + "", + "> **Audience:** CTOs, senior engineers and anyone who wants to know what an implement run changes.", + "> **Source of truth:** the playbooks in [`implement/playbooks/`](../../implement/playbooks/README.md) and", + "> [`catalogue.json`](../../audit/catalogue.json).", + ">", + *generated_note("the playbooks and the catalogue"), + "", + "---", + "", + *wrap( + f"An implement run closes the confirmed findings of an audit with **{len(playbooks)} playbooks** in " + f"{len(themes)} themes. Each theme is one branch and one pull request in the audited repository. The " + "agent never merges: the owner reviews and merges each pull request, and can skip or defer any finding. " + "Each playbook below says what it achieves, what it changes per check, and what it never does. The " + "checks themselves are in the [audit master list](audit-master.md)." + ), + "", + IMPLEMENT_FLOW, + "", + "## At a glance", + "", + "Themes run in dependency order: the order and the dependencies are in", + "[`implement/SKILL.md`](../../implement/SKILL.md#themes).", + "", + "| Theme | Playbooks | Checks closed |", + "|---|---|---|", + ] + for theme in themes: + names = ", ".join(f"[`{p.name}`](#{p.name})" for p in by_theme[theme]) + closed = sum(len(p.resolves) for p in by_theme[theme]) + out.append(f"| [{theme} {theme_names[theme]}](#{slug(f'{theme} {theme_names[theme]}')}) | {names} | {closed} |") + for theme in themes: + out += ["", f"## {theme} {theme_names[theme]}"] + for playbook in by_theme[theme]: + out += [ + "", + f"### `{playbook.name}`", + "", + playbook.goal, + "", + "| Check | What the playbook does |", + "|---|---|", + ] + for check_id, what in playbook.resolves: + title = titles.get(check_id, f"`{check_id}`") + out.append(f"| [{title}](audit-master.md#{slug(title)}) | {what} |") + if playbook.never: + out += ["", "**What it never does:**", "", playbook.never] + out += ["", f"Full playbook: [`{playbook.name}.md`](../../implement/playbooks/{playbook.name}.md)."] + if orphans: + out += ["", "## Checks with no playbook", ""] + for check in orphans: + out += wrap( + f"[{check_heading(check)}](audit-master.md#{slug(check_heading(check))}): the audit emits it, and " + "a playbook reads it as an input.", + "- ", + " ", + ) + out += [ + "", + "## Read more", + "", + "- The stages, the hard rules and the state file: [`implement/SKILL.md`](../../implement/SKILL.md).", + "- How to run it: [Implement an audit report](../how-to/implement-an-audit-report.md).", + ] + return "\n".join(out) + "\n" + + +def render(root: Path) -> dict[Path, str]: + """Both pages, keyed by their path relative to ``root``.""" + version, checks, names = read_checks(root) + playbooks = read_playbooks(root) + known = {c.id for c in checks} + for playbook in playbooks: + for check_id, _ in playbook.resolves: + if check_id not in known: + raise SourceError(f"{PLAYBOOKS_DIR}/{playbook.name}.md resolves {check_id}, which the catalogue lacks") + return {AUDIT_PAGE: render_audit(version, checks, names), IMPLEMENT_PAGE: render_implement(checks, playbooks)} + + +# --- Entry point -------------------------------------------------------------------------------------------- + + +def main(argv: Sequence[str] | None = None, root: Path = ROOT) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--check", action="store_true", help="exit 1 when a page is stale") + mode.add_argument("--write", action="store_true", help="rewrite both pages in place") + args = parser.parse_args(argv) + try: + pages = render(root) + except (SourceError, OSError, KeyError, ValueError) as exc: + print(f"build_masters: {exc}", file=sys.stderr) + return 2 + stale = [ + path + for path, text in pages.items() + if not (root / path).is_file() or (root / path).read_text(encoding="utf-8") != text + ] + if args.write: + for path in stale: + (root / path).write_text(pages[path], encoding="utf-8") + print(f"build_masters: wrote {path}") + return 0 + for path in stale: + print(f"build_masters: {path} is stale; run `make update-masters` and commit the result", file=sys.stderr) + return 1 if stale else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dev/tests/test_build_masters.py b/dev/tests/test_build_masters.py new file mode 100644 index 00000000..9d524d7d --- /dev/null +++ b/dev/tests/test_build_masters.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna + +"""Unit tests for ``build_masters.py``: link rebasing, field parsing, the stale check and malformed sources.""" + +from __future__ import annotations + +import contextlib +import io +import json +import shutil +import tempfile +import unittest +from pathlib import Path + +import _dev_paths # noqa: F401 (puts dev/ on sys.path) +import build_masters +from _dev_paths import REPO + + +class RebaseLinks(unittest.TestCase): + def test_relative_link_is_rewritten_from_the_output_folder(self) -> None: + text = "see [x](../../greenfield/README.md#61-agentsmd)" + got = build_masters.rebase_links(text, Path("audit/checks/02-agent-governance.md")) + self.assertEqual(got, "see [x](../../greenfield/README.md#61-agentsmd)") + + def test_sibling_link_gains_the_source_folder(self) -> None: + got = build_masters.rebase_links("[r](README.md)", Path("implement/playbooks/add-ci-parity.md")) + self.assertEqual(got, "[r](../../implement/playbooks/README.md)") + + def test_same_page_anchor_points_at_the_source_page(self) -> None: + got = build_masters.rebase_links("[c](#continue-on-error)", Path("implement/playbooks/add-ci-parity.md")) + self.assertEqual(got, "[c](../../implement/playbooks/add-ci-parity.md#continue-on-error)") + + def test_external_link_and_image_are_left_alone(self) -> None: + text = "[a](https://example.com/x) ![i](pic.png)" + self.assertEqual(build_masters.rebase_links(text, Path("audit/checks/01-x.md")), text) + + +class FieldText(unittest.TestCase): + def test_continuation_lines_join_and_the_next_field_stops_it(self) -> None: + body = "\nWhy it matters first line\n second line\nSeverity BLOCKER\n" + self.assertEqual(build_masters.field_text(body, "Why it matters"), "first line second line") + self.assertEqual(build_masters.field_text(body, "Severity"), "BLOCKER") + + def test_missing_field_is_empty(self) -> None: + self.assertEqual(build_masters.field_text("Severity BLOCKER\n", "Rubric"), "") + + +class RepositoryPages(unittest.TestCase): + def test_committed_pages_match_the_sources(self) -> None: + for path, text in build_masters.render(REPO).items(): + with self.subTest(page=str(path)): + self.assertEqual((REPO / path).read_text(encoding="utf-8"), text, "run `make update-masters`") + + def test_every_catalogue_check_and_playbook_appears(self) -> None: + pages = build_masters.render(REPO) + catalogue = json.loads((REPO / build_masters.CATALOGUE).read_text(encoding="utf-8")) + audit = pages[build_masters.AUDIT_PAGE] + implement = pages[build_masters.IMPLEMENT_PAGE] + for check in catalogue["checks"]: + self.assertIn(f"#### `{check['id']}` ", audit) + for playbook in check["playbooks"]: + self.assertIn(f"### `{playbook}`", implement) + + +class Sandbox(unittest.TestCase): + """A copy of the sources in a temporary root, so a test can break one.""" + + def setUp(self) -> None: + self.root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.root) + for folder in (build_masters.CHECKS_DIR, build_masters.PLAYBOOKS_DIR): + shutil.copytree(REPO / folder, self.root / folder) + (self.root / build_masters.OUT_DIR).mkdir(parents=True) + shutil.copy(REPO / build_masters.CATALOGUE, self.root / build_masters.CATALOGUE) + + def run_main(self, *argv: str) -> tuple[int, str]: + err = io.StringIO() + with contextlib.redirect_stderr(err), contextlib.redirect_stdout(io.StringIO()): + code = build_masters.main(list(argv), root=self.root) + return code, err.getvalue() + + def test_missing_pages_are_stale_until_written(self) -> None: + self.assertEqual(self.run_main("--check")[0], 1) + self.assertEqual(self.run_main("--write")[0], 0) + self.assertEqual(self.run_main("--check")[0], 0) + + def test_a_source_change_makes_the_page_stale(self) -> None: + self.run_main("--write") + path = self.root / build_masters.CHECKS_DIR / "06-ci.md" + path.write_text(path.read_text(encoding="utf-8").replace("Verification must", "Checking must", 1)) + code, err = self.run_main("--check") + self.assertEqual(code, 1) + self.assertIn("audit-master.md is stale", err) + + def test_a_family_with_no_area_is_refused(self) -> None: + path = self.root / build_masters.CATALOGUE + catalogue = json.loads(path.read_text(encoding="utf-8")) + catalogue["checks"][0]["family"] = "NEW" + path.write_text(json.dumps(catalogue), encoding="utf-8") + code, err = self.run_main("--check") + self.assertEqual(code, 2) + self.assertIn("NEW has no area", err) + + def test_a_check_with_no_section_is_refused(self) -> None: + path = self.root / build_masters.CATALOGUE + catalogue = json.loads(path.read_text(encoding="utf-8")) + catalogue["checks"][0]["id"] = "IDX-99" + path.write_text(json.dumps(catalogue), encoding="utf-8") + code, err = self.run_main("--check") + self.assertEqual(code, 2) + self.assertIn("IDX-99 has no section", err) + + def test_a_playbook_missing_from_the_list_is_refused(self) -> None: + (self.root / build_masters.PLAYBOOKS_DIR / "new-playbook.md").write_text("# Playbook\n", encoding="utf-8") + code, err = self.run_main("--check") + self.assertEqual(code, 2) + self.assertIn("new-playbook", err) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/README.md b/docs/README.md index 59906a35..d42a0d17 100644 --- a/docs/README.md +++ b/docs/README.md @@ -32,6 +32,8 @@ output to ``". To fix the findings, say "Start implementing the fix | Page | What it lists | | --- | --- | +| [Audit master list](reference/audit-master.md) | Every audit check by area: what passes, why it matters, which playbook fixes it | +| [Implement master list](reference/implement-master.md) | Every playbook by theme: its goal, what it changes per check, what it never does | | [Tools CLI](reference/tools-cli.md) | Each tool's synopsis, inputs, outputs and exit codes | | [Harness notes](reference/harness-notes.md) | What differs between Claude Code, opencode, Codex and Cursor | | [Checks to framework map](reference/how-the-checks-map-to-the-framework.md) | Each check, its requirement and its framework section | diff --git a/docs/reference/audit-master.md b/docs/reference/audit-master.md new file mode 100644 index 00000000..d55915f2 --- /dev/null +++ b/docs/reference/audit-master.md @@ -0,0 +1,787 @@ +# Audit master list + +> **Audience:** CTOs, senior engineers and anyone who wants to know what an audit looks at. +> **Source of truth:** [`catalogue.json`](../../audit/catalogue.json) and the check files in +> [`audit/checks/`](../../audit/checks/README.md). +> +> **Generated** by [`dev/build_masters.py`](../../dev/build_masters.py) from the catalogue and the check files. Do not +> edit it by hand: change a source, then run `make update-masters`. `make check` fails when this page is stale. + +--- + +The audit runs **72 checks** in 13 families (catalogue 0.3.0). 58 are scored; the rest are reported for the whole +picture and earn no credit. The engine decides 70 checks from the files alone; the agent judges 2. Each check below says +what passes and why it matters. To fix a finding, follow the playbook it names in the [implement master +list](implement-master.md). + +| Severity | Meaning | +|---|---| +| `BLOCKER` | An agent cannot work safely until this is fixed | +| `DEGRADER` | An agent can work, but slower, with more mistakes or with less trust in its result | +| `COSMETIC` | Worth fixing; it does not change what an agent can do | + +## At a glance + +| Area | The question it answers | Families | Checks | +|---|---|---|---| +| [Agent context](#agent-context) | Can an agent find the code, the rules and the documents it needs before it edits anything? | [`IDX`](#discovery-and-index-idx), [`AGT`](#agent-governance-agt), [`DOC`](#documentation-contract-doc), [`NAV`](#navigation-nav) | 23 (3 BLOCKER, 16 DEGRADER, 4 COSMETIC) | +| [Command surface and quality gates](#command-surface-and-quality-gates) | Can an agent run one command that formats, lints, type-checks and tests, and trust its answer? | [`CMD`](#command-surface-cmd), [`TOOL`](#tooling-tool), [`TST`](#tests-tst), [`BASE`](#baselines-and-the-ratchet-base) | 19 (3 BLOCKER, 14 DEGRADER, 2 COSMETIC) | +| [Enforcement in CI](#enforcement-in-ci) | Does the pipeline run the same gate the agent runs locally, and does it block the merge? | [`CI`](#continuous-integration-ci) | 5 (2 BLOCKER, 3 DEGRADER) | +| [Reproducible execution](#reproducible-execution) | Does the same command give the same result on a fresh machine, without a human in the loop? | [`EXEC`](#execution-determinism-exec) | 7 (4 BLOCKER, 3 DEGRADER) | +| [Safety boundaries and secrets](#safety-boundaries-and-secrets) | Is it written down, and enforced, what an agent must never touch or reveal? | [`CON`](#constraints-con), [`SEC`](#security-hygiene-sec) | 7 (4 BLOCKER, 2 DEGRADER, 1 COSMETIC) | +| [Team hygiene (informational)](#team-hygiene-informational) | Is the repository run like a team project? Reported for the whole picture; never scored. | [`HYG`](#hygiene-hyg) | 11 (11 COSMETIC) | + +## Agent context + +Can an agent find the code, the rules and the documents it needs before it edits anything? + +### Discovery and Index (IDX) + +3 checks: 2 BLOCKER, 1 DEGRADER. + +#### `IDX-01` codebase-memory-mcp registered for this repository + +`BLOCKER` · scored · engine check · fixed by [`register-index`](implement-master.md#register-index) + +- **Passes when:** a readable project- or user-level config names `codebase-memory-mcp`. +- **Why it matters:** Every later check and playbook assumes an agent can reach the index; without registration an agent + has no way to discover that one should exist ([`docs/framework/phased-approach.md` § Phase 1: Agentic + Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +#### `IDX-02` An index exists for this checkout + +`BLOCKER` · scored · engine check · fixed by [`register-index`](implement-master.md#register-index) + +- **Passes when:** the index answers a structural query for this checkout, or the engine finds a store row whose root + resolves to the target. +- **Why it matters:** An index of a different checkout is worse than none: it answers confidently with structure that no + longer exists ([`docs/framework/phased-approach.md` § Activities](../framework/phased-approach.md#activities)). + +#### `IDX-03` Index freshness + +`DEGRADER` · scored · engine check · fixed by [`register-index`](implement-master.md#register-index) + +- **Passes when:** the index's recorded revision matches `HEAD`, or a direct freshness query reports no drift. +- **Why it matters:** A stale graph is stale context: an agent edits based on symbols that have already moved or been + deleted ([`docs/framework/phased-approach.md` § Activities](../framework/phased-approach.md#activities)). + +### Agent Governance (AGT) + +11 checks: 1 BLOCKER, 10 DEGRADER. + +#### `AGT-01` Instruction file present, canonical, and non-trivial + +`BLOCKER` · scored · engine check · fixed by [`adapt-agents-md`](implement-master.md#adapt-agents-md) + +- **Passes when:** `AGENTS.md` exists at the repository root in canonical case, is not shadowed by a harness override in + the same directory, is at least 400 characters, and contains no stub marker. +- **Why it matters:** `AGENTS.md` is the entry point every harness is guaranteed to read; without it an agent infers + conventions from whatever code it happens to open first ([greenfield/README.md § 6.1 + AGENTS.md](../../greenfield/README.md#61-agentsmd)). + +#### `AGT-02` Component coverage by instruction files + +`DEGRADER` · scored · engine check · fixed by [`adapt-agents-md`](implement-master.md#adapt-agents-md) + +- **Passes when:** every declared component has its own `AGENTS.md` (ratio 1.0), or there are no declared components + beyond the root. +- **Why it matters:** Monorepos have different rules per component; a single root file forces one component's + conventions onto another ([Phased-Approach.md § Activities](../framework/phased-approach.md#activities)). + +#### `AGT-03` Documentation contract named + +`DEGRADER` · scored · engine check · fixed by [`adapt-agents-md`](implement-master.md#adapt-agents-md) + +- **Passes when:** the file names `docs/` as the documentation home, `docs/STYLE_GUIDE.md` as the authoring rules, and + the scratch-workspace directory as off-limits scratch, each on its own line. +- **Why it matters:** Agents must know where permanent truth lives and what is off-limits, or they treat scratch notes + as architecture ([greenfield/README.md § 5.2 Templates & Guides](../../greenfield/README.md#52-templates--guides)). + +#### `AGT-04` Definition of done stated + +`DEGRADER` · scored · engine check · fixed by [`adapt-agents-md`](implement-master.md#adapt-agents-md) + +- **Passes when:** a sentence ties a stated documentation trigger (public interface, observable behaviour, architecture, + an ADR, or documentation itself) to updating docs in the same change set. +- **Why it matters:** Docs must change in the same change set as the code they describe, or specs drift within days + ([greenfield/README.md § 5.2 Templates & Guides](../../greenfield/README.md#52-templates--guides)). + +#### `AGT-05` Seven verbs named + +`DEGRADER` · scored · engine check · fixed by [`adapt-agents-md`](implement-master.md#adapt-agents-md) + +- **Passes when:** all seven verbs (`format`, `format:check`, `lint`, `typecheck`, `test`, `check`, `security`) are + named in the instruction file and each resolves on a runner. +- **Why it matters:** The agent needs the sanctioned verification path stated, or it invents one (`npx vitest` where the + project uses something else), diverging from what CI actually runs ([greenfield/README.md § 1.4 The standardized + command surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs)). + +#### `AGT-06` Architectural boundaries declared + +`DEGRADER` · scored · engine check · fixed by [`adapt-agents-md`](implement-master.md#adapt-agents-md) + +- **Passes when:** at least two boundary rules each name a path that exists in the repository. +- **Why it matters:** Boundary rules must be stated, not inferred, or edits cross module lines nothing ever wrote down + ([greenfield/README.md § 6.1 AGENTS.md](../../greenfield/README.md#61-agentsmd)). + +#### `AGT-07` Do-not-do / deny rules present + +`DEGRADER` · scored · engine check · fixed by [`add-deny-rules`](implement-master.md#add-deny-rules) + +- **Passes when:** a prohibitions section exists and covers all four concerns (tests, generated files, the default + branch, secrets) with an explicit rule for each. +- **Why it matters:** Explicit prohibitions work; implicit ones do not — an agent that is never told "never weaken a + test" will weaken one ([Phased-Approach.md § Activities](../framework/phased-approach.md#activities)). + +#### `AGT-08` No competing instruction files, and the reach matrix is stated + +`DEGRADER` · scored · engine check · fixed by [`adapt-agents-md`](implement-master.md#adapt-agents-md) + +- **Passes when:** no vendor-only competitor exists, no shadowed pair exists, and either every known harness is reached + or the file states which harnesses it does not reach. +- **Why it matters:** `AGENTS.md`, `CLAUDE.md` and `.cursorrules` disagreeing is obeyed as ambiguity, not flagged as a + defect — inconsistent behaviour across harnesses, with no way to trace why ([Phased-Approach.md § Phase 1: Agentic + Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +#### `AGT-09` Branch and release boundary declared + +`DEGRADER` · scored · engine check · fixed by [`adapt-agents-md`](implement-master.md#adapt-agents-md) + +- **Passes when:** the file states which branches exist and whether an agent may tag, release, publish, deploy or push. +- **Why it matters:** Agents must know whether they may tag, release or push, or one triggers a release it had no + authority to cut ([Phased-Approach.md § Phase 1: Agentic + Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +#### `AGT-10` Instruction indirection resolved + +`DEGRADER` · scored · engine check · fixed by [`adapt-agents-md`](implement-master.md#adapt-agents-md) + +- **Passes when:** no configured pointer names a missing file, and no deferral phrase in the entry file names a file + left unconfigured. +- **Why it matters:** A rule the agent is told to "read elsewhere first" is only as reliable as the harness's own + willingness to follow that pointer — an unconfigured deferral is a rule that may never load ([Phased-Approach.md § + Phase 1: Agentic Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +#### `AGT-11` Instructions are specific and not inferable + +`DEGRADER` · scored · agent judgement · fixed by [`adapt-agents-md`](implement-master.md#adapt-agents-md) + +- **Passes when:** at least 80% of rule lines are specific (by the count above) and every specific line checked resolves + against the repository; no overview section exceeds roughly a fifth of the file (A14: the overview is optional and + must not dominate). +- **Why it matters:** This is where A14 (above) is judged directly: a file full of generic, inferable advice costs + inference time for no benefit, which is R-16's own finding ([Phased-Approach.md § + Activities](../framework/phased-approach.md#activities)). + +### Documentation Contract (DOC) + +4 checks: 3 DEGRADER, 1 COSMETIC. + +#### `DOC-01` docs/ exists + +`DEGRADER` · scored · engine check · fixed by [`establish-docs-contract`](implement-master.md#establish-docs-contract) + +- **Passes when:** at least one Markdown file exists anywhere under `docs/`. +- **Why it matters:** `docs/` is the declared home for permanent documentation; without it on day one, documentation has + nowhere canonical to start ([greenfield/README.md § 5.1 Diátaxis + Framework](../../greenfield/README.md#51-diátaxis-framework)). + +#### `DOC-02` docs/STYLE_GUIDE.md exists + +`DEGRADER` · scored · engine check · fixed by [`establish-docs-contract`](implement-master.md#establish-docs-contract) + +- **Passes when:** `docs/STYLE_GUIDE.md` exists and the instruction file names it. +- **Why it matters:** Agents need authoring rules they can cite instead of restating per page, or rules get restated and + drift ([greenfield/README.md § 5.2 Templates & Guides](../../greenfield/README.md#52-templates--guides)). + +#### `DOC-03` artefacts/ exists and is ignored + +`DEGRADER` · scored · engine check · fixed by [`establish-docs-contract`](implement-master.md#establish-docs-contract) + +- **Passes when:** no scratch directory exists anywhere, or one exists, is matched by an ignore rule, and nothing under + it is tracked. +- **Why it matters:** Transient scratch must be separable and off-limits, or an agent reads work-in-progress notes as + the current, settled requirement ([greenfield/README.md § 6.2 Agent + Scratchpad](../../greenfield/README.md#62-agent-scratchpad--artefacts)). + +#### `DOC-04` ADR directory and index + +`COSMETIC` · scored · engine check · fixed by [`establish-docs-contract`](implement-master.md#establish-docs-contract) + +- **Passes when:** a numbered-ADR directory exists, holds at least one entry, and an index lists every entry. +- **Why it matters:** Architectural decisions need a location and a registry, or the rationale lives only in chat and + pull requests, and agents re-litigate settled decisions ([greenfield/README.md § 5.2 Templates & + Guides](../../greenfield/README.md#52-templates--guides)). + +### Navigation (NAV) + +5 checks: 2 DEGRADER, 3 COSMETIC. + +#### `NAV-01` Declared component boundaries + +`DEGRADER` · scored · engine check · fixed by [`establish-docs-contract`](implement-master.md#establish-docs-contract) + +- **Passes when:** at least one non-root component is declared by a build manifest, or the repository is single-root and + the root itself declares a manifest. +- **Why it matters:** Constraint Engineering needs a bounded component to scope work to; without one, blast radius + cannot be scoped or reviewed ([docs/framework/overview.md § Applicability](../framework/overview.md#applicability)). + +#### `NAV-02` Obvious entry points + +`DEGRADER` · scored · engine check · fixed by [`establish-docs-contract`](implement-master.md#establish-docs-contract) + +- **Passes when:** at least one entry-point signal is found for the detected ecosystem(s). +- **Why it matters:** Agents must find where execution begins before they can reason about what is live versus dead code + ([greenfield/README.md § 1.9 Core DI interface file + public entry + point](../../greenfield/README.md#19-core-di-interface-file--public-entry-point)). + +#### `NAV-03` Structural red flags + +`COSMETIC` · scored · engine check · fixed by [`establish-docs-contract`](implement-master.md#establish-docs-contract) + +- **Passes when:** no oversized file, no catch-all directory, no checked-in binary, and no minified asset is found. +- **Why it matters:** Oversized files, catch-all directories, checked-in binaries and minified assets are exactly where + agents are drawn when they explore, and exactly where they should not be working ([Phased-Approach.md § Phase 1: + Agentic Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +#### `NAV-04` Architecture orientation artifact + +`COSMETIC` · not scored · engine check · fixed by +[`establish-docs-contract`](implement-master.md#establish-docs-contract) + +- **Passes when:** a `docs/architecture/` file, an ADR directory, or a diagram file exists (any one). +- **Why it matters:** A short map lets an agent orient before it explores instead of discovering structure by trial and + error ([Phased-Approach.md § Key Deliverables](../framework/phased-approach.md#key-deliverables)). + +#### `NAV-05` Public-symbol documentation coverage + +`COSMETIC` · not scored · engine check · fixed by +[`establish-docs-contract`](implement-master.md#establish-docs-contract) + +- **Passes when:** every public symbol found (in the one censused language) is documented. +- **Why it matters:** Phase 2 territory, reported here only as a signal: without any documentation coverage signal an + agent must read whole function bodies just to learn their signatures ([Phased-Approach.md § Key + Deliverables](../framework/phased-approach.md#key-deliverables)). + +## Command surface and quality gates + +Can an agent run one command that formats, lints, type-checks and tests, and trust its answer? + +### Command Surface (CMD) + +4 checks: 2 BLOCKER, 2 DEGRADER. + +#### `CMD-01` Seven verbs resolvable on one runner + +`BLOCKER` · scored · engine check · fixed by [`define-command-surface`](implement-master.md#define-command-surface) + +- **Passes when:** all seven verbs (`format`, `format:check`, `lint`, `typecheck`, `test`, `check`, `security`) resolve, + each on the same one runner. +- **Why it matters:** The verbs are the contract; without them resolving an agent has no shared vocabulary for "how do I + verify this" and invents its own command ([greenfield/README.md § 1.4 The standardized command + surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs)). + +#### `CMD-02` check composes the read-only verbs + +`BLOCKER` · scored · engine check · fixed by [`define-command-surface`](implement-master.md#define-command-surface) + +- **Passes when:** `check`'s body invokes, directly or by a resolved verb, all three of a format-check, a typecheck and + a test run. +- **Why it matters:** A single pre-merge gate must exist and be the same one everywhere, or different people gate on + different subsets of the verbs ([greenfield/README.md § 1.4 The standardized command + surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs)). + +#### `CMD-03` No divergent second definition + +`DEGRADER` · scored · engine check · fixed by [`define-command-surface`](implement-master.md#define-command-surface) + +- **Passes when:** at most one runner defines `check`, or every runner but the primary delegates to it. +- **Why it matters:** A runner other than the authoritative one must delegate, not compose its own gate, or nobody can + tell which definition is the real one when they disagree ([greenfield/README.md § 1.4 The standardized command + surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs)). + +#### `CMD-04` Write verbs separated from read-only verbs + +`DEGRADER` · scored · engine check · fixed by [`define-command-surface`](implement-master.md#define-command-surface) + +- **Passes when:** no read-only verb's command invokes a write verb, directly or by indirection. +- **Why it matters:** A gate that mutates the working tree while "verifying" it cannot be trusted to report on the code + that was actually committed ([greenfield/README.md § 1.4 The standardized command + surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs)). + +### Tooling (TOOL) + +6 checks: 5 DEGRADER, 1 COSMETIC. + +#### `TOOL-01` Formatter configured + +`DEGRADER` · scored · engine check · fixed by [`configure-quality-tools`](implement-master.md#configure-quality-tools) + +- **Passes when:** a formatter config file is present and a `format` verb resolves on some runner. +- **Why it matters:** Formatting must be free and non-negotiable, or every diff mixes style with substance and review + spends attention on noise ([`docs/framework/phased-approach.md` § + Activities](../framework/phased-approach.md#activities)). + +#### `TOOL-02` Linter configured + +`DEGRADER` · scored · engine check · fixed by [`configure-quality-tools`](implement-master.md#configure-quality-tools) + +- **Passes when:** a linter config file is present and a `lint` verb resolves on some runner. +- **Why it matters:** A linter catches classes of defect before tests exist to catch them, including regressions an + agent introduces that no test yet covers ([`docs/framework/phased-approach.md` § + Activities](../framework/phased-approach.md#activities)). + +#### `TOOL-03` Type checker configured + +`DEGRADER` · scored · engine check · fixed by [`configure-quality-tools`](implement-master.md#configure-quality-tools) + +- **Passes when:** at least one strict-mode signal is found (`tsconfig.json: strict`, mypy/pyright strict, + `enable`, or a `typecheck` verb with `--strict`). +- **Why it matters:** A strict type checker is the cheapest correctness gate available; without it, interface drift + between an agent's change and its callers is discovered at runtime instead of at review time + ([`docs/framework/phased-approach.md` § Activities](../framework/phased-approach.md#activities)). + +#### `TOOL-04` Dependency / security scan verb + +`DEGRADER` · scored · engine check · fixed by [`configure-quality-tools`](implement-master.md#configure-quality-tools) + +- **Passes when:** at least one resolved verb's command matches a known dependency-audit invocation. +- **Why it matters:** Third-party risk is real and agents add dependencies; without a scan, a vulnerability can enter + through the very change an agent just made ([`docs/framework/phased-approach.md` § + Activities](../framework/phased-approach.md#activities)). + +#### `TOOL-05` Local hook chain active + +`DEGRADER` · scored · engine check · fixed by +[`add-hooks-and-commit-rules`](implement-master.md#add-hooks-and-commit-rules) + +- **Passes when:** a hook chain is wired (`.pre-commit-config.yaml`, `lefthook.yml`/`.yaml`, `core.hooksPath`, or a + referenced `.husky/`). +- **Why it matters:** Enforcement must exist where an agent actually works, not only downstream in CI; a rule that lives + only in CI is a rule an agent cannot honour before it commits ([greenfield/README.md § 2.3 Git + Hooks](../../greenfield/README.md#23-git-hooks--husky--lint-staged--commitlint)). + +#### `TOOL-06` Commit convention enforced + +`COSMETIC` · scored · engine check · fixed by +[`add-hooks-and-commit-rules`](implement-master.md#add-hooks-and-commit-rules) + +- **Passes when:** a commitlint config, a `commit-msg` hook file, or an equivalent pre-commit/lefthook stage is found. +- **Why it matters:** Release automation reads commit messages; without an enforced convention, version bumps and + changelog generation silently stop working ([greenfield/README.md § 2.3 Git + Hooks](../../greenfield/README.md#23-git-hooks--husky--lint-staged--commitlint)). + +### Tests (TST) + +5 checks: 1 BLOCKER, 3 DEGRADER, 1 COSMETIC. + +#### `TST-01` Test verb exists, suite non-empty + +`BLOCKER` · scored · engine check · fixed by [`define-command-surface`](implement-master.md#define-command-surface) + +- **Passes when:** a `test` verb resolves and at least one test-source file exists. +- **Why it matters:** An agent cannot verify a change it cannot run; a resolvable `test` verb and a real suite are the + floor every other test check stands on ([greenfield/README.md § 3.2 Testing + Policies](../../greenfield/README.md#32-testing-policies-enforced-via-agentsmd)). + +#### `TST-02` Suite status known + +`DEGRADER` · scored · engine check · fixed by +[`add-baselines-and-ratchet`](implement-master.md#add-baselines-and-ratchet) + +- **Passes when:** the test verb was run as a Tier-C probe and exited 0. +- **Why it matters:** Whether the suite is green *today* determines what an agent can safely attempt; without this an + agent inherits a red baseline and cannot tell its own regressions from yours ([greenfield/README.md § 3.2 Testing + Policies](../../greenfield/README.md#32-testing-policies-enforced-via-agentsmd)). + +#### `TST-03` Skip / xfail census + +`DEGRADER` · scored · engine check · fixed by +[`add-baselines-and-ratchet`](implement-master.md#add-baselines-and-ratchet) + +- **Passes when:** the census total is zero. +- **Why it matters:** A suite weakened with skip and expected-failure markers to reach green is the fingerprint of a + gate that no longer catches the regressions it was meant to ([greenfield/README.md § 3.2 Testing + Policies](../../greenfield/README.md#32-testing-policies-enforced-via-agentsmd)). + +#### `TST-04` Fast path documented + +`DEGRADER` · scored · engine check · fixed by [`adapt-agents-md`](implement-master.md#adapt-agents-md), +[`add-baselines-and-ratchet`](implement-master.md#add-baselines-and-ratchet) + +- **Passes when:** at least one entry document shows a single-test/subset invocation. +- **Why it matters:** Agent iteration speed is the workflow; without a documented single-test invocation an agent waits + out the whole suite per attempt, or stops running tests altogether ([Phased-Approach.md § Phase 1: Agentic + Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +#### `TST-05` Coverage configuration present + +`COSMETIC` · scored · engine check · fixed by +[`add-baselines-and-ratchet`](implement-master.md#add-baselines-and-ratchet) + +- **Passes when:** any one of the routes above finds coverage configured. +- **Why it matters:** A ratchet (`BASE-03`) needs a number to hold the line on, and coverage is the number for the test + surface; with none configured there is no floor to hold ([greenfield/README.md § 3.1 + vitest.config.ts](../../greenfield/README.md#31-vitestconfigts--with-coverage)). + +### Baselines and the Ratchet (BASE) + +4 checks: 4 DEGRADER. + +#### `BASE-01` Baseline artifact exists + +`DEGRADER` · scored · engine check · fixed by +[`add-baselines-and-ratchet`](implement-master.md#add-baselines-and-ratchet) + +- **Passes when:** a baseline/suppression artifact is committed, or an equivalent mechanism is documented in `AGENTS.md` + or `CLAUDE.md`. +- **Why it matters:** Gates cannot be turned on over existing debt without mass failure; instrumentation that separates + old debt from new debt must land first ([Phased-Approach.md § + Activities](../framework/phased-approach.md#activities)). + +#### `BASE-02` Current violation counts measured + +`DEGRADER` · scored · engine check · fixed by +[`add-baselines-and-ratchet`](implement-master.md#add-baselines-and-ratchet) + +- **Passes when:** an explicit numeric violation budget is committed somewhere the engine reads (`--max-warnings 12`, + `max-issues: 5`, `violation_budget = 40`). +- **Why it matters:** The baseline must be a number, not an impression, or there is no way to tell whether debt is + growing or shrinking ([Phased-Approach.md § Activities](../framework/phased-approach.md#activities)). + +#### `BASE-03` No-new-violations mechanism + +`DEGRADER` · scored · engine check · fixed by +[`add-baselines-and-ratchet`](implement-master.md#add-baselines-and-ratchet) + +- **Passes when:** a numeric violation budget is committed, or a committed baseline artifact is referenced by name + inside a CI workflow, or a comparison step is documented in one of the four entry documents. +- **Why it matters:** The policy must be executable, not aspirational, or every change quietly adds a little more debt + on top of what was already frozen ([Phased-Approach.md § Activities](../framework/phased-approach.md#activities)). + +#### `BASE-04` Coverage floor configured + +`DEGRADER` · scored · engine check · fixed by +[`add-baselines-and-ratchet`](implement-master.md#add-baselines-and-ratchet) + +- **Passes when:** a coverage threshold is configured in test-runner configuration. +- **Why it matters:** Coverage is the most common form of the ratchet, and without a configured floor it decays + invisibly, one untested branch at a time ([greenfield/README.md § 4.1 + Workflows](../../greenfield/README.md#41-workflows)). + +## Enforcement in CI + +Does the pipeline run the same gate the agent runs locally, and does it block the merge? + +### Continuous Integration (CI) + +5 checks: 2 BLOCKER, 3 DEGRADER. + +#### `CI-01` Pipeline present, triggered on PRs + +`BLOCKER` · scored · engine check · fixed by [`add-ci-parity`](implement-master.md#add-ci-parity) + +- **Passes when:** at least one workflow file exists and declares a pull-request trigger the engine's pattern + recognises. +- **Why it matters:** Verification must be automatic and independent of the author, or "I ran the tests" before pushing + is unverifiable by anyone else ([greenfield/README.md § 4.1 Workflows](../../greenfield/README.md#41-workflows)). + +#### `CI-02` Local↔CI parity + +`BLOCKER` · scored · engine check · fixed by [`add-ci-parity`](implement-master.md#add-ci-parity) + +- **Passes when:** every framework verb a CI command invokes also resolves locally. +- **Why it matters:** The gate an agent *can* run locally must be the gate that *decides*, or an agent optimises against + a weaker local gate and discovers the real gap only at review ([greenfield/README.md § 1.4 The standardized command + surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs)). + +#### `CI-03` CI-only steps flagged + +`DEGRADER` · scored · engine check · fixed by [`add-ci-parity`](implement-master.md#add-ci-parity) + +- **Passes when:** no CI command names a runner target absent from every local runner manifest. +- **Why it matters:** A step with no local equivalent is a failure an agent cannot reproduce, so it cannot investigate + or fix what CI is failing it on ([greenfield/README.md § 1.4 The standardized command + surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs)). + +#### `CI-04` CI enforces the same gate as the local check + +`DEGRADER` · scored · agent judgement · fixed by [`add-ci-parity`](implement-master.md#add-ci-parity) + +- **Passes when:** every job running a verb (or `check`) gates on it — no `continue-on-error`, no swallowed exit code on + the gating command — and the workflow is a required status check for merging (confirmed, not just assumed from the + workflow file alone). +- **Why it matters:** A pipeline that runs the verbs but does not fail the build when one of them fails is theatre, not + enforcement ([greenfield/README.md § 1.4 The standardized command + surface](../../greenfield/README.md#14-the-standardized-command-surface--seven-verbs)). + +#### `CI-05` Workflow files are linted + +`DEGRADER` · scored · engine check · fixed by [`add-ci-parity`](implement-master.md#add-ci-parity) + +- **Passes when:** every provider with workflow files has a linter that runs locally. +- **Why it matters:** An agent cannot run a workflow on its machine. A linter that knows the CI provider is its only + check before a push: without one, a broken expression, a wrong action input or a script-injection risk is found only + after the push ([greenfield/README.md § 4.1 Workflows](../../greenfield/README.md#41-workflows)). + +## Reproducible execution + +Does the same command give the same result on a fresh machine, without a human in the loop? + +### Execution determinism (EXEC) + +7 checks: 4 BLOCKER, 3 DEGRADER. + +#### `EXEC-01` Toolchain pinned / wrapper committed + +`BLOCKER` · scored · engine check · fixed by +[`pin-execution-environment`](implement-master.md#pin-execution-environment) + +- **Passes when:** every detected pinnable ecosystem has at least one of its own pin files, or a matching + `.tool-versions` line, or (node/python/go) the manifest-level constraint. +- **Why it matters:** An agent's runtime must be the repository's, not whatever happens to be on the host — otherwise + the gate passes locally and fails in CI, or an agent reproduces nothing ([greenfield/README.md § File Tree — What Gets + Created](../../greenfield/README.md#file-tree--what-gets-created)). + +#### `EXEC-02` Lockfile committed and consistent + +`BLOCKER` · scored · engine check · fixed by +[`pin-execution-environment`](implement-master.md#pin-execution-environment) + +- **Passes when:** every detected ecosystem's lockfile exists, is tracked, and (where parsing is attempted) agrees with + the manifest. +- **Why it matters:** Every run must resolve identical dependencies; a lockfile that is missing, untracked, or stale + turns "works on my machine" into a problem the agent inherits too ([Phased-Approach.md § Phase 1: Agentic + Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +#### `EXEC-03` Non-interactive setup path + +`BLOCKER` · scored · engine check · fixed by [`define-command-surface`](implement-master.md#define-command-surface) + +- **Passes when:** a documented non-interactive install command exists, or a setup artifact exists and is named by one + of the four doc files. +- **Why it matters:** An agent cannot answer an interactive prompt or provision a database by guessing; with no + non-interactive path it is blocked before it can run anything ([Phased-Approach.md § Phase 1: Agentic + Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +#### `EXEC-04` Env-var inventory complete + +`BLOCKER` · scored · engine check · fixed by +[`env-inventory-and-secrets`](implement-master.md#env-inventory-and-secrets) + +- **Passes when:** every referenced key not on the allow-list is declared in a committed example file (including the + case of no references at all). +- **Why it matters:** A variable the code needs but nobody wrote down fails at runtime, mid-task, and the agent debugs + an environment problem it has no way to see coming ([greenfield/README.md § File Tree — What Gets + Created](../../greenfield/README.md#file-tree--what-gets-created)). + +#### `EXEC-05` No generated artifacts committed in-tree + +`DEGRADER` · scored · engine check · fixed by [`untrack-generated`](implement-master.md#untrack-generated) + +- **Passes when:** no tracked path is classified `GENERATED`, or every tracked `GENERATED` path carries a generated-file + header in its first 20 lines. +- **Why it matters:** An agent that edits a file the build regenerates loses its work silently, often not until the next + CI run or review ([Phased-Approach.md § Phase 1: Agentic + Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +#### `EXEC-06` Documented commands resolve + +`DEGRADER` · scored · engine check · fixed by [`adapt-agents-md`](implement-master.md#adapt-agents-md) + +- **Passes when:** every documented command resolves, or no fenced shell command is documented at all. +- **Why it matters:** An agent runs the documented command literally; a command that does not resolve is the most + expensive brownfield trap there is, discovered only after the agent has already acted on it ([Phased-Approach.md § + Phase 1: Agentic Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +#### `EXEC-07` No competing configs for one concern + +`DEGRADER` · scored · engine check · fixed by [`untrack-generated`](implement-master.md#untrack-generated) + +- **Passes when:** each of the five concerns has at most one matching root config (including zero). +- **Why it matters:** One concern, one authoritative answer; with two configs for the same concern an agent (or a human) + picks whichever result turns up first, which is not the same thing as picking correctly ([Phased-Approach.md § Phase + 1: Agentic Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +## Safety boundaries and secrets + +Is it written down, and enforced, what an agent must never touch or reveal? + +### Constraints (CON) + +4 checks: 2 BLOCKER, 1 DEGRADER, 1 COSMETIC. + +#### `CON-01` Allow/deny artifact present + +`BLOCKER` · scored · engine check · fixed by [`add-deny-rules`](implement-master.md#add-deny-rules) + +- **Passes when:** the prohibitions section names, in a code span, at least one path or pattern that matches a file or + directory of the repository. +- **Why it matters:** An agent cannot respect a boundary that exists only in someone's head; the deny list must be a + concrete, written artifact, not an unstated expectation ([Phased-Approach.md § + Activities](../framework/phased-approach.md#activities)). + +#### `CON-02` Test directories protected + +`BLOCKER` · scored · engine check · fixed by [`add-deny-rules`](implement-master.md#add-deny-rules) + +- **Passes when:** the prohibitions section names, in code spans, every test directory found. +- **Why it matters:** Tests are the safety net; an agent that may freely edit them is not actually gated by them, and + can make a failure disappear instead of fixing it ([docs/framework/overview.md § + Principles](../framework/overview.md#principles)). + +#### `CON-03` artefacts/ excluded from index and agent reads + +`DEGRADER` · scored · engine check · fixed by [`add-deny-rules`](implement-master.md#add-deny-rules) + +- **Passes when:** no candidate root exists, or every root that exists is matched by an ignore rule and nothing under + any of them is tracked. +- **Why it matters:** Scratch notes must not become context an agent treats as source of truth; a tracked or unignored + scratch directory pollutes every session that follows ([greenfield/README.md § 6.2 Agent + Scratchpad](../../greenfield/README.md#62-agent-scratchpad--artefacts)). + +#### `CON-04` Draft allow/deny emitted + +`COSMETIC` · not scored · engine check · no playbook (the audit emits it as an input) + +- **Passes when:** the draft was written to `do-not-do.draft.md` beside the report. +- **Why it matters:** The scan knows what is dangerous in *this* repository; without it, every adoption starts from a + generic template that fits nobody and an agent (or a human) has to rediscover the same paths by exploring + ([Phased-Approach.md § Activities](../framework/phased-approach.md#activities)). + +### Security hygiene (SEC) + +3 checks: 2 BLOCKER, 1 DEGRADER. + +#### `SEC-01` .env untracked and ignored + +`BLOCKER` · scored · engine check · fixed by +[`env-inventory-and-secrets`](implement-master.md#env-inventory-and-secrets) + +- **Passes when:** no `.env`/`.env.*` file is tracked, and ignore rules cover both shapes. +- **Why it matters:** An agent's context is transmitted and logged; a tracked `.env` file is a credential exposed + through the tool added for safety, not despite it ([Phased-Approach.md § Phase 1: Agentic + Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +#### `SEC-02` No key-shaped strings in tracked files + +`BLOCKER` · scored · engine check · fixed by +[`env-inventory-and-secrets`](implement-master.md#env-inventory-and-secrets) + +- **Passes when:** no tracked text file (or, with tracked state unavailable, no file in the whole inventory) contains a + credential-shaped line outside the repository's own allowlist. +- **Why it matters:** A hardcoded credential is the same exposure as a tracked `.env` file, just inside source or + configuration instead of a dedicated file ([Phased-Approach.md § + Activities](../framework/phased-approach.md#activities)). + +#### `SEC-03` Ignore patterns cover secret shapes + +`DEGRADER` · scored · engine check · fixed by +[`env-inventory-and-secrets`](implement-master.md#env-inventory-and-secrets) + +- **Passes when:** ignore rules cover every one of the eleven documented secret shapes. +- **Why it matters:** The guard must exist before the accident — ignore rules that do not cover a secret shape let a + single careless commit become permanent history ([Phased-Approach.md § Phase 1: Agentic + Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). + +## Team hygiene (informational) + +Is the repository run like a team project? Reported for the whole picture; never scored. + +### Hygiene (HYG) + +11 checks: 11 COSMETIC. + +#### `HYG-01` README present and non-placeholder + +`COSMETIC` · not scored · engine check · fixed by [`add-repo-hygiene`](implement-master.md#add-repo-hygiene) + +- **Passes when:** a root README exists, reads as text, has at least 200 characters of body, carries no placeholder + marker, and is not a bare title. +- **Why it matters:** An agent (or a person) must be able to orient itself in this project in seconds, not by opening + files at random. + +#### `HYG-02` CODEOWNERS present + +`COSMETIC` · not scored · engine check · fixed by [`add-repo-hygiene`](implement-master.md#add-repo-hygiene) + +- **Passes when:** a `CODEOWNERS` file exists at one of the recognised locations. +- **Why it matters:** Review routing should be machine-readable, not tribal knowledge about who owns what. + +#### `HYG-03` Pull-request template present + +`COSMETIC` · not scored · engine check · fixed by [`add-repo-hygiene`](implement-master.md#add-repo-hygiene) + +- **Passes when:** at least one `pull_request_template*` file exists. +- **Why it matters:** A reviewer needs the same information every time a change is filed, not whatever the author + happened to think to write. + +#### `HYG-04` Issue templates present + +`COSMETIC` · not scored · engine check · fixed by [`add-repo-hygiene`](implement-master.md#add-repo-hygiene) + +- **Passes when:** at least one file exists under `.github/ISSUE_TEMPLATE/`, or `.github/ISSUE_TEMPLATE.md` exists. +- **Why it matters:** A bug report or feature request filed in a structured way is faster to triage than free text. + +#### `HYG-05` CONTRIBUTING present + +`COSMETIC` · not scored · engine check · fixed by [`add-repo-hygiene`](implement-master.md#add-repo-hygiene) + +- **Passes when:** at least one `CONTRIBUTING`-prefixed document exists at a repository-level location. +- **Why it matters:** A contributor should not have to reverse-engineer the setup, the gate and the review flow from + reading commit history. + +#### `HYG-06` SECURITY present + +`COSMETIC` · not scored · engine check · fixed by [`add-repo-hygiene`](implement-master.md#add-repo-hygiene) + +- **Passes when:** at least one `SECURITY`-prefixed document exists at a repository-level location. +- **Why it matters:** A vulnerability should have a documented, private way to be reported, not left to whatever channel + the reporter happens to try first. + +#### `HYG-07` CHANGELOG present and recent + +`COSMETIC` · not scored · engine check · fixed by [`add-repo-hygiene`](implement-master.md#add-repo-hygiene) + +- **Passes when:** a changelog document exists and its last commit is within 365 days of `HEAD`'s. +- **Why it matters:** A reader needs to see what changed between two versions without diffing the whole history + themselves. + +#### `HYG-08` Release process documented + +`COSMETIC` · not scored · engine check · fixed by [`add-repo-hygiene`](implement-master.md#add-repo-hygiene) + +- **Passes when:** at least one text/Markdown document whose name contains `release` exists. +- **Why it matters:** Whoever cuts the next release should be following a written process, not reconstructing it from + whoever did it last time's memory. + +#### `HYG-09` Dependency update automation configured + +`COSMETIC` · not scored · engine check · fixed by [`add-repo-hygiene`](implement-master.md#add-repo-hygiene) + +- **Passes when:** a Dependabot or Renovate configuration file exists at a conventional location. +- **Why it matters:** Dependencies should be patched on a schedule, not discovered stale during an incident. + +#### `HYG-10` License present + +`COSMETIC` · not scored · engine check · fixed by [`add-repo-hygiene`](implement-master.md#add-repo-hygiene) + +- **Passes when:** a root license file (one of the five recognised name prefixes) exists. +- **Why it matters:** Anyone using or contributing to the code needs to know the terms under which it may be used, + without asking. + +#### `HYG-11` Branch protection declared as code + +`COSMETIC` · not scored · engine check · fixed by [`add-repo-hygiene`](implement-master.md#add-repo-hygiene) + +- **Passes when:** a branch-protection file or a matching Terraform resource exists. +- **Why it matters:** Protected-branch rules should be reviewed like any other change, not configured by hand in a + settings page nobody diffs. + +## Read more + +- The full rubric of each check (PARTIAL, FAIL, evidence, pitfalls): + [`audit/checks/`](../../audit/checks/README.md). +- Each check's requirement and framework section: + [Checks to framework map](how-the-checks-map-to-the-framework.md). +- How an audit runs: [Run an audit](../how-to/run-an-audit.md). diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 11b98731..6b5c9c5b 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -17,6 +17,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | coverage.py | Test coverage below the floor in `.coveragerc` | `make coverage` | `coverage` | `dev/tools/pyproject.toml` | | jsonschema | The tools' schema validator disagrees with the reference package | `make crosscheck` | `check` | `dev/tools/pyproject.toml` | | `dev/recipe_lint.py` | Broken links, bad `SKILL.md` frontmatter, oversize files, unknown tools, check IDs and requirement IDs | `make recipe-lint` | `check` | This repository | +| `dev/build_masters.py` | An [audit master list](audit-master.md) or [implement master list](implement-master.md) that no longer matches the catalogue, the check files or the playbooks | `make masters-check`; `make update-masters` rewrites both pages | `check` | This repository | | actionlint | Workflow mistakes: syntax, expressions, action inputs, shellcheck on `run:` blocks | `make workflow-lint` | `check` | `dev/tools/actionlint/go.mod` | | zizmor | Workflow and Dependabot security: injection, broad permissions, unpinned actions, cooldown. Runs at the pedantic persona, so a permission without a comment or a missing concurrency limit fails too | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | | typos | Misspelled words in the tracked text, code identifiers included (word list: [`_typos.toml`](../../_typos.toml)) ([Spell check](#spell-check)) | `make spell`; `make spell-fix` writes the fixes | `check` | `dev/tools/pyproject.toml` | @@ -38,8 +39,8 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | Go | Builds and runs actionlint and osv-scanner through `go tool` | — | `check`, `security` | the `go` line of each Go tool module | `make check` runs `format:check`, `typecheck`, `test`, the engine's rule check, `crosscheck`, `recipe-lint`, -`workflow-lint` and `spell`, in sequence. CI's `CI Gate` passes only when the `check`, `security`, `coverage`, `self-audit`, -`docs` and `commits` jobs pass ([Branch protection](branch-protection.md#the-required-checks)). +`masters-check`, `workflow-lint` and `spell`, in sequence. CI's `CI Gate` passes only when the `check`, `security`, +`coverage`, `self-audit`, `docs` and `commits` jobs pass ([Branch protection](branch-protection.md#the-required-checks)). ## Spell check diff --git a/docs/reference/implement-master.md b/docs/reference/implement-master.md new file mode 100644 index 00000000..ec1bc690 --- /dev/null +++ b/docs/reference/implement-master.md @@ -0,0 +1,450 @@ +# Implement master list + +> **Audience:** CTOs, senior engineers and anyone who wants to know what an implement run changes. +> **Source of truth:** the playbooks in [`implement/playbooks/`](../../implement/playbooks/README.md) and +> [`catalogue.json`](../../audit/catalogue.json). +> +> **Generated** by [`dev/build_masters.py`](../../dev/build_masters.py) from the playbooks and the catalogue. Do not +> edit it by hand: change a source, then run `make update-masters`. `make check` fails when this page is stale. + +--- + +An implement run closes the confirmed findings of an audit with **13 playbooks** in 8 themes. Each theme is one branch +and one pull request in the audited repository. The agent never merges: the owner reviews and merges each pull request, +and can skip or defer any finding. Each playbook below says what it achieves, what it changes per check, and what it +never does. The checks themselves are in the [audit master list](audit-master.md). + +```mermaid +flowchart LR + R["Final audit report
(owner-confirmed)"] --> T["Triage: keep, skip
or defer each finding"] + T --> P["Plan the next theme"] + P --> A["Apply its playbooks
on a branch, one commit per item"] + A --> C["Re-check with the engine
and run the gate"] + C --> PR["Pull request
for the owner"] + PR -->|owner merges| P + PR -->|last theme| F["Full re-audit
and comparison"] +``` + +## At a glance + +Themes run in dependency order: the order and the dependencies are in +[`implement/SKILL.md`](../../implement/SKILL.md#themes). + +| Theme | Playbooks | Checks closed | +|---|---|---| +| [T1 Command surface and quality tools](#t1-command-surface-and-quality-tools) | [`define-command-surface`](#define-command-surface), [`configure-quality-tools`](#configure-quality-tools) | 10 | +| [T2 Execution determinism](#t2-execution-determinism) | [`pin-execution-environment`](#pin-execution-environment), [`untrack-generated`](#untrack-generated) | 5 | +| [T3 Secrets and env](#t3-secrets-and-env) | [`env-inventory-and-secrets`](#env-inventory-and-secrets) | 4 | +| [T4 CI parity, hooks, commits](#t4-ci-parity-hooks-commits) | [`add-ci-parity`](#add-ci-parity), [`add-hooks-and-commit-rules`](#add-hooks-and-commit-rules) | 7 | +| [T5 Tests and baselines](#t5-tests-and-baselines) | [`add-baselines-and-ratchet`](#add-baselines-and-ratchet) | 8 | +| [T6 Docs and navigation](#t6-docs-and-navigation) | [`establish-docs-contract`](#establish-docs-contract) | 9 | +| [T7 AGENTS.md and governance](#t7-agentsmd-and-governance) | [`adapt-agents-md`](#adapt-agents-md), [`add-deny-rules`](#add-deny-rules) | 16 | +| [T8 Hygiene and index](#t8-hygiene-and-index) | [`add-repo-hygiene`](#add-repo-hygiene), [`register-index`](#register-index) | 14 | + +## T1 Command surface and quality tools + +### `define-command-surface` + +The repository exposes the seven verbs of [`spec/command-surface.md`](../../spec/command-surface.md) on one +authoritative runner per component, `check` composes `format:check`, `typecheck` and `test` in that order, any +second runner only delegates, and no read-only verb writes to the working tree. + +| Check | What the playbook does | +|---|---| +| [`CMD-01` Seven verbs resolvable on one runner](audit-master.md#cmd-01-seven-verbs-resolvable-on-one-runner) | Defines every missing verb on the authoritative runner | +| [`CMD-02` check composes the read-only verbs](audit-master.md#cmd-02-check-composes-the-read-only-verbs) | Makes `check` compose the three concerns | +| [`CMD-03` No divergent second definition](audit-master.md#cmd-03-no-divergent-second-definition) | Turns any second runner into a delegate | +| [`CMD-04` Write verbs separated from read-only verbs](audit-master.md#cmd-04-write-verbs-separated-from-read-only-verbs) | Splits write forms out of read-only verbs | +| [`EXEC-03` Non-interactive setup path](audit-master.md#exec-03-non-interactive-setup-path) | Names the non-interactive setup command beside the verbs (short section in Steps) | +| [`TST-01` Test verb exists, suite non-empty](audit-master.md#tst-01-test-verb-exists-suite-non-empty) | Makes `test` resolve on the runner (short section in Steps) | + +**What it never does:** + +- Do not create a second runner that composes its own `check`. +- Do not make `check` pass by dropping `test` or `typecheck`, or by `-x test`, unless the owner chose it. +- Do not add a verb that does nothing (`echo skipped`), and do not leave `--fix` or `--write` under a + read-only verb name. +- Do not restructure a multi-component repository into one workspace. +- Do not rewrite the instruction file; add the command lines only. + +Full playbook: [`define-command-surface.md`](../../implement/playbooks/define-command-surface.md). + +### `configure-quality-tools` + +The repository has a formatter, a linter, a type checker in its strictest practical mode and a dependency +scan, each configured in a committed file and reachable through its verb, adopted so that today's code keeps +`check` green and existing debt is counted, recorded and cannot grow. + +| Check | What the playbook does | +|---|---| +| [`TOOL-01` Formatter configured](audit-master.md#tool-01-formatter-configured) | `format`, `format:check` | +| [`TOOL-02` Linter configured](audit-master.md#tool-02-linter-configured) | `lint` | +| [`TOOL-03` Type checker configured](audit-master.md#tool-03-type-checker-configured) | `typecheck` | +| [`TOOL-04` Dependency / security scan verb](audit-master.md#tool-04-dependency--security-scan-verb) | `security` | + +**What it never does:** + +- No mass reformat and no mass auto-fix of the tree, unless the owner chooses it (then its own pull request). +- No rule turned off and no blanket suppression to get green; a baseline file is recorded as debt. +- No dependency upgrade of an existing tool as part of T1. +- No tool the owner did not approve, and no version range: pin exact versions. +- No secrets scanner inside `security`; no placeholder `security` verb. +- Do not put a failing strict check inside `check` without the owner's recorded agreement. + +Full playbook: [`configure-quality-tools.md`](../../implement/playbooks/configure-quality-tools.md). + +## T2 Execution determinism + +### `pin-execution-environment` + +Every run of the repository gets the same toolchain and the same dependency versions: each ecosystem has a +committed pin (JDK, Node, package manager), each component has a tracked lockfile that matches its manifest, and +the pins in CI, Dockerfiles and version files agree with each other. + +| Check | What the playbook does | +|---|---| +| [`EXEC-01` Toolchain pinned / wrapper committed](audit-master.md#exec-01-toolchain-pinned--wrapper-committed) | Commits a pin for every detected ecosystem and makes every other place that names a version agree with it | +| [`EXEC-02` Lockfile committed and consistent](audit-master.md#exec-02-lockfile-committed-and-consistent) | Commits a lockfile that resolves exactly what the manifest resolves today | +| [`EXEC-03` Non-interactive setup path](audit-master.md#exec-03-non-interactive-setup-path) | Check only (step 9): [`define-command-surface`](../../implement/playbooks/define-command-surface.md) owns the fix | + +**What it never does:** + +- Do not choose a runtime major for the owner, and do not move a major in this theme without the owner's + decision and a green build on it. +- Do not hand-edit a lockfile or regenerate one that already passes. +- Do not run `npm install`, `npm update` or `./gradlew dependencies --update-locks` to "refresh" while + locking: the point is to record what resolves today. +- Do not add a root workspace manifest to a per-component repository unless the owner asks. +- Do not edit workflow files without `ci.edits_authorised`. +- Do not build images or start containers by hand: use the repository's scripts. +- Do not install a version manager the owner did not approve. + +Full playbook: [`pin-execution-environment.md`](../../implement/playbooks/pin-execution-environment.md). + +### `untrack-generated` + +No build output is tracked in the repository, or the output that has to stay tracked says so in its first +lines, and every concern (lint, format, test, types, security) has one authoritative root config. + +| Check | What the playbook does | +|---|---| +| [`EXEC-05` No generated artifacts committed in-tree](audit-master.md#exec-05-no-generated-artifacts-committed-in-tree) | Finds tracked build output, proves nothing reads the tracked copy, and untracks it with the owner's yes (or adds a header where one fits) | +| [`EXEC-07` No competing configs for one concern](audit-master.md#exec-07-no-competing-configs-for-one-concern) | Detection and documentation only (step 8): lists competing root configs, asks which is authoritative, documents deliberate delegation | + +**What it never does:** + +- Do not untrack a path that a deploy path reads, or may read, without the owner's recorded yes. +- Do not untrack a secret-bearing file here; that is the secrets theme's decision (rotation, history). +- Do not untrack a lockfile, the Gradle wrapper or any hand-written file the ignore rule matches by accident: + fix the rule. +- Do not delete working-tree files (`git rm` without `--cached`) and do not rewrite history. +- Do not delete, merge or rename a competing config (EXEC-07). +- Do not start containers or the local stack by hand to test a reader. + +Full playbook: [`untrack-generated.md`](../../implement/playbooks/untrack-generated.md). + +## T3 Secrets and env + +### `env-inventory-and-secrets` + +An agent can list every variable a script, build or application reads without seeing one value: a committed +example file declares the names (required or optional), no secret-bearing file is tracked without a recorded +owner decision, and the ignore rules cover every secret shape the audit probes. + +| Check | What the playbook does | +|---|---| +| [`EXEC-04` Env-var inventory complete](audit-master.md#exec-04-env-var-inventory-complete) | Builds the names-only inventory and commits an example file that declares every variable the code reads | +| [`SEC-01` .env untracked and ignored](audit-master.md#sec-01-env-untracked-and-ignored) | Finds tracked `.env`-shaped files and hands each to the owner for a decision; never edits one | +| [`SEC-02` No key-shaped strings in tracked files](audit-master.md#sec-02-no-key-shaped-strings-in-tracked-files) | Counts key-shaped lines by path and line number, wires the repository's existing scanner, never prints a hit | +| [`SEC-03` Ignore patterns cover secret shapes](audit-master.md#sec-03-ignore-patterns-cover-secret-shapes) | Adds the missing secret shapes to the ignore file, with a tracked-file check before and after | + +**What it never does:** + +- Do not print, quote, log or copy a value from any env, properties, realm, keystore or credentials file, and + do not open a `.env`-like file. +- Do not rotate a credential, remove one from history or run a history-rewriting command: report and stop. +- Do not touch a tracked env file the owner keeps; do not add an ignore rule or `git rm --cached` for it. +- Do not re-raise a finding the owner rejected or skipped; new observations go to the pull request body. +- Do not template, edit or "clean" realm or identity-provider exports. +- Do not install a scanner the owner has not approved. +- Do not put shell builtins or script-internal names in the example file. +- Do not start containers or the local stack by hand to test a change; use the repository's scripts. + +Full playbook: [`env-inventory-and-secrets.md`](../../implement/playbooks/env-inventory-and-secrets.md). + +## T4 CI parity, hooks, commits + +### `add-ci-parity` + +The gate an agent runs locally is the gate CI runs. CI calls the same verbs (`check`, and `security` where it +exists), every scan or test step either blocks the merge or says in its name that it is report-only with its debt +recorded, and a committed record shows which CI job the platform requires before a merge. + +| Check | What the playbook does | +|---|---| +| [`CI-01` Pipeline present, triggered on PRs](audit-master.md#ci-01-pipeline-present-triggered-on-prs) | Verification only in most repositories: a workflow with a pull-request trigger exists. Adds or fixes the trigger block when it does not | +| [`CI-02` Local↔CI parity](audit-master.md#ci-02-localci-parity) | CI runs the local verbs, directly or through a wrapper script that calls them | +| [`CI-03` CI-only steps flagged](audit-master.md#ci-03-ci-only-steps-flagged) | Every CI step is a local command; a step that exists only in CI is flagged in its name and in the CI doc | +| [`CI-04` CI enforces the same gate as the local check](audit-master.md#ci-04-ci-enforces-the-same-gate-as-the-local-check) | No gating step swallows its exit code; non-gating steps are visibly report-only; the required-check wiring is recorded with evidence | +| [`CI-05` Workflow files are linted](audit-master.md#ci-05-workflow-files-are-linted) | A workflow linter that knows the CI provider runs from the local surface and in CI | + +**What it never does:** + +- Do not add a third-party `uses:` action without owner approval; do not install a scanner the owner has not + approved. +- Do not rename the merge-gate job, drop a job from its `needs`, or lower a gate that already blocks. +- Do not leave a green no-op (a step that prints "skipped" and exits 0), and do not hide debt: a non-blocking + step says so in its name. +- Do not claim a required check exists without the owner-supplied evidence; do not mark CI-04 `done` where the + platform cannot enforce it. +- Do not edit workflows outside a PR, or beyond the scope of `ci.edits_authorised`. +- Do not mass-fix findings the scan reveals. + +Full playbook: [`add-ci-parity.md`](../../implement/playbooks/add-ci-parity.md). + +### `add-hooks-and-commit-rules` + +A commit runs a fast local hook chain before it is created, with one committed configuration and a documented +one-time install. The commit-message convention is either enforced locally or explicitly recorded as enforced +elsewhere (the pull-request title) by the owner's decision. + +| Check | What the playbook does | +|---|---| +| [`TOOL-05` Local hook chain active](audit-master.md#tool-05-local-hook-chain-active) | Commits a hook-manager configuration the engine reads as wired, with fast changed-files hooks | +| [`TOOL-06` Commit convention enforced](audit-master.md#tool-06-commit-convention-enforced) | Offers three options; the owner may skip when the title check is enough (see Commit rule) | + +**What it never does:** + +- Do not install the hook manager on the owner's machine or add it as a repository dependency without approval. +- Do not run tests, a full lint, a full-tree format or any network call in a hook. +- Do not advise `--no-verify` or add a bypass variable. +- Do not re-propose a `skipped` commit-convention finding. +- Do not pick a manager by what looks familiar or by what this framework uses: detect, then recommend by stack. +- Do not duplicate the secret-scanner version: reuse the CI pin. + +Full playbook: [`add-hooks-and-commit-rules.md`](../../implement/playbooks/add-hooks-and-commit-rules.md). + +## T5 Tests and baselines + +### `add-baselines-and-ratchet` + +Every kind of existing debt the repository carries is counted, committed as a baseline, and guarded inside the gate +so the count can only go down; the test suite's coverage is measured and held by a floor that can only go up; and +the suite's status and its skip markers are recorded, so an agent can tell old debt from the debt it adds. + +**The rule this playbook enforces: a baseline only goes down, a floor only goes up.** Raising a count or lowering a +floor needs the owner, and is recorded as described in [Continue on error](../../implement/playbooks/add-baselines-and-ratchet.md#continue-on-error). + +| Check | What the playbook does | +|---|---| +| [`BASE-01` Baseline artifact exists](audit-master.md#base-01-baseline-artifact-exists) | Tool-native baselines plus one committed counts file, named in `AGENTS.md` | +| [`BASE-02` Current violation counts measured](audit-master.md#base-02-current-violation-counts-measured) | Each tool's full-tree count, written down | +| [`BASE-03` No-new-violations mechanism](audit-master.md#base-03-no-new-violations-mechanism) | A compare step inside `format:check`, run by CI | +| [`BASE-04` Coverage floor configured](audit-master.md#base-04-coverage-floor-configured) | Coverage threshold in the test runner's own configuration | +| [`TST-02` Suite status known](audit-master.md#tst-02-suite-status-known) | Suite status recorded from a real run | +| [`TST-03` Skip / xfail census](audit-master.md#tst-03-skip--xfail-census) | Skip-marker census, each marker judged | +| [`TST-04` Fast path documented](audit-master.md#tst-04-fast-path-documented) | One fenced single-test command in `AGENTS.md` (shared with `adapt-agents-md`) | +| [`TST-05` Coverage configuration present](audit-master.md#tst-05-coverage-configuration-present) | Coverage tool configured and a coverage verb | + +**What it never does:** + +- No mass fix, mass reformat or bulk deletion of skip markers to reach a lower count. +- No suppression of a new violation; a baseline holds today's debt only. +- No coverage floor above today's measured number, and no floor taken from a run known to misreport. +- No upgrade of a tool or runner only to get its native baseline format, without the owner. +- No agent edit that raises a count or lowers a floor; no script that rewrites a baseline or floor by itself. +- No new member of `check` for a ratchet; it goes in `format:check`. +- Do not start or stop services outside the repository's own scripts. + +Full playbook: [`add-baselines-and-ratchet.md`](../../implement/playbooks/add-baselines-and-ratchet.md). + +## T6 Docs and navigation + +### `establish-docs-contract` + +An agent finds permanent knowledge in one place: `docs/` has an index, a short style guide that the instruction +file names, a decision log with a template and an index, and a short architecture page with a diagram that was +checked. Scratch work stays out of version control. Structural traps (very large files, binaries) are listed, +not moved. + +| Check | What the playbook does | +|---|---| +| [`DOC-01` docs/ exists](audit-master.md#doc-01-docs-exists) | Confirms `docs/` holds Markdown, adds `docs/README.md` as the index; never creates a second docs home | +| [`DOC-02` docs/STYLE_GUIDE.md exists](audit-master.md#doc-02-docsstyle_guidemd-exists) | Adds a one-page `docs/STYLE_GUIDE.md` and one line in the instruction file that names it | +| [`DOC-03` artefacts/ exists and is ignored](audit-master.md#doc-03-artefacts-exists-and-is-ignored) | Verifies the scratch directory rule; adds the ignore rule only if a scratch directory exists | +| [`DOC-04` ADR directory and index](audit-master.md#doc-04-adr-directory-and-index) | Adds `docs/adr/` with a template, an index and two or three first records marked `proposed` | +| [`NAV-01` Declared component boundaries](audit-master.md#nav-01-declared-component-boundaries) | Verification only: lists the declared components in the architecture page | +| [`NAV-02` Obvious entry points](audit-master.md#nav-02-obvious-entry-points) | Verification only: names the entry points in the architecture page | +| [`NAV-03` Structural red flags](audit-master.md#nav-03-structural-red-flags) | Census of oversized files, catch-all directories, binaries and minified assets; document, never move | +| [`NAV-04` Architecture orientation artifact](audit-master.md#nav-04-architecture-orientation-artifact) | Writes `docs/architecture/overview.md` from the code index, with one validated Mermaid diagram | +| [`NAV-05` Public-symbol documentation coverage](audit-master.md#nav-05-public-symbol-documentation-coverage) | Verification only: records the ratio; no code is edited | + +**What it never does:** + +- Do not mass-reformat or mass-fix existing Markdown, and do not turn off a lint rule to make a count fall. +- Do not create a second `docs/` or restructure the existing report folders. +- Do not copy content, paths, names or values from a private repository or workspace into an open-source + repository's docs or ADRs, or from any target repository into the framework. +- Do not invent the reason behind a decision. Inferred means "proposed" and says it is inferred. +- Do not move, resize, delete or untrack files for NAV-03. +- Do not put values from env files, credentials, realm exports or keystores in any page. +- Do not install a docs tool the owner has not approved, and do not use an unpinned `npx` in CI. +- Do not accept an ADR on the owner's behalf. + +Full playbook: [`establish-docs-contract.md`](../../implement/playbooks/establish-docs-contract.md). + +## T7 AGENTS.md and governance + +### `adapt-agents-md` + +The canonical instruction file states this repository's own facts, and only those: the seven verbs as the team runs +them, a single-test command, where documentation lives, when docs change with code, which paths are boundaries, +what an agent may do with branches and releases, which harnesses read the file, and nothing an agent could infer +from the code. Every line is true against the repository, and the file stays within the 400-line budget. + +**The rule this playbook enforces: the playbook drafts, the owner edits.** `AGENTS.md` is the owner's voice. Each +finding gets the smallest diff that closes it, proposed and reviewed one at a time. The file is never rewritten +wholesale. + +| Check | What the playbook does | +|---|---| +| [`AGT-01` Instruction file present, canonical, and non-trivial](audit-master.md#agt-01-instruction-file-present-canonical-and-non-trivial) | Verification: a canonical root `AGENTS.md`, correctly cased, no stub markers, no override file | +| [`AGT-02` Component coverage by instruction files](audit-master.md#agt-02-component-coverage-by-instruction-files) | Verification; a component `AGENTS.md` only when the owner asks (component scoping is planned for a later framework version) | +| [`AGT-03` Documentation contract named](audit-master.md#agt-03-documentation-contract-named) | One line each for `docs/`, `docs/STYLE_GUIDE.md` and the scratch directory | +| [`AGT-04` Definition of done stated](audit-master.md#agt-04-definition-of-done-stated) | One definition-of-done sentence tying documentation triggers to the same change set | +| [`AGT-05` Seven verbs named](audit-master.md#agt-05-seven-verbs-named) | The seven verbs named as whole tokens, in the runner's own spelling, annotated with inline comments | +| [`AGT-06` Architectural boundaries declared](audit-master.md#agt-06-architectural-boundaries-declared) | Two or more boundary rules, each naming a path that exists | +| [`AGT-08` No competing instruction files, and the reach matrix is stated](audit-master.md#agt-08-no-competing-instruction-files-and-the-reach-matrix-is-stated) | The "This file is read by ..." line; vendor pointer files kept and reconciled, never deleted unasked | +| [`AGT-09` Branch and release boundary declared](audit-master.md#agt-09-branch-and-release-boundary-declared) | One release-boundary sentence built from the state and the repository | +| [`AGT-10` Instruction indirection resolved](audit-master.md#agt-10-instruction-indirection-resolved) | A "read X first" deferral becomes an ordinary link, with the must-always rules moved into `AGENTS.md` | +| [`AGT-11` Instructions are specific and not inferable](audit-master.md#agt-11-instructions-are-specific-and-not-inferable) | Generic text moves to `docs/`; contradicted rules become target state with the exception count; stale lines corrected | +| [`EXEC-06` Documented commands resolve](audit-master.md#exec-06-documented-commands-resolve) | Every fenced command in `AGENTS.md` resolves; the one command a finding names is fixed inline | +| [`TST-04` Fast path documented](audit-master.md#tst-04-fast-path-documented) | Verification only, when T5 wrote the fenced single-test line; written here when the item sits in T7 | + +**What it never does:** + +- Do not rewrite `AGENTS.md` wholesale, reorder its sections, or reflow its lines. +- Do not delete a vendor instruction file, or re-propose a finding the owner rejected, unasked. +- Do not edit the prohibitions section; that is [`add-deny-rules`](../../implement/playbooks/add-deny-rules.md). +- Do not edit code to make a rule true, and do not drop a rule because the code breaks it. +- Do not create a harness configuration file to point at a second rules file. +- Do not write a fact an earlier theme has not made true (a verb that does not resolve, a docs page that does not + exist), and do not name a harness in the reach line that does not load the file. +- Do not add generic prose, persona text or an overview to fill a section. +- Do not copy content from a private repository or workspace into an open-source one, or from any target into the + framework. + +Full playbook: [`adapt-agents-md.md`](../../implement/playbooks/adapt-agents-md.md). + +### `add-deny-rules` + +The instruction file has one prohibitions section, under a heading that says so, with an explicit rule for each of +the four concerns (tests, generated files and scratch, the default branch, secrets). Every test directory is named +in a code span, so an agent cannot "fix" a failing test by editing it. Scratch and tool-state directories are +ignored and untracked. The owner has a report of what each harness configuration in the repository lets an agent +do without asking, compared with the section. + +**The rule this playbook enforces: tests may change only together with the code that needs the change.** The test +rule is bounded, never a flat ban: the definition of done asks for tests, so a flat "never edit `test/`" would +contradict it. + +| Check | What the playbook does | +|---|---| +| [`AGT-07` Do-not-do / deny rules present](audit-master.md#agt-07-do-not-do--deny-rules-present) | One rule per concern in the prohibitions section: tests, generated files and scratch, default branch, secrets | +| [`CON-01` Allow/deny artifact present](audit-master.md#con-01-allowdeny-artifact-present) | The rules name real paths in code spans (the test directories, the generated folders, the scratch directory) | +| [`CON-02` Test directories protected](audit-master.md#con-02-test-directories-protected) | Every test directory the engine finds is named in a code span in the bounded test rule | +| [`CON-03` artefacts/ excluded from index and agent reads](audit-master.md#con-03-artefacts-excluded-from-index-and-agent-reads) | Scratch and tool-state directories ignored and untracked; the `.gitignore` step is shared with T6 | + +**What it never does:** + +- Do not edit anything in `AGENTS.md` outside the prohibitions section. +- Do not write a flat ban on test directories; the rule is bounded. +- Do not edit, create or delete a harness permission file unless the owner recorded `agents_md.harness_deny=yes`, + and never create one. +- Do not untrack, ignore or delete a file the owner has not confirmed is scratch, and do not change which env files + are tracked; state the owner's decision instead. +- Do not paste the CON-04 draft whole: an entry that matches nothing in the repository is noise. +- Do not open, print or quote a secret store while listing the paths. + +Full playbook: [`add-deny-rules.md`](../../implement/playbooks/add-deny-rules.md). + +## T8 Hygiene and index + +### `add-repo-hygiene` + +The repository tells a newcomer, human or agent, who owns what (CODEOWNERS), how to contribute (README, +CONTRIBUTING, pull-request and issue templates), how to report a vulnerability (SECURITY), how releases happen +(a release-process page and, where the release tool writes one, the changelog), and where the rules for merging +live (a branch-protection page). Each file is small, true against the repository, and sits where the audit reads it. + +**The rule this playbook enforces: write only what is true, and only what the repository needs.** All eleven checks +are cosmetic and unscored, so a long, generic template that nobody keeps true is worse than a short page that is. +A value the playbook cannot know (a mailbox, a response time, a second reviewer) is a named placeholder plus an owner +action in the state, never an invented contact. + +| Check | What the playbook does | +|---|---| +| [`HYG-01` README present and non-placeholder](audit-master.md#hyg-01-readme-present-and-non-placeholder) | A root README of at least 200 characters of body, no placeholder marker, not a bare title; written from the repository's own files | +| [`HYG-02` CODEOWNERS present](audit-master.md#hyg-02-codeowners-present) | A `CODEOWNERS` file at `.github/` with a default rule, the existing path rules kept, and a commented-out second-reviewer placeholder | +| [`HYG-03` Pull-request template present](audit-master.md#hyg-03-pull-request-template-present) | A `.github/pull_request_template.md` that asks for what the repository's gate and PR rules require | +| [`HYG-04` Issue templates present](audit-master.md#hyg-04-issue-templates-present) | At least one file under `.github/ISSUE_TEMPLATE/` (bug report, feature request) | +| [`HYG-05` CONTRIBUTING present](audit-master.md#hyg-05-contributing-present) | A `CONTRIBUTING` document naming the repository's own setup, gate, branch and review flow | +| [`HYG-06` SECURITY present](audit-master.md#hyg-06-security-present) | A `SECURITY` document: how to report privately, with owner-supplied contact values as a named placeholder | +| [`HYG-07` CHANGELOG present and recent](audit-master.md#hyg-07-changelog-present-and-recent) | Verification, and a decision: a changelog the release tool writes is not hand-written (Steps 6) | +| [`HYG-08` Release process documented](audit-master.md#hyg-08-release-process-documented) | A short document (name contains `release`) describing the release process in prose | +| [`HYG-09` Dependency update automation configured](audit-master.md#hyg-09-dependency-update-automation-configured) | Verification only: a Dependabot or Renovate file already exists in most repositories | +| [`HYG-10` License present](audit-master.md#hyg-10-license-present) | Verification only; the playbook never chooses a licence | +| [`HYG-11` Branch protection declared as code](audit-master.md#hyg-11-branch-protection-declared-as-code) | A page whose path contains `branch-protection` that records the rules in the repository's own words | + +**What it never does:** + +- Do not overwrite an existing hygiene file with a template; propose a diff. +- Do not hand-write or seed a changelog that a release tool owns, and do not tag or release. +- Do not enable required code-owner review, and do not edit any GitHub or GitLab setting. +- Do not invent a mailbox, a response time or a second reviewer. +- Do not copy `RELEASE_PROCESS.md` whole; write the short page. +- Do not copy a private runbook, another repository's page, or its values into a repository's file. +- Do not choose a licence. +- Do not create a file in a component directory expecting it to count. +- Do not reformat existing pages (`format.mass_reformat`). + +Full playbook: [`add-repo-hygiene.md`](../../implement/playbooks/add-repo-hygiene.md). + +### `register-index` + +An agent that opens the repository can reach the code index without being told how: the index server is registered +where the harness the team uses reads it, an index exists for this checkout, and the repository says when to refresh +it. An audit run on any machine reaches the same verdict, because the registration an audit relies on is committed, +not left in one developer's home directory. + +**The rule this playbook enforces: user-level configuration needs consent.** The playbook never edits a file under +`$HOME` or an XDG directory unless the owner said so and the state records it. A committed project-level file is +offered, never assumed: it makes every contributor's agent start a local tool. + +| Check | What the playbook does | +|---|---| +| [`IDX-01` codebase-memory-mcp registered for this repository](audit-master.md#idx-01-codebase-memory-mcp-registered-for-this-repository) | A project-level MCP config that names the index server for the harness the team uses, or a documented user-level registration with the gap recorded | +| [`IDX-02` An index exists for this checkout](audit-master.md#idx-02-an-index-exists-for-this-checkout) | The index is built for this checkout (`index_repository`) and answers a structural query | +| [`IDX-03` Index freshness](audit-master.md#idx-03-index-freshness) | A freshness rule in the instruction file and a freshness query (`detect_changes`) as the evidence | + +**What it never does:** + +- Do not edit a file under `$HOME` or an XDG directory without a recorded `index.user_config_consent=yes`. +- Do not commit a config with an absolute path to one developer's machine, a token or any secret. +- Do not create configuration for harnesses the team does not use. +- Do not install the index tool, add a git hook or edit CI. +- Do not print or quote a user-level config file; report the path and whether it names the server. +- Do not claim the engine's IDX-03 verdict can pass today, and do not read its `UNKNOWN` as stale. +- Do not index a different checkout and report it as this one's. + +Full playbook: [`register-index.md`](../../implement/playbooks/register-index.md). + +## Checks with no playbook + +- [`CON-04` Draft allow/deny emitted](audit-master.md#con-04-draft-allowdeny-emitted): the audit emits it, and a + playbook reads it as an input. + +## Read more + +- The stages, the hard rules and the state file: [`implement/SKILL.md`](../../implement/SKILL.md). +- How to run it: [Implement an audit report](../how-to/implement-an-audit-report.md). diff --git a/implement/AGENTS.md b/implement/AGENTS.md index b571aefb..49dd7b18 100644 --- a/implement/AGENTS.md +++ b/implement/AGENTS.md @@ -27,6 +27,9 @@ folder. It never merges, force-pushes or pushes tags. See [ADR 0003](docs/architecture/adrs/0003-push-and-pr-policy.md). - Read [`playbooks/README.md`](playbooks/README.md) before you add or change a playbook. Keep the anatomy it fixes, section for section. +- A change to a playbook, or to the playbook list in `playbooks/README.md`, changes the generated + [implement master list](../docs/reference/implement-master.md). Run `make update-masters` from the repository + root and commit the result in the same change. - A playbook does not repeat the shared procedure in `SKILL.md` (Stages 3 and 4). It does not restate a stack profile; it links to [`audit/stacks/`](../audit/stacks/README.md). - The tools import the shared helpers (`_common.py` and others) from `audit/tools/`. Import From ae3dee8a4e13bced0a45e892c0c68a07abb137d4 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 14:48:51 +0530 Subject: [PATCH 036/111] test(engine): run the HYG-09 and EXEC-08 tests in-process so coverage sees them The new tests went through support.run_audit, which starts the engine in a subprocess that coverage.py does not trace. The new rule code counted as uncovered, and the CI coverage job fell to 90% against the 91% floor (locally 91%, because machine-local IDX checks cover 40 more lines there). The tests now build the inventory and stack and call the check function, as test_exec_pack.py does; the assertions are unchanged and the fixture matrix still runs the subprocess path. Local coverage: 760 lines missed, down from 830. Roadmap-Step: C3 Co-Authored-By: Claude Opus 5.5 --- .../tests/test_exec08_inline_versions.py | 26 ++++++++++++++++--- audit/engine/tests/test_hyg09_policy.py | 26 ++++++++++++++++--- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/audit/engine/tests/test_exec08_inline_versions.py b/audit/engine/tests/test_exec08_inline_versions.py index 1f92eaef..6158b195 100644 --- a/audit/engine/tests/test_exec08_inline_versions.py +++ b/audit/engine/tests/test_exec08_inline_versions.py @@ -19,6 +19,13 @@ import support # noqa: E402 +from audit.rules.checks.execution import check_exec08 as CHECK # noqa: E402 +from audit.rules.registry import REGISTRY_BY_ID # noqa: E402 +from audit.scan import build_inventory # noqa: E402 +from audit.stack import detect_stack # noqa: E402 + +CHECK_ID = "EXEC-08" + MAKE_HEAD = "check:\n" @@ -28,14 +35,25 @@ def make(*recipe: str) -> str: class Exec08Tests(unittest.TestCase): def check(self, files: dict[str, str]) -> dict: + """Run the check in-process (so coverage traces it) on a repository built from ``files``.""" with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) for rel, text in files.items(): support.write(root, rel, text) - report = support.run_audit(root) - check = next(c for c in report["checks"] if c["id"] == "EXEC-08") - check["findings"] = [f for f in report.get("findings", []) if f["check"] == "EXEC-08"] - return check + inventory = build_inventory(root) + outcome = CHECK( + spec=REGISTRY_BY_ID[CHECK_ID], + target=None, + inventory=inventory, + stack=detect_stack(inventory), + components=None, + session=None, + ) + return { + "verdict": outcome.verdict.value, + "summary": outcome.summary, + "findings": [finding.to_dict() for finding in outcome.findings], + } def tools(self, check: dict) -> list[str]: return sorted(e["note"] for f in check["findings"] for e in f["evidence"]) diff --git a/audit/engine/tests/test_hyg09_policy.py b/audit/engine/tests/test_hyg09_policy.py index b09b0853..10f73aa2 100644 --- a/audit/engine/tests/test_hyg09_policy.py +++ b/audit/engine/tests/test_hyg09_policy.py @@ -20,6 +20,13 @@ import support # noqa: E402 +from audit.rules.checks.hyg import check_hyg09 as CHECK # noqa: E402 +from audit.rules.registry import REGISTRY_BY_ID # noqa: E402 +from audit.scan import build_inventory # noqa: E402 +from audit.stack import detect_stack # noqa: E402 + +CHECK_ID = "HYG-09" + PACKAGE_JSON = '{"name": "x", "version": "0.1.0", "private": true}\n' WORKFLOW = ( "name: CI\non: [pull_request]\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - run: make check\n" @@ -37,14 +44,25 @@ def entry(ecosystem: str, *extra: str) -> str: class Hyg09PolicyTests(unittest.TestCase): def check(self, files: dict[str, str]) -> dict: + """Run the check in-process (so coverage traces it) on a repository built from ``files``.""" with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) for rel, text in files.items(): support.write(root, rel, text) - report = support.run_audit(root) - check = next(c for c in report["checks"] if c["id"] == "HYG-09") - check["findings"] = [f for f in report.get("findings", []) if f["check"] == "HYG-09"] - return check + inventory = build_inventory(root) + outcome = CHECK( + spec=REGISTRY_BY_ID[CHECK_ID], + target=None, + inventory=inventory, + stack=detect_stack(inventory), + components=None, + session=None, + ) + return { + "verdict": outcome.verdict.value, + "summary": outcome.summary, + "findings": [finding.to_dict() for finding in outcome.findings], + } def test_every_ecosystem_covered_passes(self): check = self.check( From 0f4a6e063036280f2fefa82b31b2fa3d7fba639b Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 15:10:35 +0530 Subject: [PATCH 037/111] build(dev): run make coverage as CI does, so a local pass means a CI pass PR #79 passed make coverage locally (91%) and failed the CI coverage job (90%). The IDX checks read the user's index store and MCP configs from $HOME, which a CI runner does not have, so a developer's machine covered about 43 more lines. make coverage now runs its suites with an empty HOME under dev/dist/coverage/ (git-ignored), CI=true (the positive-fixture test skips as in CI) and AUDIT_INDEX_STORE and XDG_* unset. uv's cache, Python and tool directories are resolved from the real home first, so nothing is downloaded again. Locally this gives CI's exact count: 803 lines missed (idx.py 96, probes.py 14). AGENTS.md and the dev-tooling page say so. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 5 +++-- Makefile | 16 ++++++++++++---- docs/reference/dev-tooling.md | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7ee4a835..d08493b8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,8 +68,9 @@ no vendor-specific instruction file exists beside it. The developer runs `lefthook install` once per clone; an agent never does. Hooks do not replace the gate: CI runs `make check`, `make security` and `make docs-lint`. - `make coverage` runs the engine's, the audit tools' and the implement tools' suites under coverage.py. It fails - below the floor in `.coveragerc` (`fail_under`). CI runs it as the `coverage` job. The floor is debt that only goes - up: raise it when coverage rises, never lower it to get green. + below the floor in `.coveragerc` (`fail_under`). CI runs it as the `coverage` job. It runs as CI does (an empty + `HOME` and `CI=true`), so a local pass means a CI pass. The floor is debt that only goes up: raise it when + coverage rises, never lower it to get green. - Quality baseline: existing lint debt is counted in `.quality-baseline.json` (ruff, mypy, Markdown lint outside the ratchet) and the secrets in `.secrets.baseline`. No new violations: `make format-check`, `make typecheck` and `make docs-lint` compare the live count with the baseline and fail when it went up (a ratchet). A baseline only goes diff --git a/Makefile b/Makefile index 1c74c15a..ad5ea5af 100644 --- a/Makefile +++ b/Makefile @@ -129,12 +129,20 @@ test: # Runs the engine's suite and the recipe tools' suite under coverage.py and fails when the total drops below the # floor in .coveragerc (`fail_under`). Not part of `check`; CI runs it as its own job. Needs uv. +# +# It runs as CI runs it, so its number is CI's number. HOME is an empty directory: the IDX checks read the user's +# index store and MCP configs from $HOME, which a CI runner does not have. CI=true lets the positive-fixture test +# skip as it does in CI (its scaffolder is found through $HOME). uv's own directories are resolved from the real +# home first, so uvx still finds its cache and tools and nothing is downloaded again. +COVERAGE_HOME := $(CURDIR)/dev/dist/coverage/home +COVERAGE_ENV = env -u AUDIT_INDEX_STORE -u XDG_CONFIG_HOME -u XDG_CACHE_HOME HOME=$(COVERAGE_HOME) CI=true \ + UV_CACHE_DIR="$$(uv cache dir)" UV_PYTHON_INSTALL_DIR="$$(uv python dir)" UV_TOOL_DIR="$$(uv tool dir)" coverage: - mkdir -p $(dir $(COVERAGE_DATA)) + rm -rf $(COVERAGE_HOME) && mkdir -p $(COVERAGE_HOME) rm -f $(COVERAGE_DATA) - cd $(ENGINE) && COVERAGE_FILE=$(COVERAGE_DATA) $(COVERAGE) run --rcfile=$(CURDIR)/.coveragerc -m unittest discover -s tests - COVERAGE_FILE=$(COVERAGE_DATA) $(COVERAGE) run --rcfile=.coveragerc --append -m unittest discover -s audit/tools/tests - COVERAGE_FILE=$(COVERAGE_DATA) $(COVERAGE) run --rcfile=.coveragerc --append -m unittest discover -s implement/tools/tests + cd $(ENGINE) && $(COVERAGE_ENV) COVERAGE_FILE=$(COVERAGE_DATA) $(COVERAGE) run --rcfile=$(CURDIR)/.coveragerc -m unittest discover -s tests + $(COVERAGE_ENV) COVERAGE_FILE=$(COVERAGE_DATA) $(COVERAGE) run --rcfile=.coveragerc --append -m unittest discover -s audit/tools/tests + $(COVERAGE_ENV) COVERAGE_FILE=$(COVERAGE_DATA) $(COVERAGE) run --rcfile=.coveragerc --append -m unittest discover -s implement/tools/tests COVERAGE_FILE=$(COVERAGE_DATA) $(COVERAGE) report --rcfile=.coveragerc # The engine on this repository, read-only. Reports go under dist/self-audit/ (git-ignored); only artefacts/ and dist/ diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 11b98731..7f32eeb8 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -86,7 +86,7 @@ fails its gate. When a count goes down, lower the baseline in the same change. | --- | --- | --- | | ruff, mypy and Markdown lint counts | `.quality-baseline.json` | `make format:check`, `make typecheck`, `make docs-lint` | | Known key-shaped strings | `.secrets.baseline` | `make security` | -| Coverage floor | `.coveragerc` (`fail_under`) | `make coverage` | +| Coverage floor | `.coveragerc` (`fail_under`) | `make coverage` (runs as CI does: an empty `HOME` and `CI=true`) | | Readiness findings | `.audit-baseline.json` | `make self-audit` | ## Local hooks From 85a2a949fe63ea59d3bc159a35fec1f076d72fcf Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 15:13:53 +0530 Subject: [PATCH 038/111] docs: regenerate the master lists for EXEC-08 dev added EXEC-08 and moved the catalogue to 0.4.0, so masters-check failed on the merged tree. make update-masters brings both pages back in line. Co-Authored-By: Claude Opus 5.5 --- docs/reference/audit-master.md | 25 +++++++++++++++++++------ docs/reference/implement-master.md | 12 ++++++++---- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/docs/reference/audit-master.md b/docs/reference/audit-master.md index d55915f2..38a73bc0 100644 --- a/docs/reference/audit-master.md +++ b/docs/reference/audit-master.md @@ -9,8 +9,8 @@ --- -The audit runs **72 checks** in 13 families (catalogue 0.3.0). 58 are scored; the rest are reported for the whole -picture and earn no credit. The engine decides 70 checks from the files alone; the agent judges 2. Each check below says +The audit runs **73 checks** in 13 families (catalogue 0.4.0). 59 are scored; the rest are reported for the whole +picture and earn no credit. The engine decides 71 checks from the files alone; the agent judges 2. Each check below says what passes and why it matters. To fix a finding, follow the playbook it names in the [implement master list](implement-master.md). @@ -27,7 +27,7 @@ list](implement-master.md). | [Agent context](#agent-context) | Can an agent find the code, the rules and the documents it needs before it edits anything? | [`IDX`](#discovery-and-index-idx), [`AGT`](#agent-governance-agt), [`DOC`](#documentation-contract-doc), [`NAV`](#navigation-nav) | 23 (3 BLOCKER, 16 DEGRADER, 4 COSMETIC) | | [Command surface and quality gates](#command-surface-and-quality-gates) | Can an agent run one command that formats, lints, type-checks and tests, and trust its answer? | [`CMD`](#command-surface-cmd), [`TOOL`](#tooling-tool), [`TST`](#tests-tst), [`BASE`](#baselines-and-the-ratchet-base) | 19 (3 BLOCKER, 14 DEGRADER, 2 COSMETIC) | | [Enforcement in CI](#enforcement-in-ci) | Does the pipeline run the same gate the agent runs locally, and does it block the merge? | [`CI`](#continuous-integration-ci) | 5 (2 BLOCKER, 3 DEGRADER) | -| [Reproducible execution](#reproducible-execution) | Does the same command give the same result on a fresh machine, without a human in the loop? | [`EXEC`](#execution-determinism-exec) | 7 (4 BLOCKER, 3 DEGRADER) | +| [Reproducible execution](#reproducible-execution) | Does the same command give the same result on a fresh machine, without a human in the loop? | [`EXEC`](#execution-determinism-exec) | 8 (4 BLOCKER, 4 DEGRADER) | | [Safety boundaries and secrets](#safety-boundaries-and-secrets) | Is it written down, and enforced, what an agent must never touch or reveal? | [`CON`](#constraints-con), [`SEC`](#security-hygiene-sec) | 7 (4 BLOCKER, 2 DEGRADER, 1 COSMETIC) | | [Team hygiene (informational)](#team-hygiene-informational) | Is the repository run like a team project? Reported for the whole picture; never scored. | [`HYG`](#hygiene-hyg) | 11 (11 COSMETIC) | @@ -530,7 +530,7 @@ Does the same command give the same result on a fresh machine, without a human i ### Execution determinism (EXEC) -7 checks: 4 BLOCKER, 3 DEGRADER. +8 checks: 4 BLOCKER, 4 DEGRADER. #### `EXEC-01` Toolchain pinned / wrapper committed @@ -603,6 +603,17 @@ Does the same command give the same result on a fresh machine, without a human i picks whichever result turns up first, which is not the same thing as picking correctly ([Phased-Approach.md § Phase 1: Agentic Bootstrap](../framework/phased-approach.md#phase-1-agentic-bootstrap-)). +#### `EXEC-08` Tool versions visible to dependency automation + +`DEGRADER` · scored · engine check · fixed by +[`pin-execution-environment`](implement-master.md#pin-execution-environment) + +- **Passes when:** no literal tool version in any file read, or the file is matched by a Renovate `customManagers` (or + `regexManagers`) file pattern. +- **Why it matters:** Dependabot and Renovate read manifests, not commands. A tool version typed into a `Makefile`, a + workflow or a hook gets no security alert and no update, and an agent asked to upgrade the tool must find every copy + by hand. + ## Safety boundaries and secrets Is it written down, and enforced, what an agent must never touch or reveal? @@ -759,8 +770,10 @@ Is the repository run like a team project? Reported for the whole picture; never `COSMETIC` · not scored · engine check · fixed by [`add-repo-hygiene`](implement-master.md#add-repo-hygiene) -- **Passes when:** a Dependabot or Renovate configuration file exists at a conventional location. -- **Why it matters:** Dependencies should be patched on a schedule, not discovered stale during an incident. +- **Passes when:** a configuration exists, every detected ecosystem has an entry, and no entry has the trap. +- **Why it matters:** Dependencies should be patched on a schedule, not discovered stale during an incident. A + configuration that silently misses an ecosystem, or whose rules do not reach security updates, looks like a policy and + is not one. #### `HYG-10` License present diff --git a/docs/reference/implement-master.md b/docs/reference/implement-master.md index ec1bc690..1cdf3873 100644 --- a/docs/reference/implement-master.md +++ b/docs/reference/implement-master.md @@ -33,7 +33,7 @@ Themes run in dependency order: the order and the dependencies are in | Theme | Playbooks | Checks closed | |---|---|---| | [T1 Command surface and quality tools](#t1-command-surface-and-quality-tools) | [`define-command-surface`](#define-command-surface), [`configure-quality-tools`](#configure-quality-tools) | 10 | -| [T2 Execution determinism](#t2-execution-determinism) | [`pin-execution-environment`](#pin-execution-environment), [`untrack-generated`](#untrack-generated) | 5 | +| [T2 Execution determinism](#t2-execution-determinism) | [`pin-execution-environment`](#pin-execution-environment), [`untrack-generated`](#untrack-generated) | 6 | | [T3 Secrets and env](#t3-secrets-and-env) | [`env-inventory-and-secrets`](#env-inventory-and-secrets) | 4 | | [T4 CI parity, hooks, commits](#t4-ci-parity-hooks-commits) | [`add-ci-parity`](#add-ci-parity), [`add-hooks-and-commit-rules`](#add-hooks-and-commit-rules) | 7 | | [T5 Tests and baselines](#t5-tests-and-baselines) | [`add-baselines-and-ratchet`](#add-baselines-and-ratchet) | 8 | @@ -98,14 +98,16 @@ Full playbook: [`configure-quality-tools.md`](../../implement/playbooks/configur ### `pin-execution-environment` Every run of the repository gets the same toolchain and the same dependency versions: each ecosystem has a -committed pin (JDK, Node, package manager), each component has a tracked lockfile that matches its manifest, and -the pins in CI, Dockerfiles and version files agree with each other. +committed pin (JDK, Node, package manager), each component has a tracked lockfile that matches its manifest, the +pins in CI, Dockerfiles and version files agree with each other, and every tool version the gate runs sits in a +manifest that dependency automation reads. | Check | What the playbook does | |---|---| | [`EXEC-01` Toolchain pinned / wrapper committed](audit-master.md#exec-01-toolchain-pinned--wrapper-committed) | Commits a pin for every detected ecosystem and makes every other place that names a version agree with it | | [`EXEC-02` Lockfile committed and consistent](audit-master.md#exec-02-lockfile-committed-and-consistent) | Commits a lockfile that resolves exactly what the manifest resolves today | | [`EXEC-03` Non-interactive setup path](audit-master.md#exec-03-non-interactive-setup-path) | Check only (step 9): [`define-command-surface`](../../implement/playbooks/define-command-surface.md) owns the fix | +| [`EXEC-08` Tool versions visible to dependency automation](audit-master.md#exec-08-tool-versions-visible-to-dependency-automation) | Moves each tool version written into a command into a manifest with a lockfile, and has the command read it (step 10) | **What it never does:** @@ -118,6 +120,7 @@ the pins in CI, Dockerfiles and version files agree with each other. - Do not edit workflow files without `ci.edits_authorised`. - Do not build images or start containers by hand: use the repository's scripts. - Do not install a version manager the owner did not approve. +- Do not upgrade a tool while moving its version into a manifest. Full playbook: [`pin-execution-environment.md`](../../implement/playbooks/pin-execution-environment.md). @@ -392,7 +395,7 @@ action in the state, never an invented contact. | [`HYG-06` SECURITY present](audit-master.md#hyg-06-security-present) | A `SECURITY` document: how to report privately, with owner-supplied contact values as a named placeholder | | [`HYG-07` CHANGELOG present and recent](audit-master.md#hyg-07-changelog-present-and-recent) | Verification, and a decision: a changelog the release tool writes is not hand-written (Steps 6) | | [`HYG-08` Release process documented](audit-master.md#hyg-08-release-process-documented) | A short document (name contains `release`) describing the release process in prose | -| [`HYG-09` Dependency update automation configured](audit-master.md#hyg-09-dependency-update-automation-configured) | Verification only: a Dependabot or Renovate file already exists in most repositories | +| [`HYG-09` Dependency update automation configured](audit-master.md#hyg-09-dependency-update-automation-configured) | An update entry for each detected ecosystem, no `target-branch` beside `ignore` or `groups`, and the owner's update policy (Step 8) | | [`HYG-10` License present](audit-master.md#hyg-10-license-present) | Verification only; the playbook never chooses a licence | | [`HYG-11` Branch protection declared as code](audit-master.md#hyg-11-branch-protection-declared-as-code) | A page whose path contains `branch-protection` that records the rules in the repository's own words | @@ -405,6 +408,7 @@ action in the state, never an invented contact. - Do not copy `RELEASE_PROCESS.md` whole; write the short page. - Do not copy a private runbook, another repository's page, or its values into a repository's file. - Do not choose a licence. +- Do not replace Renovate with Dependabot or the reverse, and do not turn on security updates in the settings. - Do not create a file in a component directory expecting it to count. - Do not reformat existing pages (`format.mass_reformat`). From 61d0e2de4a150e1d18fa83f676145d19e59c2d90 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 16:07:24 +0530 Subject: [PATCH 039/111] feat(audit): record the framework release and install in each report Report schema 0.2 replaces provenance.recipe and provenance.framework_revision with one required object, provenance.framework: release (the release version), revision (12-digit commit, or null outside a checkout) and install (checkout, bundle or copy). The schema reads 0.2 only; there is no backward compatibility before the first outside user or 1.0.0. report.py detects the install from the folder above the recipe: a git top level is a checkout, a SKILL.md there is a bundle, anything else is a copy. FRAMEWORK_RELEASE replaces RECIPE_VERSION and keeps the release-please marker. The Markdown Provenance table now opens with a Framework row and shows the catalogue in its own row. The nine recipe-report examples and eval fixtures move to 0.2 and their Markdown is re-rendered. The state examples get the example report's new sha256, and the secrets baseline drops the five full-SHA entries. Roadmap-Step: V2 Co-Authored-By: Claude Opus 5.5 --- .secrets.baseline | 49 +++-------------- RELEASE_PROCESS.md | 8 +-- audit/schema/SCHEMA.md | 35 +++++++----- audit/schema/audit-report.schema.json | 32 ++++++----- ...service.audit-report.20261010T094207Z.json | 10 ++-- ...e-service.audit-report.20261010T094207Z.md | 6 +-- audit/schema/examples/minimal.json | 10 ++-- audit/schema/examples/minimal.md | 6 +-- ...ess-state.example-service.in-progress.json | 2 +- .../readiness-state.example-service.json | 2 +- ...ntic-tdd.audit-draft.20261010T065738Z.json | 10 ++-- ...tant-sdk.audit-draft.20261010T065739Z.json | 10 ++-- ...t-server.audit-draft.20261010T065739Z.json | 10 ++-- ...t-server-first-audit-20261028T131409Z.json | 10 ++-- ...ant-server-first-audit-20261028T131409Z.md | 10 ++-- ...-server.audit-report.20261106T123400Z.json | 10 ++-- ...nt-server.audit-report.20261106T123400Z.md | 6 +-- audit/tools/_engine_adapter.py | 2 +- audit/tools/render_report.py | 22 +++++--- audit/tools/report.py | 45 ++++++++++++---- audit/tools/tests/test_engine_adapter.py | 2 +- audit/tools/tests/test_import_engine.py | 2 +- audit/tools/tests/test_render_report.py | 22 +++++--- audit/tools/tests/test_report.py | 54 ++++++++++++++++--- audit/tools/tests/test_validate_report.py | 4 +- ...ockroom-python-audit-20261106T085315Z.json | 10 ++-- ...stockroom-python-audit-20261106T085315Z.md | 6 +-- ...ockroom-python-audit-20261106T085315Z.json | 10 ++-- ...stockroom-python-audit-20261106T085315Z.md | 6 +-- dev/tests/test_release_config.py | 6 +-- docs/reference/tools-cli.md | 4 +- 31 files changed, 242 insertions(+), 179 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index b49ceb1e..c8aa6ed5 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -279,7 +279,7 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/readiness-state.example-service.in-progress.json", - "hashed_secret": "1396ee91052c2457767d76ce883d3c222ef52651", + "hashed_secret": "03179aaeae53aacc8a97dd50d94f55a4a770ba5f", "is_verified": false, "line_number": 17 } @@ -288,19 +288,12 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/readiness-state.example-service.json", - "hashed_secret": "1396ee91052c2457767d76ce883d3c222ef52651", + "hashed_secret": "03179aaeae53aacc8a97dd50d94f55a4a770ba5f", "is_verified": false, "line_number": 17 } ], "audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json": [ - { - "type": "Hex High Entropy String", - "filename": "audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json", - "hashed_secret": "2b54530ea1fbffda86f6e0429ab07696b5b5e9a8", - "is_verified": false, - "line_number": 9 - }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json", @@ -340,13 +333,6 @@ } ], "audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json": [ - { - "type": "Hex High Entropy String", - "filename": "audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json", - "hashed_secret": "2b54530ea1fbffda86f6e0429ab07696b5b5e9a8", - "is_verified": false, - "line_number": 9 - }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json", @@ -386,13 +372,6 @@ } ], "audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json": [ - { - "type": "Hex High Entropy String", - "filename": "audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json", - "hashed_secret": "2b54530ea1fbffda86f6e0429ab07696b5b5e9a8", - "is_verified": false, - "line_number": 9 - }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json", @@ -539,9 +518,9 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-first-audit-20261028T131409Z.json", - "hashed_secret": "ca3209f301c873dca5edefee204232611ca9a908", + "hashed_secret": "b9bbeca69c7ec12a4b808e759fa7048dfd406c08", "is_verified": false, - "line_number": 9 + "line_number": 7 }, { "type": "Hex High Entropy String", @@ -562,9 +541,9 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/server-worked-example/digital-assistant-server.audit-report.20261106T123400Z.json", - "hashed_secret": "752286e3a9bbde17785f11045825b600c0fccf29", + "hashed_secret": "d5ebd289e97253ca7d215705bbd3334b629e1df5", "is_verified": false, - "line_number": 9 + "line_number": 7 }, { "type": "Hex High Entropy String", @@ -1464,13 +1443,6 @@ } ], "dev/eval/implement-fixtures/testdata/reports/base/stockroom-python-audit-20261106T085315Z.json": [ - { - "type": "Hex High Entropy String", - "filename": "dev/eval/implement-fixtures/testdata/reports/base/stockroom-python-audit-20261106T085315Z.json", - "hashed_secret": "77502ed09eda3ee8c73cd2cd63a0cd86b066624e", - "is_verified": false, - "line_number": 9 - }, { "type": "Hex High Entropy String", "filename": "dev/eval/implement-fixtures/testdata/reports/base/stockroom-python-audit-20261106T085315Z.json", @@ -1487,13 +1459,6 @@ } ], "dev/eval/implement-fixtures/testdata/reports/planted/stockroom-python-audit-20261106T085315Z.json": [ - { - "type": "Hex High Entropy String", - "filename": "dev/eval/implement-fixtures/testdata/reports/planted/stockroom-python-audit-20261106T085315Z.json", - "hashed_secret": "77502ed09eda3ee8c73cd2cd63a0cd86b066624e", - "is_verified": false, - "line_number": 9 - }, { "type": "Hex High Entropy String", "filename": "dev/eval/implement-fixtures/testdata/reports/planted/stockroom-python-audit-20261106T085315Z.json", @@ -1519,5 +1484,5 @@ } ] }, - "generated_at": "2026-11-09T08:43:23Z" + "generated_at": "2026-11-09T10:22:05Z" } diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 5f02aa86..25fff8be 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -69,7 +69,7 @@ The version is written in five places, and release-please changes all five in th - [`.release-please-manifest.json`](.release-please-manifest.json): the last released version. - `__version__` in [`audit/engine/audit/__init__.py`](audit/engine/audit/__init__.py): what `audit --version` and the report's `provenance.engine.version` print. -- `RECIPE_VERSION` in [`audit/tools/report.py`](audit/tools/report.py): the report's `provenance.recipe.version`. +- `FRAMEWORK_RELEASE` in [`audit/tools/report.py`](audit/tools/report.py): the report's `provenance.framework.release`. - `RECIPE_VERSION` in [`implement/tools/progress.py`](implement/tools/progress.py): the state's `recipe.version`. - The new entry in [`CHANGELOG.md`](CHANGELOG.md). @@ -192,9 +192,9 @@ The push to `main` starts `release-please.yml`. A minute later the release pull gh pr list --base main --head release-please--branches--main ``` -2. Read its diff. It changes `.release-please-manifest.json`, `__version__` in the engine, the two - `RECIPE_VERSION` lines and `CHANGELOG.md`. A second commit, `style(changelog): ...`, holds the markdownlint - fixes. +2. Read its diff. It changes `.release-please-manifest.json`, `__version__` in the engine, `FRAMEWORK_RELEASE` + in `report.py`, `RECIPE_VERSION` in `progress.py` and `CHANGELOG.md`. A second commit, + `style(changelog): ...`, holds the markdownlint fixes. 3. Check the version. To force a different one, see [Choose the version](#choose-the-version). 4. Without the App token, close and reopen the pull request to start CI: diff --git a/audit/schema/SCHEMA.md b/audit/schema/SCHEMA.md index 819cfd9a..31129743 100644 --- a/audit/schema/SCHEMA.md +++ b/audit/schema/SCHEMA.md @@ -10,7 +10,7 @@ | Item | Description | |---|---| -| [`audit-report.schema.json`](audit-report.schema.json) | The canonical result of an audit. JSON Schema 2020-12, `schema_version` `0.1`. | +| [`audit-report.schema.json`](audit-report.schema.json) | The canonical result of an audit. JSON Schema 2020-12, `schema_version` `0.2`. | | [`audit-intake.schema.json`](audit-intake.schema.json) | The answers to the audit's Stage 0 questions, given in a file. `intake_version` `0.1`. | | [`catalogue.schema.json`](catalogue.schema.json) | The check catalogue, [`../catalogue.json`](../catalogue.json): every check in one ID namespace. | | [`facts.schema.json`](facts.schema.json) | The file `facts.py` writes: counts and names that describe a repository, with no verdict. `facts_version` `0.1`. See [Facts file](#facts-file). | @@ -31,18 +31,24 @@ The Markdown report, which [`tools/render_report.py`](../tools/render_report.py) | Version | Where | Meaning | |---|---|---| -| `schema_version` | the report | The shape of the report. `0.1` today. | +| `schema_version` | the report | The shape of the report. `0.2` today; see the note below the table. | | `intake_version` | the intake file | The shape of the intake file. `0.1` today. | | `facts_version` | the facts file | The shape of the facts file. `0.1` today. | | `schema_version` | the readiness state | The shape of the state file, a separate schema from the report's. `0.2` today; `0.2` added the optional `source_reports[].sha256`, and `0.1` states stay valid. | | `catalogue_version` | `provenance` | The check catalogue the report was built against, `MAJOR.MINOR.PATCH`, as in [`catalogue.json`](../catalogue.json). The bump rules are in [`catalogue.schema.json`](catalogue.schema.json). | -| `provenance.recipe.version` | `provenance` | The audit recipe that produced the report. | +| `provenance.framework.release` | `provenance` | The framework release that produced the report. | | `provenance.engine.version`, `ruleset_hash` | `provenance` | The deterministic engine and its ruleset. | +Report schema `0.2` replaced `provenance.recipe` and `provenance.framework_revision` with `provenance.framework`. +The tools do not read a `0.1` report. The release is the version in +[`.release-please-manifest.json`](../../.release-please-manifest.json). + Policy: - A `0.x` schema may break between releases. The version is `1.0` once the audit recipe and the implementation recipe have both run end to end on two repositories. Until then nothing may rely on a field staying put. - A consumer refuses a `schema_version` it does not know. It never guesses. +- No backward compatibility yet: a new `0.x` replaces the old one, and the tools read only the current one. + This holds until the first outside user or `1.0`, whichever comes first. - A schema change that adds a field is a new `0.x`. So is every change to a field's meaning, even where the field name stays. `0.1` itself stays open until the first tool that reads it ships (S2.2), so edits before then do not bump the version. - The schemas, the check catalogue, the check rubrics and the tools are one shared layer. A report records `catalogue_version` so that a later run can warn when its catalogue differs. @@ -53,7 +59,7 @@ Policy: ```text report ├── schema_version, status -├── provenance recipe, framework_revision, catalogue_version, engine?, agent, target (label, path, …), started_at, finalised_at? +├── provenance framework (release, revision, install), catalogue_version, engine?, agent, target (label, path, …), started_at, finalised_at? ├── run_context eval_mode, confirmation_policy, provider_exposure_acknowledged, index, probes, excluded, known_components, stack_profile? ├── stack ecosystems, package_managers, ci_providers, test_frameworks, hook_managers, notes? ├── components[] name, path, declared, declared_by?, manifest?, agents_md? @@ -77,11 +83,11 @@ A `?` marks an optional field. Every object is closed: a field the schema does n ## Commit ids -The tools write every commit id in a report (`provenance.framework_revision` and `provenance.target.git_sha`) as its +The tools write every commit id in a report (`provenance.framework.revision` and `provenance.target.git_sha`) as its first 12 hex digits. A full 40-digit id has the same shape as an access token, and secret scanners (gitleaks' default -rules, for one) flag it in a file they read, so a report committed to a repository with a scan would fail it. The schema -still accepts a full id so that earlier reports stay valid. Twelve digits name the commit unambiguously in any realistic -repository, and `git` accepts them. +rules, for one) flag it in a file they read, so a report committed to a repository with a scan would fail it. +`provenance.framework.revision` accepts only 12 digits. `provenance.target.git_sha` still accepts a full id. +Twelve digits name the commit unambiguously in any realistic repository, and `git` accepts them. ## Fields @@ -95,14 +101,19 @@ repository, and `git` accepts them. | Field | Meaning | |---|---| -| `recipe` | Name and version of the audit recipe. | -| `framework_revision` | Short Git SHA (12 hex digits) of the framework repository the recipe ran from. `null` when it was not a git checkout. See Commit ids. | +| `framework.release` | The framework release of the tools that wrote the report. See Versioning. | +| `framework.revision` | Short Git SHA (12 hex digits) of the framework checkout the tools ran from. `null` when `install` is not `checkout`, or git could not read it. See Commit ids. | +| `framework.install` | `checkout` (git), `bundle` (an installed skill) or `copy` (anything else). See below. | | `catalogue_version` | See Versioning. | | `engine` | Absent until an engine report is imported. `name`, `version`, `ruleset_revision`, `ruleset_hash`, and `scanned_at` as the engine reported them. | | `agent` | `harness` and `model`. Self-reported by the operator or the agent and unverified. `none` for both when no agent took part, as in a report that only imports an engine run. | | `target` | `label`, `path`, `git_sha` (12 hex digits; see Commit ids), `branch` and `dirty`. `label` is the short repository name used in output file names (see Output files). The last three are `null` when the target is not a git repository. `report.py init` records the path resolved to an absolute one, so that later calls from another directory find the target; anonymise it before sharing a report if the repository is private. | | `started_at`, `finalised_at` | ISO 8601 with an offset. `finalised_at` is set with `status: final`. | +`framework.install` is `checkout` for a git checkout of the framework repository, and `bundle` for an installed +skill bundle, with `SKILL.md` beside the tools' `scripts/` folder. It is `copy` for anything else, such as a source +archive with no `.git` folder. + ### `run_context` The intake answers, as the run used them. [`report.py init`](#intake-file) copies them in. @@ -379,8 +390,8 @@ Every field of the engine's schema v2 is either carried or named as not carried. | `schema_version` | Not carried. The adapter accepts `"2"` and refuses any other value. | | `provenance.tool`, `version`, `ruleset_revision`, `ruleset_hash`, `scanned_at` | `provenance.engine.name`, `version`, `ruleset_revision`, `ruleset_hash`, `scanned_at` | | `provenance.target`, `git_sha`, `git_branch`, `git_dirty` | Not copied. `init` recorded the target's facts from the directory itself, and the draft keeps them. The adapter requires the engine's `target` to be the same directory as the draft's, and its `git_sha` to name the same commit as the draft's (the draft keeps 12 digits, the engine reports all 40) when both are known, and refuses a report that differs in either. A commit that only one side knows is a warning. | -| `provenance.framework_revision` | Not carried. The report records its own `provenance.framework_revision`. | -| `provenance.phase` | Not carried. A 0.1 report covers Phase 1; the adapter refuses an engine report for another phase. | +| `provenance.framework_revision` | Not carried. The report records its own `provenance.framework`. | +| `provenance.phase` | Not carried. An audit report covers Phase 1; the adapter refuses an engine report for another phase. | | `provenance.probes_enabled` | Not carried. `run_context.probes.allowed` says it. | | `summary.phase1_score` | `summary.phase1_score`, recomputed from the checks; the validator rejects a mismatch. The adapter compares the engine's number with the recomputed one and warns when they differ, and keeps the recomputed one. | | `summary.checks_scoreable` | `summary.scored_checks` | diff --git a/audit/schema/audit-report.schema.json b/audit/schema/audit-report.schema.json index 7158c98d..1850888d 100644 --- a/audit/schema/audit-report.schema.json +++ b/audit/schema/audit-report.schema.json @@ -20,8 +20,8 @@ "additionalProperties": false, "properties": { "schema_version": { - "description": "Version of this schema. 0.x versions may break between releases; consumers refuse a version they do not know.", - "const": "0.1" + "description": "Version of this schema. 0.x versions may break between releases; consumers refuse a version they do not know. 0.2 replaced provenance.recipe and provenance.framework_revision with provenance.framework; a 0.1 report is not read (no backward compatibility before the first outside user or 1.0.0).", + "const": "0.2" }, "status": { "description": "draft until every finding has a disposition and the report is validated; set to final only by `report.py finalise`.", @@ -29,23 +29,31 @@ }, "provenance": { "type": "object", - "required": ["recipe", "framework_revision", "catalogue_version", "agent", "target", "started_at"], + "required": ["framework", "catalogue_version", "agent", "target", "started_at"], "additionalProperties": false, "properties": { - "recipe": { + "framework": { + "description": "The framework that made the report: which release, which commit, and how it was installed.", "type": "object", - "required": ["name", "version"], + "required": ["release", "revision", "install"], "additionalProperties": false, "properties": { - "name": { "type": "string", "minLength": 1, "maxLength": 100 }, - "version": { "type": "string", "minLength": 1, "maxLength": 40 } + "release": { + "description": "The framework release version (.release-please-manifest.json) of the tools that wrote the report.", + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+(-[0-9A-Za-z.-]+)?$" + }, + "revision": { + "description": "Short Git SHA (12 hex digits) of the framework checkout the tools ran from; null when install is not checkout, or git could not read it. A full id is not written because secret scanners read it as an access token.", + "type": ["string", "null"], + "pattern": "^[0-9a-f]{12}$" + }, + "install": { + "description": "checkout: a git checkout of the framework repository. bundle: an installed skill bundle (a SKILL.md beside the tools' scripts/ folder). copy: anything else, such as a source archive without .git.", + "enum": ["checkout", "bundle", "copy"] + } } }, - "framework_revision": { - "description": "Short Git SHA (12 hex digits) of the framework repository the recipe ran from; null when it was not a git checkout. A full 40-digit id is also accepted so earlier reports stay valid; the tools write the short form because a full id is what secret scanners read as an access token.", - "type": ["string", "null"], - "pattern": "^([0-9a-f]{12}|[0-9a-f]{40})$" - }, "catalogue_version": { "description": "Version of the check catalogue the report was built against: MAJOR.MINOR.PATCH, as in catalogue.json.", "type": "string", diff --git a/audit/schema/examples/example-service.audit-report.20261010T094207Z.json b/audit/schema/examples/example-service.audit-report.20261010T094207Z.json index 3b299726..a27464e8 100644 --- a/audit/schema/examples/example-service.audit-report.20261010T094207Z.json +++ b/audit/schema/examples/example-service.audit-report.20261010T094207Z.json @@ -1,12 +1,12 @@ { - "schema_version": "0.1", + "schema_version": "0.2", "status": "final", "provenance": { - "recipe": { - "name": "brownfield-audit", - "version": "0.1.0" + "framework": { + "release": "0.3.2", + "revision": "cafecafecafe", + "install": "checkout" }, - "framework_revision": "cafecafecafecafecafecafecafecafecafecafe", "catalogue_version": "0.4.0", "engine": { "name": "python-agentic-audit", diff --git a/audit/schema/examples/example-service.audit-report.20261010T094207Z.md b/audit/schema/examples/example-service.audit-report.20261010T094207Z.md index b8b06cd6..15584b37 100644 --- a/audit/schema/examples/example-service.audit-report.20261010T094207Z.md +++ b/audit/schema/examples/example-service.audit-report.20261010T094207Z.md @@ -7,17 +7,17 @@ | Field | Value | |---|---| +| Framework | `v0.3.2`, checkout `cafecafecafe` | | Target | `/work/example-service` | | Commit | `deadbeefdead` on `main` — clean working tree | | Status | final | | Started | `2026-10-10T09:00:00+00:00` | | Finalised | `2026-10-10T09:42:07+00:00` | -| Recipe | `brownfield-audit` 0.1.0, catalogue `0.4.0` | +| Catalogue | `0.4.0` | | Engine | `python-agentic-audit` 0.1.0, ruleset `2026-09-28.13`, ruleset hash `a1b2a1b2a1b2a1b2`, scanned `2026-10-10T09:01:30+00:00` | -| Framework revision | `cafecafecafe` | | Agent (self-reported) | `claude-code` with `claude-sonnet-5-5` | | Probes | none: no command of the target was run | -| Report schema | `0.1` | +| Report schema | `0.2` | ## Readiness summary diff --git a/audit/schema/examples/minimal.json b/audit/schema/examples/minimal.json index ac9b0c28..ebc34bfa 100644 --- a/audit/schema/examples/minimal.json +++ b/audit/schema/examples/minimal.json @@ -1,12 +1,12 @@ { - "schema_version": "0.1", + "schema_version": "0.2", "status": "draft", "provenance": { - "recipe": { - "name": "brownfield-audit", - "version": "0.1.0" + "framework": { + "release": "0.3.2", + "revision": null, + "install": "bundle" }, - "framework_revision": null, "catalogue_version": "0.4.0", "agent": { "harness": "claude-code", diff --git a/audit/schema/examples/minimal.md b/audit/schema/examples/minimal.md index 6881108c..9ed61175 100644 --- a/audit/schema/examples/minimal.md +++ b/audit/schema/examples/minimal.md @@ -10,17 +10,17 @@ | Field | Value | |---|---| +| Framework | `v0.3.2`, installed as a bundle | | Target | `/work/example-service` | | Commit | none recorded | | Status | draft | | Started | `2026-10-10T09:00:00+00:00` | | Finalised | — | -| Recipe | `brownfield-audit` 0.1.0, catalogue `0.4.0` | +| Catalogue | `0.4.0` | | Engine | No engine report is part of this report. | -| Framework revision | — | | Agent (self-reported) | `claude-code` with `claude-sonnet-5-5` | | Probes | none: no command of the target was run | -| Report schema | `0.1` | +| Report schema | `0.2` | ## Readiness summary diff --git a/audit/schema/examples/readiness-state.example-service.in-progress.json b/audit/schema/examples/readiness-state.example-service.in-progress.json index 0c7997a9..b52980e5 100644 --- a/audit/schema/examples/readiness-state.example-service.in-progress.json +++ b/audit/schema/examples/readiness-state.example-service.in-progress.json @@ -14,7 +14,7 @@ "path": "docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json", "git_sha": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", "catalogue_version": "0.4.0", - "sha256": "7144f9fe6f688dd9b72315249a632ff6101fd9a24b5ef6ff715d7c9c8a8162c9" + "sha256": "51e8e6afb0125d46d0fa12d60657929f89ba8ffde27894a59366aacdd0b94b98" } ], "preferences": [ diff --git a/audit/schema/examples/readiness-state.example-service.json b/audit/schema/examples/readiness-state.example-service.json index fc81735a..6a20360e 100644 --- a/audit/schema/examples/readiness-state.example-service.json +++ b/audit/schema/examples/readiness-state.example-service.json @@ -14,7 +14,7 @@ "path": "docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json", "git_sha": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", "catalogue_version": "0.4.0", - "sha256": "7144f9fe6f688dd9b72315249a632ff6101fd9a24b5ef6ff715d7c9c8a8162c9" + "sha256": "51e8e6afb0125d46d0fa12d60657929f89ba8ffde27894a59366aacdd0b94b98" } ], "preferences": [], diff --git a/audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json b/audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json index e5aed3f0..0b5bffa4 100644 --- a/audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json +++ b/audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json @@ -1,12 +1,12 @@ { - "schema_version": "0.1", + "schema_version": "0.2", "status": "draft", "provenance": { - "recipe": { - "name": "brownfield-audit", - "version": "0.1.0" + "framework": { + "release": "0.3.2", + "revision": "7eedf23c99c3", + "install": "checkout" }, - "framework_revision": "7eedf23c99c39f94cad435e8a52eb28642308a92", "catalogue_version": "0.4.0", "engine": { "name": "python-agentic-audit", diff --git a/audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json b/audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json index 4b1ec644..4361fa0a 100644 --- a/audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json +++ b/audit/schema/examples/realistic/digital-assistant-sdk.audit-draft.20261010T065739Z.json @@ -1,12 +1,12 @@ { - "schema_version": "0.1", + "schema_version": "0.2", "status": "draft", "provenance": { - "recipe": { - "name": "brownfield-audit", - "version": "0.1.0" + "framework": { + "release": "0.3.2", + "revision": "7eedf23c99c3", + "install": "checkout" }, - "framework_revision": "7eedf23c99c39f94cad435e8a52eb28642308a92", "catalogue_version": "0.4.0", "engine": { "name": "python-agentic-audit", diff --git a/audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json b/audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json index 8fbfbaa8..392537ca 100644 --- a/audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json +++ b/audit/schema/examples/realistic/digital-assistant-server.audit-draft.20261010T065739Z.json @@ -1,12 +1,12 @@ { - "schema_version": "0.1", + "schema_version": "0.2", "status": "draft", "provenance": { - "recipe": { - "name": "brownfield-audit", - "version": "0.1.0" + "framework": { + "release": "0.3.2", + "revision": "7eedf23c99c3", + "install": "checkout" }, - "framework_revision": "7eedf23c99c39f94cad435e8a52eb28642308a92", "catalogue_version": "0.4.0", "engine": { "name": "python-agentic-audit", diff --git a/audit/schema/examples/server-worked-example/digital-assistant-server-first-audit-20261028T131409Z.json b/audit/schema/examples/server-worked-example/digital-assistant-server-first-audit-20261028T131409Z.json index a5c665db..a818b963 100644 --- a/audit/schema/examples/server-worked-example/digital-assistant-server-first-audit-20261028T131409Z.json +++ b/audit/schema/examples/server-worked-example/digital-assistant-server-first-audit-20261028T131409Z.json @@ -1,12 +1,12 @@ { - "schema_version": "0.1", + "schema_version": "0.2", "status": "final", "provenance": { - "recipe": { - "name": "brownfield-audit", - "version": "0.1.0" + "framework": { + "release": "0.3.2", + "revision": "b4bf27df7915", + "install": "checkout" }, - "framework_revision": "b4bf27df7915d5ec7d79ced607ac03a5567adc7d", "catalogue_version": "0.2.2", "engine": { "name": "python-agentic-audit", diff --git a/audit/schema/examples/server-worked-example/digital-assistant-server-first-audit-20261028T131409Z.md b/audit/schema/examples/server-worked-example/digital-assistant-server-first-audit-20261028T131409Z.md index d90ad591..6f72f7ca 100644 --- a/audit/schema/examples/server-worked-example/digital-assistant-server-first-audit-20261028T131409Z.md +++ b/audit/schema/examples/server-worked-example/digital-assistant-server-first-audit-20261028T131409Z.md @@ -7,17 +7,17 @@ | Field | Value | |---|---| +| Framework | `v0.3.2`, checkout `b4bf27df7915` | | Target | `` | | Commit | `4a48e1976e42` on `dev` — clean working tree | | Status | final | | Started | `2026-10-28T12:49:11+00:00` | | Finalised | `2026-10-28T13:14:09+00:00` | -| Recipe | `brownfield-audit` 0.1.0, catalogue `0.2.2` | +| Catalogue | `0.2.2` | | Engine | `python-agentic-audit` 0.1.0, ruleset `2026-10-19.2`, ruleset hash `970e6792f98eeeb5`, scanned `2026-10-28T12:49:15+00:00` | -| Framework revision | `b4bf27df7915` | | Agent (self-reported) | `Claude Code` with `claude-sonnet-5-5` | | Probes | none: no command of the target was run | -| Report schema | `0.1` | +| Report schema | `0.2` | ## Readiness summary @@ -276,8 +276,8 @@ All 71 checks, grouped by family. The command surface, the credential-shaped-lin | Check | Title | Severity | Method | Scored | Verdict | Engine | Summary | |---|---|---|---|---|---|---|---| -| `IDX-01` | codebase-memory-mcp registered for this repository | BLOCKER | automated | yes | PASS | PASS | codebase-memory-mcp registered in: /.config/opencode/opencode.jsonc (user) | -| `IDX-02` | An index exists for this checkout | BLOCKER | automated | yes | PASS | PASS | index store /.cache/codebase-memory-mcp contains a project rooted at | +| `IDX-01` | codebase-memory-mcp registered for this repository | BLOCKER | automated | yes | PASS | PASS | codebase-memory-mcp registered in: \/.config/opencode/opencode.jsonc (user) | +| `IDX-02` | An index exists for this checkout | BLOCKER | automated | yes | PASS | PASS | index store \/.cache/codebase-memory-mcp contains a project rooted at \ | | `IDX-03` | Index freshness | DEGRADER | automated | yes | PARTIAL | UNKNOWN | Index built 2026-10-28T09:43:48Z; HEAD 4a48e19 (committed 11:27Z) is the one later commit, touching only a workflow, a CI script and a CI README. | ### AGT diff --git a/audit/schema/examples/server-worked-example/digital-assistant-server.audit-report.20261106T123400Z.json b/audit/schema/examples/server-worked-example/digital-assistant-server.audit-report.20261106T123400Z.json index b61df745..7babf45b 100644 --- a/audit/schema/examples/server-worked-example/digital-assistant-server.audit-report.20261106T123400Z.json +++ b/audit/schema/examples/server-worked-example/digital-assistant-server.audit-report.20261106T123400Z.json @@ -1,12 +1,12 @@ { - "schema_version": "0.1", + "schema_version": "0.2", "status": "final", "provenance": { - "recipe": { - "name": "brownfield-audit", - "version": "0.1.0" + "framework": { + "release": "0.3.2", + "revision": "c132d974bdc5", + "install": "checkout" }, - "framework_revision": "c132d974bdc53c333963c1c8053734b7ba96e5f1", "catalogue_version": "0.2.8", "engine": { "name": "python-agentic-audit", diff --git a/audit/schema/examples/server-worked-example/digital-assistant-server.audit-report.20261106T123400Z.md b/audit/schema/examples/server-worked-example/digital-assistant-server.audit-report.20261106T123400Z.md index fbee8c22..8ff02a10 100644 --- a/audit/schema/examples/server-worked-example/digital-assistant-server.audit-report.20261106T123400Z.md +++ b/audit/schema/examples/server-worked-example/digital-assistant-server.audit-report.20261106T123400Z.md @@ -7,17 +7,17 @@ | Field | Value | |---|---| +| Framework | `v0.3.2`, checkout `c132d974bdc5` | | Target | `` | | Commit | `ffce29f9c23a` — clean working tree | | Status | final | | Started | `2026-11-06T09:17:56+00:00` | | Finalised | `2026-11-06T12:34:00+00:00` | -| Recipe | `brownfield-audit` 0.1.0, catalogue `0.2.8` | +| Catalogue | `0.2.8` | | Engine | `python-agentic-audit` 0.1.0, ruleset `2026-11-06.1`, ruleset hash `970e6792f98eeeb5`, scanned `2026-11-06T09:18:01+00:00` | -| Framework revision | `c132d974bdc5` | | Agent (self-reported) | `Claude Code` with `claude-sonnet-5-5` | | Probes | none: no command of the target was run | -| Report schema | `0.1` | +| Report schema | `0.2` | ## Readiness summary diff --git a/audit/tools/_engine_adapter.py b/audit/tools/_engine_adapter.py index e58c2384..ba1a262e 100644 --- a/audit/tools/_engine_adapter.py +++ b/audit/tools/_engine_adapter.py @@ -345,7 +345,7 @@ def check_engine_report(engine: Any) -> None: phase = engine["provenance"]["phase"] if phase != PHASE: refuse( - f"the engine report is for phase {phase}, and a 0.1 report covers phase {PHASE}", + f"the engine report is for phase {phase}, and an audit report covers phase {PHASE}", [f"engine report: /provenance/phase: expected {PHASE}, found {phase}"], ) diff --git a/audit/tools/render_report.py b/audit/tools/render_report.py index 21328ecd..3a7e01e0 100644 --- a/audit/tools/render_report.py +++ b/audit/tools/render_report.py @@ -408,12 +408,24 @@ def agent_text(agent: Mapping[str, Any]) -> str: return f"{code(agent['harness'])} with {code(agent['model'])}" +def framework_text(framework: Mapping[str, Any]) -> str: + """The release, then how it was installed: `v0.3.2, checkout `fcd5c7fcbff1``, `v0.3.2, installed as a bundle`.""" + release = code(f"v{framework['release']}") + if framework["install"] == "checkout": + revision = framework["revision"] + return ( + f"{release}, checkout {code(revision[:COMMIT_CHARS])}" if revision else f"{release}, checkout (no revision)" + ) + if framework["install"] == "bundle": + return f"{release}, installed as a bundle" + return f"{release}, a copy with no git history" + + def provenance_section(report: Mapping[str, Any]) -> list[str]: """What was audited, with what, and when: the first thing a reader sees after the title.""" provenance = report["provenance"] target = provenance["target"] engine = provenance.get("engine") - revision = provenance["framework_revision"] probes = report["run_context"]["probes"] if target["git_sha"]: commit = code(target["git_sha"][:COMMIT_CHARS]) @@ -423,18 +435,14 @@ def provenance_section(report: Mapping[str, Any]) -> list[str]: else: commit = "none recorded" rows = [ + ["Framework", framework_text(provenance["framework"])], ["Target", code(target["path"])], ["Commit", commit], ["Status", report["status"] + (" (evaluation run)" if report["run_context"]["eval_mode"] else "")], ["Started", code(provenance["started_at"])], ["Finalised", code(provenance["finalised_at"]) if "finalised_at" in provenance else "—"], - [ - "Recipe", - f"{code(provenance['recipe']['name'])} {text(provenance['recipe']['version'])}, " - f"catalogue {code(provenance['catalogue_version'])}", - ], + ["Catalogue", code(provenance["catalogue_version"])], ["Engine", engine_text(engine) if engine else "No engine report is part of this report."], - ["Framework revision", code(revision[:COMMIT_CHARS]) if revision else "—"], ["Agent (self-reported)", agent_text(provenance["agent"])], [ "Probes", diff --git a/audit/tools/report.py b/audit/tools/report.py index e538c4c2..90e6c9ae 100644 --- a/audit/tools/report.py +++ b/audit/tools/report.py @@ -83,9 +83,9 @@ ), ) -RECIPE_NAME = "brownfield-audit" -RECIPE_VERSION = "0.3.2" # x-release-please-version -SCHEMA_VERSION = "0.1" +#: The framework release these tools belong to: `provenance.framework.release`. release-please rewrites the line. +FRAMEWORK_RELEASE = "0.3.2" # x-release-please-version +SCHEMA_VERSION = "0.2" DRAFT_KIND = "audit-draft" DEFAULT_PROBE_TIMEOUT_S = 120 FINAL_KIND = "audit-report" @@ -173,10 +173,26 @@ def same_commit(recorded: str, scanned: str) -> bool: return scanned.startswith(recorded) or recorded.startswith(scanned) -def framework_revision() -> str | None: - """The commit of the framework repository these tools run from, or None when they are not in a checkout of it.""" - root = vr.RECIPES.parent.resolve() - if shutil.which("git") is None or git_toplevel(root) != root: +def framework_install(root: Path) -> str: + """How the framework at `root` is installed: `checkout`, `bundle` or `copy` (`provenance.framework.install`). + + `root` is the folder above the recipe folder: the repository root in a checkout, the skill folder in a bundle, + where `SKILL.md` sits beside `scripts/`. A checkout is a git working tree whose top is `root`, or, when git is + missing, a `root` that holds `.git`. + """ + if shutil.which("git") is not None: + if git_toplevel(root) == root: + return "checkout" + elif (root / ".git").exists(): + return "checkout" + if (root / "SKILL.md").is_file(): + return "bundle" + return "copy" + + +def framework_revision(root: Path) -> str | None: + """The short commit of the framework checkout at `root`, or None when git cannot read one.""" + if shutil.which("git") is None: return None head = vr.git_output(root, ["rev-parse", "HEAD"]) if head is None or head.returncode != 0 or not GIT_SHA.match(head.stdout.strip()): @@ -184,6 +200,14 @@ def framework_revision() -> str | None: return head.stdout.strip()[:SHORT_SHA_CHARS] +def framework_facts(root: Path | None = None) -> dict[str, Any]: + """`provenance.framework`: the release, the commit (checkout only) and the install of the framework at `root`.""" + root = (root or vr.RECIPES.parent).resolve() + install = framework_install(root) + revision = framework_revision(root) if install == "checkout" else None + return {"release": FRAMEWORK_RELEASE, "revision": revision, "install": install} + + # --- The intake --------------------------------------------------------------------------------------- @@ -240,7 +264,7 @@ def new_draft( target: Path, git: Mapping[str, Any], catalogue_version: str, - framework: str | None, + framework: Mapping[str, Any], started_at: str, ) -> dict[str, Any]: """An empty draft: provenance and run context from the intake, no checks, no findings. @@ -253,8 +277,7 @@ def new_draft( "schema_version": SCHEMA_VERSION, "status": "draft", "provenance": { - "recipe": {"name": RECIPE_NAME, "version": RECIPE_VERSION}, - "framework_revision": framework, + "framework": dict(framework), "catalogue_version": catalogue_version, "agent": {"harness": intake["harness"], "model": intake["model"]}, "target": { @@ -306,7 +329,7 @@ def run_init(args: argparse.Namespace) -> _common.Result: target=target, git=git, catalogue_version=str(vr.load_catalogue()["catalogue_version"]), - framework=framework_revision(), + framework=framework_facts(), started_at=moment.isoformat(), ) outcome = vr.validate_document(document) diff --git a/audit/tools/tests/test_engine_adapter.py b/audit/tools/tests/test_engine_adapter.py index 80fb9a29..723dba33 100644 --- a/audit/tools/tests/test_engine_adapter.py +++ b/audit/tools/tests/test_engine_adapter.py @@ -1341,7 +1341,7 @@ def test_a_schema_version_other_than_2(self) -> None: def test_a_phase_other_than_1(self) -> None: error = refused(self, er.report(TARGET, provenance=er.provenance(TARGET, phase=2))) - self.assertEqual(str(error), "the engine report is for phase 2, and a 0.1 report covers phase 1") + self.assertEqual(str(error), "the engine report is for phase 2, and an audit report covers phase 1") self.assertEqual(error.errors, ["engine report: /provenance/phase: expected 1, found 2"]) def test_the_wrong_shape_with_one_error_and_with_two(self) -> None: diff --git a/audit/tools/tests/test_import_engine.py b/audit/tools/tests/test_import_engine.py index abd7664e..a90a6d11 100644 --- a/audit/tools/tests/test_import_engine.py +++ b/audit/tools/tests/test_import_engine.py @@ -86,7 +86,7 @@ def test_the_draft_records_the_engine_in_the_order_of_the_schema(self) -> None: ) self.assertEqual( list(provenance), - ["recipe", "framework_revision", "catalogue_version", "engine", "agent", "target", "started_at"], + ["framework", "catalogue_version", "engine", "agent", "target", "started_at"], ) def test_the_draft_stays_valid_and_keeps_the_target_it_was_given(self) -> None: diff --git a/audit/tools/tests/test_render_report.py b/audit/tools/tests/test_render_report.py index d2221737..4c726a06 100644 --- a/audit/tools/tests/test_render_report.py +++ b/audit/tools/tests/test_render_report.py @@ -471,14 +471,14 @@ def test_a_requirement_is_listed_once_in_the_summary_section_only(self) -> None: class ProvenanceTests(RenderCase): FIELDS = [ + "Framework", "Target", "Commit", "Status", "Started", "Finalised", - "Recipe", + "Catalogue", "Engine", - "Framework revision", "Agent (self-reported)", "Probes", "Report schema", @@ -508,16 +508,16 @@ def test_the_worked_example_says_what_was_audited_with_what_and_when(self) -> No self.assertEqual(values["Status"], "final") self.assertEqual(values["Started"], "`2026-10-10T09:00:00+00:00`") self.assertEqual(values["Finalised"], "`2026-10-10T09:42:07+00:00`") - self.assertEqual(values["Recipe"], "`brownfield-audit` 0.1.0, catalogue `0.4.0`") + self.assertEqual(values["Framework"], "`v0.3.2`, checkout `cafecafecafe`") + self.assertEqual(values["Catalogue"], "`0.4.0`") self.assertEqual( values["Engine"], "`python-agentic-audit` 0.1.0, ruleset `2026-09-28.13`, ruleset hash `a1b2a1b2a1b2a1b2`, " "scanned `2026-10-10T09:01:30+00:00`", ) - self.assertEqual(values["Framework revision"], "`cafecafecafe`") self.assertEqual(values["Agent (self-reported)"], "`claude-code` with `claude-sonnet-5-5`") self.assertEqual(values["Probes"], "none: no command of the target was run") - self.assertEqual(values["Report schema"], "`0.1`") + self.assertEqual(values["Report schema"], "`0.2`") def test_a_dirty_working_tree_is_in_bold_and_an_unknown_one_says_nothing(self) -> None: report = service() @@ -558,10 +558,18 @@ def test_probes_that_may_run_are_named_with_their_timeout(self) -> None: report["run_context"]["probes"] = {"allowed": ["test", "lint"], "timeout_s": 90} self.assertEqual(self.provenance(report)["Probes"], "`test`, `lint`, timeout 90 s") - def test_a_report_with_no_engine_or_framework_revision_says_so(self) -> None: + def test_a_report_with_no_engine_from_a_bundle_says_so(self) -> None: values = self.provenance(minimal()) self.assertEqual(values["Engine"], "No engine report is part of this report.") - self.assertEqual(values["Framework revision"], "—") + self.assertEqual(values["Framework"], "`v0.3.2`, installed as a bundle") + + def test_the_framework_row_says_how_the_framework_was_installed(self) -> None: + report = service() + framework = report["provenance"]["framework"] + framework.update(revision=None) + self.assertEqual(self.provenance(report)["Framework"], "`v0.3.2`, checkout (no revision)") + framework.update(install="copy") + self.assertEqual(self.provenance(report)["Framework"], "`v0.3.2`, a copy with no git history") def test_the_engine_row_names_only_what_the_report_records(self) -> None: report = service() diff --git a/audit/tools/tests/test_report.py b/audit/tools/tests/test_report.py index b8db8e0b..95ff4593 100644 --- a/audit/tools/tests/test_report.py +++ b/audit/tools/tests/test_report.py @@ -232,7 +232,7 @@ def test_it_creates_a_named_draft_that_validates(self) -> None: self.assertEqual(self.drafts(), [path.name]) document = read_json(path) self.assertEqual(vr.validate_document(document).errors, []) - self.assertEqual((document["schema_version"], document["status"]), ("0.1", "draft")) + self.assertEqual((document["schema_version"], document["status"]), ("0.2", "draft")) self.assertEqual((document["checks"], document["findings"], document["components"]), ([], [], [])) self.assertEqual(document["summary"], _summary.compute_summary(document)) self.assertEqual(stderr, f"report: started {path.name} for my-service\n") @@ -290,7 +290,8 @@ def test_a_label_that_is_not_allowed_is_an_intake_error_and_writes_nothing(self) def test_the_provenance_is_what_the_intake_and_the_catalogue_say(self) -> None: document = read_json(self.start()) provenance = document["provenance"] - self.assertEqual(provenance["recipe"], {"name": "brownfield-audit", "version": rb.RECIPE_VERSION}) + self.assertEqual(provenance["framework"], rb.framework_facts()) + self.assertEqual(provenance["framework"]["release"], rb.FRAMEWORK_RELEASE) self.assertEqual(provenance["agent"], {"harness": "claude-code", "model": "claude-sonnet-5-5"}) self.assertEqual(provenance["catalogue_version"], vr.load_catalogue()["catalogue_version"]) self.assertEqual(provenance["started_at"], "2026-10-10T09:30:00+00:00") @@ -332,15 +333,54 @@ def test_an_intake_without_the_optional_answers_gets_the_defaults(self) -> None: self.assertEqual((context["excluded"], context["known_components"]), ([], [])) def test_the_framework_revision_is_this_checkout_or_nothing(self) -> None: - document = read_json(self.start()) - recorded = document["provenance"]["framework_revision"] + recorded = read_json(self.start())["provenance"]["framework"] root = vr.RECIPES.parent if rb.git_toplevel(root) == root.resolve(): - self.assertEqual(recorded, _git.run_git(root, "rev-parse", "HEAD")[: rb.SHORT_SHA_CHARS]) + self.assertEqual(recorded["install"], "checkout") + self.assertEqual(recorded["revision"], _git.run_git(root, "rev-parse", "HEAD")[: rb.SHORT_SHA_CHARS]) else: - self.assertIsNone(recorded) + self.assertIsNone(recorded["revision"]) with mock.patch.object(rb, "git_toplevel", return_value=self.root): - self.assertIsNone(rb.framework_revision()) + self.assertIsNone(rb.framework_facts()["revision"]) + + def test_a_framework_checkout_records_its_short_commit(self) -> None: + root = self.root / "framework" + _git.make_repo(root) + head = _git.run_git(root, "rev-parse", "HEAD") + self.assertEqual( + rb.framework_facts(root), + {"release": rb.FRAMEWORK_RELEASE, "revision": head[: rb.SHORT_SHA_CHARS], "install": "checkout"}, + ) + + def test_a_skill_md_beside_the_scripts_is_a_bundle_with_no_revision(self) -> None: + root = self.root / "agentic-readiness-audit" + (root / "scripts").mkdir(parents=True) + (root / "SKILL.md").write_text("---\nname: agentic-readiness-audit\n---\n", encoding="utf-8") + self.assertEqual( + rb.framework_facts(root), {"release": rb.FRAMEWORK_RELEASE, "revision": None, "install": "bundle"} + ) + + def test_a_bundle_inside_another_repository_is_still_a_bundle(self) -> None: + _git.make_repo(self.root / "project") + root = self.root / "project" / ".claude" / "skills" / "agentic-readiness-audit" + root.mkdir(parents=True) + (root / "SKILL.md").write_text("---\nname: agentic-readiness-audit\n---\n", encoding="utf-8") + self.assertEqual(rb.framework_facts(root)["install"], "bundle") + + def test_anything_else_is_a_copy_with_no_revision(self) -> None: + root = self.root / "source-archive" + root.mkdir() + self.assertEqual( + rb.framework_facts(root), {"release": rb.FRAMEWORK_RELEASE, "revision": None, "install": "copy"} + ) + + def test_without_git_a_dot_git_folder_is_a_checkout_with_no_revision(self) -> None: + root = self.root / "framework" + (root / ".git").mkdir(parents=True) + with mock.patch.object(shutil, "which", return_value=None): + self.assertEqual( + rb.framework_facts(root), {"release": rb.FRAMEWORK_RELEASE, "revision": None, "install": "checkout"} + ) class TargetFactsTests(BuilderCase): diff --git a/audit/tools/tests/test_validate_report.py b/audit/tools/tests/test_validate_report.py index 49f741c4..9fb0d028 100644 --- a/audit/tools/tests/test_validate_report.py +++ b/audit/tools/tests/test_validate_report.py @@ -110,7 +110,7 @@ def test_a_document_that_is_not_an_object_is_a_schema_error(self) -> None: def test_a_report_with_a_version_this_tool_does_not_know_is_refused(self) -> None: report = base() - report["schema_version"] = "0.2" + report["schema_version"] = "0.1" self.assertTrue(any("/schema_version" in error for error in vr.validate_document(report).errors)) def test_the_schema_errors_are_capped(self) -> None: @@ -802,7 +802,7 @@ def test_a_valid_report_is_exit_0_with_a_short_summary(self) -> None: { "valid": True, "report": str(WORKED), - "schema_version": "0.1", + "schema_version": "0.2", "status": "final", "checks": 12, "findings": 10, diff --git a/dev/eval/implement-fixtures/testdata/reports/base/stockroom-python-audit-20261106T085315Z.json b/dev/eval/implement-fixtures/testdata/reports/base/stockroom-python-audit-20261106T085315Z.json index 61d13f91..52af060d 100644 --- a/dev/eval/implement-fixtures/testdata/reports/base/stockroom-python-audit-20261106T085315Z.json +++ b/dev/eval/implement-fixtures/testdata/reports/base/stockroom-python-audit-20261106T085315Z.json @@ -1,12 +1,12 @@ { - "schema_version": "0.1", + "schema_version": "0.2", "status": "final", "provenance": { - "recipe": { - "name": "brownfield-audit", - "version": "0.1.0" + "framework": { + "release": "0.3.2", + "revision": "ee5da5b99005", + "install": "checkout" }, - "framework_revision": "ee5da5b99005ca561d3f71fe1a69f9b4e505ab19", "catalogue_version": "0.2.8", "engine": { "name": "python-agentic-audit", diff --git a/dev/eval/implement-fixtures/testdata/reports/base/stockroom-python-audit-20261106T085315Z.md b/dev/eval/implement-fixtures/testdata/reports/base/stockroom-python-audit-20261106T085315Z.md index ec1dd8c4..94f60475 100644 --- a/dev/eval/implement-fixtures/testdata/reports/base/stockroom-python-audit-20261106T085315Z.md +++ b/dev/eval/implement-fixtures/testdata/reports/base/stockroom-python-audit-20261106T085315Z.md @@ -7,17 +7,17 @@ | Field | Value | |---|---| +| Framework | `v0.3.2`, checkout `ee5da5b99005` | | Target | `stockroom` | | Commit | `091b85c59f6a` on `main` — clean working tree | | Status | final | | Started | `2026-11-06T08:53:11+00:00` | | Finalised | `2026-11-06T08:53:15+00:00` | -| Recipe | `brownfield-audit` 0.1.0, catalogue `0.2.8` | +| Catalogue | `0.2.8` | | Engine | `python-agentic-audit` 0.1.0, ruleset `2026-11-06.1`, ruleset hash `970e6792f98eeeb5`, scanned `2026-11-06T08:53:11+00:00` | -| Framework revision | `ee5da5b99005` | | Agent (self-reported) | none: no agent took part | | Probes | none: no command of the target was run | -| Report schema | `0.1` | +| Report schema | `0.2` | ## Readiness summary diff --git a/dev/eval/implement-fixtures/testdata/reports/planted/stockroom-python-audit-20261106T085315Z.json b/dev/eval/implement-fixtures/testdata/reports/planted/stockroom-python-audit-20261106T085315Z.json index 91c766bf..612bb58c 100644 --- a/dev/eval/implement-fixtures/testdata/reports/planted/stockroom-python-audit-20261106T085315Z.json +++ b/dev/eval/implement-fixtures/testdata/reports/planted/stockroom-python-audit-20261106T085315Z.json @@ -1,12 +1,12 @@ { - "schema_version": "0.1", + "schema_version": "0.2", "status": "final", "provenance": { - "recipe": { - "name": "brownfield-audit", - "version": "0.1.0" + "framework": { + "release": "0.3.2", + "revision": "ee5da5b99005", + "install": "checkout" }, - "framework_revision": "ee5da5b99005ca561d3f71fe1a69f9b4e505ab19", "catalogue_version": "0.2.8", "engine": { "name": "python-agentic-audit", diff --git a/dev/eval/implement-fixtures/testdata/reports/planted/stockroom-python-audit-20261106T085315Z.md b/dev/eval/implement-fixtures/testdata/reports/planted/stockroom-python-audit-20261106T085315Z.md index 27db79c0..67d23cfb 100644 --- a/dev/eval/implement-fixtures/testdata/reports/planted/stockroom-python-audit-20261106T085315Z.md +++ b/dev/eval/implement-fixtures/testdata/reports/planted/stockroom-python-audit-20261106T085315Z.md @@ -7,17 +7,17 @@ | Field | Value | |---|---| +| Framework | `v0.3.2`, checkout `ee5da5b99005` | | Target | `stockroom` | | Commit | `091b85c59f6a` on `main` — clean working tree | | Status | final | | Started | `2026-11-06T08:53:15+00:00` | | Finalised | `2026-11-06T08:53:19+00:00` | -| Recipe | `brownfield-audit` 0.1.0, catalogue `0.2.8` | +| Catalogue | `0.2.8` | | Engine | `python-agentic-audit` 0.1.0, ruleset `2026-11-06.1`, ruleset hash `970e6792f98eeeb5`, scanned `2026-11-06T08:53:11+00:00` | -| Framework revision | `ee5da5b99005` | | Agent (self-reported) | none: no agent took part | | Probes | none: no command of the target was run | -| Report schema | `0.1` | +| Report schema | `0.2` | ## Readiness summary diff --git a/dev/tests/test_release_config.py b/dev/tests/test_release_config.py index 619ec054..387d086c 100644 --- a/dev/tests/test_release_config.py +++ b/dev/tests/test_release_config.py @@ -9,8 +9,8 @@ marker that release-please looks for or is not an ``extra-files`` entry, or when a hand-written ``[Unreleased]`` heading comes back into the changelog that release-please writes. -The stamps are the engine's ``__version__`` and the two recipes' ``RECIPE_VERSION`` (the audit report's -``provenance.recipe.version`` and the implement state's ``recipe.version``). The skill bundles' ``metadata.version`` +The stamps are the engine's ``__version__``, ``FRAMEWORK_RELEASE`` in ``report.py`` (the audit report's +``provenance.framework.release``) and ``RECIPE_VERSION`` in ``progress.py`` (the implement state's ``recipe.version``). The skill bundles' ``metadata.version`` is copied from the manifest at build time; ``test_build_skills`` checks it. """ @@ -31,7 +31,7 @@ #: Every file that holds a copy of the release version, and the pattern of the one line that holds it. STAMPS = { ENGINE_INIT: VERSION_LINE, - "audit/tools/report.py": re.compile(r'^RECIPE_VERSION = "(?P[^"]+)"(?P.*)$', re.MULTILINE), + "audit/tools/report.py": re.compile(r'^FRAMEWORK_RELEASE = "(?P[^"]+)"(?P.*)$', re.MULTILINE), "implement/tools/progress.py": re.compile(r'^RECIPE_VERSION = "(?P[^"]+)"(?P.*)$', re.MULTILINE), } diff --git a/docs/reference/tools-cli.md b/docs/reference/tools-cli.md index f69ffefe..762891a8 100644 --- a/docs/reference/tools-cli.md +++ b/docs/reference/tools-cli.md @@ -194,7 +194,7 @@ Starts a draft from an intake file. It reads the intake (the answers to the audi | `provenance.target.path` | The intake's `target_path`, resolved to an absolute path. | | `provenance.target.git_sha`, `branch`, `dirty` | Read from git with read-only commands. The commit id is recorded as its first 12 hex digits, because a full id is what secret scanners read as an access token. Each is `null` when it cannot be known: the target is not a repository, the repository has no commit, `HEAD` is detached (no branch), or git is missing. A target that is a subfolder of a repository records nothing and gives a warning, because a commit id says nothing about a subfolder. | | `provenance.target.label` | The intake's `repo_label`, or a label derived from the target directory's name. | -| `provenance.framework_revision` | The framework checkout's commit (12 hex digits), or `null` outside one. | +| `provenance.framework` | The release (`FRAMEWORK_RELEASE` in `report.py`), the framework checkout's commit (12 hex digits, or `null` outside a checkout) and the install: `checkout`, `bundle` or `copy`. | | `provenance.catalogue_version` | The version of [`catalogue.json`](../../audit/catalogue.json). | | `provenance.agent` | The intake's `harness` and `model`. | | `provenance.started_at` | The UTC start time. Its `YYYYMMDDTHHMMSSZ` form is the stamp in the file name. | @@ -288,7 +288,7 @@ Nothing the engine said is dropped without a warning, except a placeholder and a | Message | Fix | |---|---| | `the engine report has schema_version X, and the adapter reads only '2'` | Scan again with [`engine.py`](#enginepy). | -| `the engine report is for phase N, and a 0.1 report covers phase 1` | Scan again without `--phase`. | +| `the engine report is for phase N, and an audit report covers phase 1` | Scan again without `--phase`. | | `the engine report has the wrong shape: N errors` | Each error starts `engine report: /pointer:` and names the field. The file is not an engine report, or was edited. | | `the engine report cannot be imported: N problems` | A check is not in [`catalogue.json`](../../audit/catalogue.json), or only an agent can judge it, or appears twice, or a finding names a check the report does not hold. The engine and the catalogue are from different revisions of the framework. | | `the engine report is not of this draft's target` | `data.errors` names both directories, or both commits. Scan the draft's own repository at its commit, or start a new draft with `init`. | From 5e50513fca567104e1b71aabefbc4feab5c2efba Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 16:08:51 +0530 Subject: [PATCH 040/111] ci: run the check matrix on Linux only The macOS legs set the wait per pull request: 8.5-13 min, against about 4 min on Linux (GitHub run logs, 2026-11-09). The matrix keeps Python 3.11 and 3.14 on ubuntu-latest. CI Gate stays the one required check, so no ruleset change is needed. Issue #82 tracks restoring macOS and adding Windows. Roadmap-Step: CS1.1 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b79a858a..a8b3a4b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,8 @@ # CI runs the same verbs a human or agent runs locally (see the Makefile): `make check` and `make security`. # `make security` needs the network: its dependency audit (osv-scanner) queries osv.dev. # Python 3.11 is the supported minimum and 3.14 the newest release (Q8); Windows is supported through WSL only. +# CI runs on Linux only for now, to keep the wait per pull request short. Restoring macOS and adding Windows is +# tracked in issue #82. # The `docs` job runs `make docs-lint` and the Mermaid check when a Markdown file changed; it needs Node, so it # stays out of `make check` and the matrix. It also annotates the prose of the changed files, which never fails. # It runs on every pull request and on every push to the two protected branches, `dev` and `main` @@ -29,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] python: ["3.11", "3.14"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From b0b37421bfe8c002bab930e6f65e63ccadb5d09a Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 16:08:51 +0530 Subject: [PATCH 041/111] docs: say that CI tests Linux only for now The README platform list and the release checklist no longer claim macOS is tested in CI. Both point to issue #82. Roadmap-Step: CS1.2 Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +++- RELEASE_PROCESS.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bb7a5b01..9937b5f7 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,9 @@ it pushes a branch only if you opt in. ## Platforms - **Linux:** tested. -- **macOS:** the tools are tested in CI. The Claude Code path above is not yet checked by hand. macOS ships an older +- **macOS:** the tools were tested in CI until November 2026. CI runs on Linux only for now; issue + [#82](https://github.com/Nistapp/codebase-agentic-readiness-framework/issues/82) tracks bringing macOS back. + The Claude Code path above is not yet checked by hand. macOS ships an older Python; install Python 3.11 or newer, for example with Homebrew. - **Windows:** not tested. WSL may work. diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 1e09a36f..d5c61911 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -389,7 +389,7 @@ promotion ([Step 1](#step-1-make-sure-that-dev-is-ready)): - [ ] Both rulesets are active, and [Branch protection](docs/reference/branch-protection.md) says so with the date. - [ ] `SECURITY.md` is present. Private vulnerability reporting, secret scanning and push protection are on. - [ ] The root `README.md` quick start works from a fresh clone in a clean directory (Linux checked; macOS by the - maintainer, or stated as CI-only). + maintainer, or stated as not tested while CI is Linux only, issue #82). - [ ] The docs state platform support truthfully: no native Windows claim. - [ ] The promotion pull request from `dev` into `main` passed `CI Gate` and `Check source branch`, and is merged with a merge commit. From 010b3586b98bac66174d8d9fcf7d9c60247ca23a Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 16:23:26 +0530 Subject: [PATCH 042/111] build: run the test suites at once on Linux and split coverage per suite `make test` starts a sub-make for its four suites (test-engine, test-audit-tools, test-implement-tools, test-dev). On Linux with GNU make 4 or newer it passes TEST_JOBS (-j4 --output-sync=target): 67 s instead of 133 s locally, five runs out of five green, largest process about 67 MB. Elsewhere the suites run one by one; macOS ships make 3.81, which has no --output-sync. `make test TEST_JOBS=` forces a serial run. `check` stays serial, as spec/command-surface.md fixes its order. The whole file keeps .NOTPARALLEL; only the suites' sub-make leaves it out, because make 4.3 reads any .NOTPARALLEL as the whole file. `make coverage` now runs coverage-engine, coverage-audit-tools and coverage-implement-tools, each with its own data file, then coverage-combine. The combine step fails when a suite's file is missing. Same total as before: 8797 statements, 760 missed, 91%. Roadmap-Step: CS2.1 Co-Authored-By: Claude Opus 5.5 --- Makefile | 66 ++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 96be310f..ca14fc5e 100644 --- a/Makefile +++ b/Makefile @@ -11,13 +11,15 @@ # (alias: make format-check) # make lint read-only lint # make typecheck mypy, strict, on the typed scope below -# make test the engine's unittest suite, then the recipe tools' (the catalogue checks), then the dev checks' +# make test the four unittest suites: the engine's, the audit tools' (the catalogue checks), the +# implement tools' and the dev checks'; at once on Linux (TEST_JOBS), else one by one # make check THE gate: format:check -> typecheck -> test, plus the engine's rule check, the schema # cross-check, the recipe linter, the master-list freshness check, # the workflow lint and the spell check # make security secrets scan against the committed baseline, then the dependency audit (osv-scanner) of # the dev tools' lockfiles; needs Go and the network -# make coverage (extra) both unittest suites under coverage.py; fails below the floor in .coveragerc +# make coverage (extra) the engine's and the tools' suites under coverage.py; fails below the floor in +# .coveragerc. CI runs it in parts: coverage- per job, then coverage-combine # make self-audit (extra) the engine on this repository, read-only, against .audit-baseline.json; exit 2 on a regression # # `format:check` has a colon, which make reads as a rule separator. The target is spelled `format\:check` @@ -87,11 +89,27 @@ ENGINE := audit/engine TYPED_SOURCES := $(shell find audit/tools implement/tools dev -name '*.py' -not -path '*/testdata/*' -not -path '*/dist/*' 2>/dev/null | sort) $(ENGINE)/build.py .DEFAULT_GOAL := help + +# The suites of `test`. On Linux with GNU make 4 or newer, `make test` runs them at once, in a sub-make with +# TEST_JOBS: about 80 s on CI instead of 170 s, and about 200 MB of memory in all (measured 2026-11-09). Elsewhere +# they run one by one: macOS ships GNU make 3.81, which has no --output-sync. `make test TEST_JOBS=` runs them one by +# one on Linux too. --output-sync=target prints each suite's output in one piece, when that suite ends. +TEST_SUITES := test-engine test-audit-tools test-implement-tools test-dev +ifeq ($(shell uname -s 2>/dev/null)-$(filter 3.%,$(MAKE_VERSION)),Linux-) +TEST_JOBS ?= -j4 --output-sync=target +endif + +# Everything else runs one target at a time, even under `make -j`: spec/command-surface.md fixes the order of +# `check` (format:check, then typecheck, then test). Only the sub-make that `test` starts for its suites is parallel. +# GNU make 4.3 (Ubuntu 24.04) reads any .NOTPARALLEL as "the whole file", so the sub-make leaves it out. +ifndef TEST_SUITES_SUBMAKE .NOTPARALLEL: -.PHONY: help setup format format-check format\:check lint typecheck test coverage verify-rules crosscheck recipe-lint masters-check update-masters check security update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint spell spell-fix +endif +.PHONY: help setup format format-check format\:check lint typecheck test $(TEST_SUITES) coverage coverage-engine coverage-audit-tools coverage-implement-tools coverage-combine verify-rules crosscheck recipe-lint masters-check update-masters check security update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint spell spell-fix help: @echo "verbs: format format:check (format-check) lint typecheck test check security" + @echo "parts: test-engine test-audit-tools test-implement-tools test-dev coverage-engine coverage-audit-tools coverage-implement-tools coverage-combine" @echo "extra: setup (checks prerequisites, installs nothing) coverage self-audit update-audit-baseline verify-rules crosscheck recipe-lint masters-check update-masters workflow-lint commit-lint spell spell-fix update-secrets-baseline validate-skills" @echo "docs: docs-lint docs-lint-all docs-prose [FILES=...] docs-mermaid FILES=... changelog-fix" @@ -123,19 +141,49 @@ typecheck: $(QUALITY_BASELINE) mypy -- $(MYPY) --config-file mypy.ini $(TYPED_SOURCES) test: + $(MAKE) --no-print-directory TEST_SUITES_SUBMAKE=1 $(TEST_JOBS) $(TEST_SUITES) + +test-engine: cd $(ENGINE) && $(PYTHON) -m unittest discover -s tests + +test-audit-tools: $(PYTHON) -m unittest discover -s audit/tools/tests + +test-implement-tools: $(PYTHON) -m unittest discover -s implement/tools/tests + +test-dev: $(PYTHON) -m unittest discover -s dev/tests -# Runs the engine's suite and the recipe tools' suite under coverage.py and fails when the total drops below the -# floor in .coveragerc (`fail_under`). Not part of `check`; CI runs it as its own job. Needs uv. -coverage: +# Runs the engine's suite and the tools' suites under coverage.py and fails when the total drops below the floor in +# .coveragerc (`fail_under`). Not part of `check`; needs uv. Each coverage- writes its own data file +# (.coverage.), so CI runs them in separate jobs; coverage-combine merges them and reports. It fails when a +# suite's file is missing, so a partial run cannot pass on a partial total. coverage.py does not measure dev/. +COVERAGE_SUITES := coverage-engine coverage-audit-tools coverage-implement-tools +coverage: coverage-engine coverage-audit-tools coverage-implement-tools coverage-combine + +coverage-engine: + mkdir -p $(dir $(COVERAGE_DATA)) + rm -f $(COVERAGE_DATA).engine + cd $(ENGINE) && COVERAGE_FILE=$(COVERAGE_DATA).engine $(COVERAGE) run --rcfile=$(CURDIR)/.coveragerc -m unittest discover -s tests + +coverage-audit-tools: + mkdir -p $(dir $(COVERAGE_DATA)) + rm -f $(COVERAGE_DATA).audit-tools + COVERAGE_FILE=$(COVERAGE_DATA).audit-tools $(COVERAGE) run --rcfile=.coveragerc -m unittest discover -s audit/tools/tests + +coverage-implement-tools: mkdir -p $(dir $(COVERAGE_DATA)) + rm -f $(COVERAGE_DATA).implement-tools + COVERAGE_FILE=$(COVERAGE_DATA).implement-tools $(COVERAGE) run --rcfile=.coveragerc -m unittest discover -s implement/tools/tests + +# `coverage combine` deletes the files it merges, so a later combine cannot reuse a stale suite's data. +coverage-combine: + @for suite in $(COVERAGE_SUITES:coverage-%=%); do \ + test -f $(COVERAGE_DATA).$$suite || { echo "coverage-combine: no data for $$suite; run make coverage-$$suite first" >&2; exit 1; }; \ + done rm -f $(COVERAGE_DATA) - cd $(ENGINE) && COVERAGE_FILE=$(COVERAGE_DATA) $(COVERAGE) run --rcfile=$(CURDIR)/.coveragerc -m unittest discover -s tests - COVERAGE_FILE=$(COVERAGE_DATA) $(COVERAGE) run --rcfile=.coveragerc --append -m unittest discover -s audit/tools/tests - COVERAGE_FILE=$(COVERAGE_DATA) $(COVERAGE) run --rcfile=.coveragerc --append -m unittest discover -s implement/tools/tests + COVERAGE_FILE=$(COVERAGE_DATA) $(COVERAGE) combine --rcfile=.coveragerc $(COVERAGE_SUITES:coverage-%=$(COVERAGE_DATA).%) COVERAGE_FILE=$(COVERAGE_DATA) $(COVERAGE) report --rcfile=.coveragerc # The engine on this repository, read-only. Reports go under dist/self-audit/ (git-ignored); only artefacts/ and dist/ From 0dfe7eb302e867030ef93db6b966550ed5e6aca0 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 16:23:26 +0530 Subject: [PATCH 043/111] ci: run the steps of make check as parallel jobs The `check` matrix becomes jobs that run at once: - static: format-check, typecheck, verify-rules, recipe-lint, masters-check, workflow-lint and spell; - crosscheck on Python 3.11 and 3.14; - test-engine, test-tools and test-dev on Python 3.11, the first two under coverage, uploading their data files; - test-py314: `make test verify-rules`, which also runs the suites in parallel on every pull request; - coverage: downloads the data files and runs coverage-combine. CI Gate now reads every result from toJSON(needs), so a job in `needs` cannot go unchecked. dev/tests/test_ci_parity.py fails when a step of `check` or a suite of `test` has no job, when coverage-combine misses a measured suite, or when a job is missing from CI Gate's needs. Each of those five gaps was planted once and caught. Roadmap-Step: CS2.2 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 201 +++++++++++++++++++--------- AGENTS.md | 5 +- CONTRIBUTING.md | 5 +- dev/tests/test_ci_parity.py | 101 ++++++++++++++ docs/reference/branch-protection.md | 13 +- docs/reference/dev-tooling.md | 22 +-- 6 files changed, 269 insertions(+), 78 deletions(-) create mode 100644 dev/tests/test_ci_parity.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8b3a4b2..cea5c781 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,8 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # Copyright (C) 2026 K. C. Ramakrishna # -# CI runs the same verbs a human or agent runs locally (see the Makefile): `make check` and `make security`. +# CI runs the same verbs a human or agent runs locally (see the Makefile): `make check`, split into jobs that run at +# once, and `make security`. # `make security` needs the network: its dependency audit (osv-scanner) queries osv.dev. # Python 3.11 is the supported minimum and 3.14 the newest release (Q8); Windows is supported through WSL only. # CI runs on Linux only for now, to keep the wait per pull request short. Restoring macOS and adding Windows is @@ -25,13 +26,35 @@ concurrency: cancel-in-progress: true jobs: - check: - name: check (${{ matrix.os }}, Python ${{ matrix.python }}) - runs-on: ${{ matrix.os }} + # `make check` split into jobs that run at once; dev/tests/test_ci_parity.py fails when a step of `check` or a + # suite of `test` has no job here. The steps that are cheap share one job, because each job pays about 15 s of + # setup. Python 3.11 runs every step; Python 3.14 runs the runtime code: the suites and the schema cross-check. + static: + name: static (format, typecheck, rules, recipe lint, masters, workflow lint, spell) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version-file: .python-version + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version-file: dev/tools/pyproject.toml + # Go builds actionlint (`make workflow-lint`) from dev/tools/actionlint/go.mod, through the Go module proxy. + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: dev/tools/actionlint/go.mod + cache-dependency-path: dev/tools/actionlint/go.sum + - run: make format-check typecheck verify-rules recipe-lint masters-check workflow-lint spell + + crosscheck: + name: crosscheck (Python ${{ matrix.python }}) + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - os: [ubuntu-latest] python: ["3.11", "3.14"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -43,37 +66,99 @@ jobs: - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version-file: dev/tools/pyproject.toml - # Go builds actionlint (`make workflow-lint`) from dev/tools/actionlint/go.mod, through the Go module proxy. - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + - run: make crosscheck + + # The engine's suite under coverage.py. Its data file goes to the `coverage` job, which combines every suite's. + test-engine: + name: test-engine (Python 3.11, coverage) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - go-version-file: dev/tools/actionlint/go.mod - cache-dependency-path: dev/tools/actionlint/go.sum - - run: make check + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version-file: .python-version + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version-file: dev/tools/pyproject.toml + - run: make coverage-engine + env: + CI: "true" + # The data file's name starts with a dot, so it is a hidden file to upload-artifact. + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: coverage-engine + path: dev/dist/coverage/.coverage.engine + include-hidden-files: true + if-no-files-found: error + retention-days: 1 - security: - name: security + # The audit tools' and the implement tools' suites under coverage.py; one job, because the second takes 2 s. + test-tools: + name: test-tools (Python 3.11, coverage) runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - # The minimum Python the repository pins (.python-version); the `check` matrix above spans 3.11 and 3.14. - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version-file: .python-version - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version-file: dev/tools/pyproject.toml - # Go builds osv-scanner, the dependency audit in `make security`, from dev/tools/osv/go.mod. - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + - run: make coverage-audit-tools coverage-implement-tools + env: + CI: "true" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - go-version-file: dev/tools/osv/go.mod - cache-dependency-path: dev/tools/osv/go.sum - - run: make security + name: coverage-tools + path: | + dev/dist/coverage/.coverage.audit-tools + dev/dist/coverage/.coverage.implement-tools + include-hidden-files: true + if-no-files-found: error + retention-days: 1 + + # The dev checks' suite. coverage.py does not measure dev/ (.coveragerc), so this job runs it plainly. + test-dev: + name: test-dev (Python 3.11) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version-file: .python-version + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version-file: dev/tools/pyproject.toml + - run: make test-dev + + # Every suite on the newest Python, through the `test` verb itself. On Linux, `make test` runs its four suites at + # once (TEST_JOBS in the Makefile), so this job also proves the parallel run on every pull request. + test-py314: + name: test (Python 3.14) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.14" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version-file: dev/tools/pyproject.toml + - run: make test verify-rules - # The engine's and the recipe tools' suites under coverage.py; fails below the floor in .coveragerc. + # Combines the suites' data files and fails below the floor in .coveragerc (`fail_under`). `coverage-combine` + # fails when a suite's data file is missing, so a lost upload cannot pass with a partial total. coverage: name: coverage + needs: [test-engine, test-tools] runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -85,9 +170,33 @@ jobs: - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: version-file: dev/tools/pyproject.toml - - run: make coverage - env: - CI: "true" + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: coverage-* + merge-multiple: true + path: dev/dist/coverage + - run: make coverage-combine + + security: + name: security + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The minimum Python the repository pins (.python-version). + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version-file: .python-version + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + version-file: dev/tools/pyproject.toml + # Go builds osv-scanner, the dependency audit in `make security`, from dev/tools/osv/go.mod. + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: dev/tools/osv/go.mod + cache-dependency-path: dev/tools/osv/go.sum + - run: make security # The engine on this repository against the committed ratchet (.audit-baseline.json): exit 2 when a finding appears # that the baseline did not have. Read-only; the report stays under dist/self-audit/. Linux only. @@ -173,52 +282,24 @@ jobs: fi python3 dev/commit_lint.py --range "$BASE..$HEAD_SHA" - # One required check that stands for the whole matrix, so adding or renaming a `check` leg (a Python - # version, an OS) never also requires a branch-protection edit. + # One required check that stands for every job above, so adding, splitting or renaming a job (a suite, a Python + # version, an OS) never also requires a branch-protection edit. It reads every result in `needs`, so a job listed + # there cannot be left unchecked. ci-gate: name: CI Gate - needs: [check, security, coverage, self-audit, docs, commits] + needs: [static, crosscheck, test-engine, test-tools, test-dev, test-py314, coverage, security, self-audit, docs, commits] if: always() runs-on: ubuntu-latest steps: - - name: Require check, security, coverage, self-audit, docs and commits to have succeeded + - name: Require every needed job to have succeeded # The results come in through env, never expanded into the script (zizmor: template-injection). env: - check: ${{ needs.check.result }} - security: ${{ needs.security.result }} - coverage: ${{ needs.coverage.result }} - self_audit: ${{ needs.self-audit.result }} - docs: ${{ needs.docs.result }} - commits: ${{ needs.commits.result }} + NEEDS: ${{ toJSON(needs) }} run: | - echo "check: $check" - echo "security: $security" - echo "coverage: $coverage" - echo "self-audit: $self_audit" - echo "docs: $docs" - echo "commits: $commits" - if [[ "$check" != "success" ]]; then - echo "::error::check did not pass" - exit 1 - fi - if [[ "$security" != "success" ]]; then - echo "::error::security did not pass" - exit 1 - fi - if [[ "$coverage" != "success" ]]; then - echo "::error::coverage did not pass" - exit 1 - fi - if [[ "$self_audit" != "success" ]]; then - echo "::error::self-audit did not pass" - exit 1 - fi - if [[ "$docs" != "success" ]]; then - echo "::error::docs did not pass" - exit 1 - fi - if [[ "$commits" != "success" ]]; then - echo "::error::commits did not pass" + jq -r 'to_entries[] | "\(.key): \(.value.result)"' <<< "$NEEDS" + failed=$(jq -r '[to_entries[] | select(.value.result != "success") | .key] | join(", ")' <<< "$NEEDS") + if [[ -n "$failed" ]]; then + echo "::error::did not pass: $failed" exit 1 fi echo "CI Gate passed." diff --git a/AGENTS.md b/AGENTS.md index c1c28d58..4a20fb32 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ make format # rewrite formatting in place make format:check # read-only: formatting and lint rules (alias: make format-check) make lint # read-only lint make typecheck # mypy, strict, on audit/tools, implement/tools and dev, and the engine's build.py -make test # the engine's suite, then the audit tools', the implement tools' and the dev checks' +make test # the engine's, the audit tools', the implement tools' and the dev checks' suites (at once on Linux) make check # THE gate: format:check, typecheck, test, the engine's --verify-rules, the schema # cross-check, recipe-lint, masters-check, workflow-lint (actionlint, zizmor, check-jsonschema), spell (typos) make security # secrets scan, then the dependency audit (osv-scanner; needs Go and the network) @@ -69,7 +69,8 @@ no vendor-specific instruction file exists beside it. The developer runs `lefthook install` once per clone; an agent never does. Hooks do not replace the gate: CI runs `make check`, `make security` and `make docs-lint`. - `make coverage` runs the engine's, the audit tools' and the implement tools' suites under coverage.py. It fails - below the floor in `.coveragerc` (`fail_under`). CI runs it as the `coverage` job. The floor is debt that only goes + below the floor in `.coveragerc` (`fail_under`). CI runs it in parts (`coverage-` per job, then + `coverage-combine` in the `coverage` job). The floor is debt that only goes up: raise it when coverage rises, never lower it to get green. - Quality baseline: existing lint debt is counted in `.quality-baseline.json` (ruff, mypy, Markdown lint outside the ratchet) and the secrets in `.secrets.baseline`. No new violations: `make format-check`, `make typecheck` and diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 16aee46c..2a76ce8f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,12 +40,13 @@ the Makefile. The rules an agent follows are in [`AGENTS.md`](AGENTS.md); they a ## The gates -Run these before you propose a change. CI runs the same commands. +Run these before you propose a change. CI runs the same commands: the steps of `make check` as separate jobs +that run at once ([The required checks](docs/reference/branch-protection.md#the-required-checks)). | Command | What it does | | --- | --- | | `make format` | Rewrites formatting in place. Run it before the first `make check` | -| `make check` | The gate: formatting and lint, `mypy` strict, the engine's, the tools' and the dev checks' tests, the engine's rule check, the schema cross-check, the recipe linter and the workflow lint. It takes a couple of minutes | +| `make check` | The gate: formatting and lint, `mypy` strict, the engine's, the tools' and the dev checks' tests, the engine's rule check, the schema cross-check, the recipe linter and the workflow lint. It takes a couple of minutes. The steps run one by one, in the order the spec fixes. On Linux, `make test` runs its four suites at once (`TEST_JOBS`; `make test TEST_JOBS=` runs them one by one) | | `make spell` | Part of `make check`. typos on the tracked text; the word list is `_typos.toml`. `make spell-fix` writes the suggested fixes | | `make workflow-lint` | Part of `make check`. actionlint, zizmor and check-jsonschema on `.github/`: workflow mistakes, workflow security, and `dependabot.yml` against its schema. Offline | | `make commit-lint` | Checks that each commit subject on your branch follows Conventional Commits (`RANGE=origin/dev..HEAD` by default). CI runs the same check on every pull request | diff --git a/dev/tests/test_ci_parity.py b/dev/tests/test_ci_parity.py new file mode 100644 index 00000000..df2ace50 --- /dev/null +++ b/dev/tests/test_ci_parity.py @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna + +"""CI runs every step of ``make check``, split into parallel jobs. + +CI does not call ``make check`` as one command: ``.github/workflows/ci.yml`` runs its steps as separate jobs, so +they run at once. These tests fail when a step of ``check`` (or a suite of ``test``) has no CI job, so a step +added to ``check`` cannot silently drop out of the gate that decides a pull request. +""" + +from __future__ import annotations + +import re +import unittest + +import _dev_paths # noqa: F401 (puts dev/ on sys.path) +from _dev_paths import REPO + +MAKEFILE = REPO / "Makefile" +CI = REPO / ".github" / "workflows" / "ci.yml" +#: A workflow step that runs make: `run: make a b c`, alone on its line. +CI_MAKE = re.compile(r"^\s*(?:-\s+)?run:\s*make\s+(?P[^#|&;]+?)\s*$", re.MULTILINE) + + +def make_rule(text: str, name: str) -> list[str]: + """The prerequisites of the rule ``name``, in order.""" + match = re.search(rf"^{re.escape(name)}:(?!=)(?P[^\n;]*)$", text, re.MULTILINE) + if match is None: + raise AssertionError(f"the Makefile has no rule {name!r}") + return match.group("needs").split() + + +def make_variable(text: str, name: str) -> list[str]: + """The words of the simple variable ``name`` (``NAME := ...``).""" + match = re.search(rf"^{re.escape(name)}\s*:=(?P[^\n]*)$", text, re.MULTILINE) + if match is None: + raise AssertionError(f"the Makefile has no variable {name!r}") + return match.group("value").split() + + +def ci_targets(text: str) -> set[str]: + """Every make target a CI step names, without options and variable assignments.""" + return { + word + for match in CI_MAKE.finditer(text) + for word in match.group("args").split() + if not word.startswith("-") and "=" not in word + } + + +class CheckRunsInCI(unittest.TestCase): + def setUp(self) -> None: + self.makefile = MAKEFILE.read_text(encoding="utf-8") + self.ci = ci_targets(CI.read_text(encoding="utf-8")) + self.suites = make_variable(self.makefile, "TEST_SUITES") + self.coverage_suites = make_variable(self.makefile, "COVERAGE_SUITES") + + def test_every_step_of_check_has_a_ci_job(self) -> None: + steps = [step for step in make_rule(self.makefile, "check") if step != "test"] + self.assertIn("test", make_rule(self.makefile, "check")) + self.assertEqual([step for step in steps if step not in self.ci], [], "steps of check that CI never runs") + + def test_every_test_suite_has_a_ci_job(self) -> None: + # A suite counts when CI runs it plainly (test-dev) or under coverage (coverage-engine runs test-engine's suite). + missing = [ + suite + for suite in self.suites + if suite not in self.ci and suite.replace("test-", "coverage-", 1) not in self.ci + ] + self.assertEqual(missing, [], "test suites that CI never runs") + + def test_ci_runs_the_whole_test_verb_too(self) -> None: + # The Python 3.14 job runs `make test`: every suite, the way a person runs it. + self.assertIn("test", self.ci) + + def test_coverage_combines_every_suite_ci_measures(self) -> None: + self.assertIn("coverage-combine", self.ci) + self.assertEqual(make_rule(self.makefile, "coverage"), [*self.coverage_suites, "coverage-combine"]) + for target in self.coverage_suites: + with self.subTest(target=target): + self.assertIn(target, self.ci) + self.assertIn(target.replace("coverage-", "test-", 1), self.suites) + + def test_ci_gate_needs_every_other_job(self) -> None: + # A job that CI Gate does not need can fail, or never finish, while the gate passes. + text = CI.read_text(encoding="utf-8") + jobs = re.findall(r"^ (?P[a-z0-9-]+):$", text[text.index("\njobs:\n") :], re.MULTILINE) + needs = re.search(r"^ ci-gate:\n(?: .*\n)*? needs: \[(?P[^\]]*)\]", text, re.MULTILINE) + self.assertIsNotNone(needs, "ci-gate has no needs list") + assert needs is not None + self.assertEqual( + sorted(job for job in jobs if job != "ci-gate"), sorted(n.strip() for n in needs["needs"].split(",")) + ) + + def test_the_parser_reads_a_step_list(self) -> None: + sample = " - run: make format-check typecheck\n run: make -j4 --output-sync=target test X=1\n" + self.assertEqual(ci_targets(sample), {"format-check", "typecheck", "test"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/reference/branch-protection.md b/docs/reference/branch-protection.md index f60b86cc..1b5fdd82 100644 --- a/docs/reference/branch-protection.md +++ b/docs/reference/branch-protection.md @@ -67,13 +67,20 @@ Both checks come from GitHub Actions (integration id `15368`). ### `CI Gate` `CI Gate` is the last job in [`.github/workflows/ci.yml`](../../.github/workflows/ci.yml). CI runs on every pull -request and on every push to `dev` and `main`. `CI Gate` needs six jobs and fails unless each one succeeded: +request and on every push to `dev` and `main`. `CI Gate` needs eleven jobs and fails unless each one succeeded. +The jobs run on Ubuntu (issue #82 tracks macOS and Windows). The first six run the steps of `make check` at once; +`dev/tests/test_ci_parity.py` fails when a step of `check` has no job: | Job | What it runs | | --- | --- | -| `check` | `make check` on Ubuntu and macOS, Python 3.11 and 3.14. It includes `make workflow-lint` | +| `static` | `make format-check typecheck verify-rules recipe-lint masters-check workflow-lint spell` | +| `crosscheck` | `make crosscheck` on Python 3.11 and 3.14 | +| `test-engine` | `make coverage-engine`: the engine's suite under coverage.py, on Python 3.11 | +| `test-tools` | `make coverage-audit-tools coverage-implement-tools` on Python 3.11 | +| `test-dev` | `make test-dev` on Python 3.11 | +| `test-py314` | `make test verify-rules` on Python 3.14; on Linux `make test` runs its four suites at once | +| `coverage` | `make coverage-combine`: merges the suites' data files and fails below the floor in `.coveragerc` | | `security` | `make security` | -| `coverage` | `make coverage`, which fails below the floor in `.coveragerc` | | `self-audit` | `make self-audit`: the engine on this repository; exit 2 when a finding appears that `.audit-baseline.json` did not have | | `docs` | `make docs-lint` and the Mermaid render check, when Markdown changed | | `commits` | `dev/commit_lint.py` on each commit that the pull request or push adds: Conventional Commits subjects; merge commits are skipped | diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 6b5c9c5b..5cbfd079 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -11,17 +11,17 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | Tool | What it catches | Command | In CI | Version pinned in | | --- | --- | --- | --- | --- | -| ruff | Python formatting and lint errors (rules in `ruff.toml`) | `make format`, `make format:check`, `make lint` | `check` | `dev/tools/pyproject.toml` | -| mypy | Python type errors, strict mode (`mypy.ini`) | `make typecheck` | `check` | `dev/tools/pyproject.toml` | -| unittest | Failing tests: the engine, the audit tools, the implement tools, the dev checks | `make test` | `check` | Python standard library | -| coverage.py | Test coverage below the floor in `.coveragerc` | `make coverage` | `coverage` | `dev/tools/pyproject.toml` | -| jsonschema | The tools' schema validator disagrees with the reference package | `make crosscheck` | `check` | `dev/tools/pyproject.toml` | -| `dev/recipe_lint.py` | Broken links, bad `SKILL.md` frontmatter, oversize files, unknown tools, check IDs and requirement IDs | `make recipe-lint` | `check` | This repository | -| `dev/build_masters.py` | An [audit master list](audit-master.md) or [implement master list](implement-master.md) that no longer matches the catalogue, the check files or the playbooks | `make masters-check`; `make update-masters` rewrites both pages | `check` | This repository | -| actionlint | Workflow mistakes: syntax, expressions, action inputs, shellcheck on `run:` blocks | `make workflow-lint` | `check` | `dev/tools/actionlint/go.mod` | -| zizmor | Workflow and Dependabot security: injection, broad permissions, unpinned actions, cooldown. Runs at the pedantic persona, so a permission without a comment or a missing concurrency limit fails too | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | -| typos | Misspelled words in the tracked text, code identifiers included (word list: [`_typos.toml`](../../_typos.toml)) ([Spell check](#spell-check)) | `make spell`; `make spell-fix` writes the fixes | `check` | `dev/tools/pyproject.toml` | -| check-jsonschema | `.github/workflows/*.yml` and `.github/dependabot.yml` that do not match their SchemaStore schemas | `make workflow-lint` | `check` | `dev/tools/pyproject.toml` | +| ruff | Python formatting and lint errors (rules in `ruff.toml`) | `make format`, `make format:check`, `make lint` | `static` | `dev/tools/pyproject.toml` | +| mypy | Python type errors, strict mode (`mypy.ini`) | `make typecheck` | `static` | `dev/tools/pyproject.toml` | +| unittest | Failing tests: the engine, the audit tools, the implement tools, the dev checks | `make test` (on Linux, its four suites at once) | `test-engine`, `test-tools`, `test-dev`, `test-py314` | Python standard library | +| coverage.py | Test coverage below the floor in `.coveragerc` | `make coverage` (in CI: `coverage-`, then `coverage-combine`) | `test-engine`, `test-tools`, `coverage` | `dev/tools/pyproject.toml` | +| jsonschema | The tools' schema validator disagrees with the reference package | `make crosscheck` | `crosscheck` | `dev/tools/pyproject.toml` | +| `dev/recipe_lint.py` | Broken links, bad `SKILL.md` frontmatter, oversize files, unknown tools, check IDs and requirement IDs | `make recipe-lint` | `static` | This repository | +| `dev/build_masters.py` | An [audit master list](audit-master.md) or [implement master list](implement-master.md) that no longer matches the catalogue, the check files or the playbooks | `make masters-check`; `make update-masters` rewrites both pages | `static` | This repository | +| actionlint | Workflow mistakes: syntax, expressions, action inputs, shellcheck on `run:` blocks | `make workflow-lint` | `static` | `dev/tools/actionlint/go.mod` | +| zizmor | Workflow and Dependabot security: injection, broad permissions, unpinned actions, cooldown. Runs at the pedantic persona, so a permission without a comment or a missing concurrency limit fails too | `make workflow-lint` | `static` | `dev/tools/pyproject.toml` | +| typos | Misspelled words in the tracked text, code identifiers included (word list: [`_typos.toml`](../../_typos.toml)) ([Spell check](#spell-check)) | `make spell`; `make spell-fix` writes the fixes | `static` | `dev/tools/pyproject.toml` | +| check-jsonschema | `.github/workflows/*.yml` and `.github/dependabot.yml` that do not match their SchemaStore schemas | `make workflow-lint` | `static` | `dev/tools/pyproject.toml` | | detect-secrets | Key-shaped strings not in `.secrets.baseline` | `make security` | `security` | `dev/tools/pyproject.toml` | | osv-scanner | A known vulnerability in a dev tool or its dependencies (Python, Node, Go lockfiles), unless [`osv-scanner.toml`](../../dev/tools/osv-scanner.toml) accepts it ([Dependency audit](#dependency-audit)) | `make security` | `security` | `dev/tools/osv/go.mod` | | markdownlint-cli2 | Markdown rule violations (`.markdownlint-cli2.jsonc`) | `make docs-lint`; `make changelog-fix` rewrites `CHANGELOG.md` with the fixes | `docs`; `format-changelog` in `release-please.yml` | `dev/tools/package.json` | From 8df5231f739d4d7dc1e6530a06642910b38c22c5 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 16:27:54 +0530 Subject: [PATCH 044/111] ci: run CI on merge groups of the dev merge queue Adds the merge_group trigger (checks_requested). A merge queue waits for CI Gate on the merge group, so without the trigger no pull request could land once the queue is on. In a merge group the pull_request payload is empty and `before` is unset. The docs and commits jobs now fall back to merge_group.base_sha and head_sha; before this, `commits` would have passed in the queue without checking a subject. A parity test fails when the trigger goes, or when a job reads pull_request.base.sha or head.sha without the merge-group fallback. Roadmap-Step: CS3.1 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 19 +++++++++++++------ dev/tests/test_ci_parity.py | 12 ++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cea5c781..61612440 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,12 +9,18 @@ # tracked in issue #82. # The `docs` job runs `make docs-lint` and the Mermaid check when a Markdown file changed; it needs Node, so it # stays out of `make check` and the matrix. It also annotates the prose of the changed files, which never fails. -# It runs on every pull request and on every push to the two protected branches, `dev` and `main` +# It runs on every pull request, on every merge group of the `dev` merge queue, and on every push to the two +# protected branches, `dev` and `main` # (docs/reference/branch-protection.md). name: CI on: pull_request: + # The merge queue on `dev` tests each pull request merged onto the latest `dev` before it lands. CI Gate must + # report here too, or the queue waits until its timeout. Each group has its own ref, so the concurrency group + # below never cancels one group for another. + merge_group: + types: [checks_requested] push: branches: [main, dev] @@ -225,7 +231,7 @@ jobs: - name: List changed Markdown id: changed env: - BASE: ${{ github.event.pull_request.base.sha || github.event.before }} + BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} HEAD_SHA: ${{ github.sha }} run: | list="$RUNNER_TEMP/changed-md.txt" @@ -261,8 +267,9 @@ jobs: name: Validate Mermaid blocks in changed Markdown run: xargs -d '\n' -a "$RUNNER_TEMP/changed-md.txt" python3 dev/mermaid_check.py --no-sandbox - # Every commit subject that the pull request (or the push) adds must follow Conventional Commits. The rule is in - # dev/commit_lint.py, which the lefthook commit-msg hook also runs. Merge commits are skipped. + # Every commit subject that the pull request (or the merge group, or the push) adds must follow Conventional + # Commits. The rule is in dev/commit_lint.py, which the lefthook commit-msg hook also runs. Merge commits are + # skipped. commits: name: commits runs-on: ubuntu-latest @@ -273,8 +280,8 @@ jobs: fetch-depth: 0 - name: Check the commit subjects env: - BASE: ${{ github.event.pull_request.base.sha || github.event.before }} - HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }} run: | if [[ -z "$BASE" || "$BASE" =~ ^0+$ ]] || ! git cat-file -e "$BASE^{commit}" 2>/dev/null; then echo "No base commit to compare with (a new branch); nothing to check." diff --git a/dev/tests/test_ci_parity.py b/dev/tests/test_ci_parity.py index df2ace50..b34e98e2 100644 --- a/dev/tests/test_ci_parity.py +++ b/dev/tests/test_ci_parity.py @@ -92,6 +92,18 @@ def test_ci_gate_needs_every_other_job(self) -> None: sorted(job for job in jobs if job != "ci-gate"), sorted(n.strip() for n in needs["needs"].split(",")) ) + def test_the_merge_queue_runs_ci_with_its_own_base_and_head(self) -> None: + # The `dev` ruleset has a merge queue: CI Gate must report on merge_group. In a merge group the pull_request + # payload is empty, so a job that diffs BASE..HEAD from it alone checks nothing there. + text = CI.read_text(encoding="utf-8") + self.assertRegex(text, r"(?m)^ merge_group:\n types: \[checks_requested\]$") + for line in text.splitlines(): + with self.subTest(line=line.strip()): + if "github.event.pull_request.base.sha" in line: + self.assertIn("github.event.merge_group.base_sha", line) + if "github.event.pull_request.head.sha" in line: + self.assertIn("github.event.merge_group.head_sha", line) + def test_the_parser_reads_a_step_list(self) -> None: sample = " - run: make format-check typecheck\n run: make -j4 --output-sync=target test X=1\n" self.assertEqual(ci_targets(sample), {"format-check", "typecheck", "test"}) From 4457c33fdd3d9bd9595a1a94e2421196ec4d16a0 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 16:27:54 +0530 Subject: [PATCH 045/111] chore: add a merge queue to the dev ruleset dev-protection.json gains a merge_queue rule: merge commit, ALLGREEN, up to 5 entries built and merged at once, merge as soon as one entry is ready, 30 min for CI Gate. The up-to-date rule stays on. The docs say how a pull request into dev now lands: `gh pr merge ` adds it to the queue, and GitHub merges it once CI Gate passes on it merged onto the latest dev. Pull requests into main are unchanged. The file applies to GitHub only after the merge_group trigger is on dev (the procedure in docs/reference/branch-protection.md). Roadmap-Step: CS3.2 Co-Authored-By: Claude Opus 5.5 --- .github/rulesets/dev-protection.json | 12 ++++++++++++ CONTRIBUTING.md | 3 ++- RELEASE_PROCESS.md | 9 +++++---- docs/how-to/contribute-a-change.md | 3 ++- docs/reference/branch-protection.md | 17 ++++++++++++----- 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/rulesets/dev-protection.json b/.github/rulesets/dev-protection.json index 811abb30..f3dd1f4a 100644 --- a/.github/rulesets/dev-protection.json +++ b/.github/rulesets/dev-protection.json @@ -44,6 +44,18 @@ } ] } + }, + { + "type": "merge_queue", + "parameters": { + "check_response_timeout_minutes": 30, + "grouping_strategy": "ALLGREEN", + "max_entries_to_build": 5, + "max_entries_to_merge": 5, + "merge_method": "MERGE", + "min_entries_to_merge": 1, + "min_entries_to_merge_wait_minutes": 5 + } } ] } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2a76ce8f..1150f51e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,7 +98,8 @@ Two branches are protected: `dev` and `main`. You cannot commit or push to them - Open the pull request into `dev`. The required check is `CI Gate`. - Do not open a pull request into `main`. Only the maintainer does that, and only from `dev`, to make a release. - Do not edit `CHANGELOG.md` or a version. release-please writes them from your commit subjects at release time. -- Merges use a merge commit. Do not squash or rebase. +- Merges use a merge commit. Do not squash or rebase. Into `dev`, the maintainer merges through the merge queue, + which tests your branch merged onto the latest `dev` before it lands. The step-by-step guide is [Contribute a change](docs/how-to/contribute-a-change.md). The rules and the release steps are in [`RELEASE_PROCESS.md`](RELEASE_PROCESS.md). The ruleset settings are in diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index d5c61911..ebf23970 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -18,7 +18,8 @@ pushes, opens a pull request, merges or starts a workflow ([`AGENTS.md`](AGENTS. example `fix/report-score` or `docs/release-process`. 5. Every pull request must pass the required check `CI Gate`. A pull request into `main` must also pass `Check source branch`. -6. Merge with a merge commit. Do not squash. Do not rebase. +6. Merge with a merge commit. Do not squash. Do not rebase. A pull request into `dev` goes through the merge queue: + `gh pr merge ` adds it, and GitHub merges it when `CI Gate` passes on it merged onto the latest `dev`. 7. Commit subjects follow Conventional Commits. They decide the next version, and they become the changelog. 8. A release is a tag `vX.Y.Z` that release-please makes on `main` when the release pull request is merged. @@ -229,10 +230,10 @@ release commit on `main`. It brings the new version and the changelog entry into 2. Without the App token, close and reopen it to start CI. 3. If `dev` moved since the release, click "Update branch" (the `dev` ruleset requires an up-to-date branch). -4. When `CI Gate` passes, merge it with a merge commit. GitHub deletes the branch: +4. When `CI Gate` passes, add it to the merge queue. GitHub merges it with a merge commit and deletes the branch: ```bash - gh pr merge --merge + gh pr merge ``` Merge it before the next promotion. If you forget, nothing breaks: the files do not conflict. But `dev` shows the old @@ -333,7 +334,7 @@ Dependabot opens a pull request only for a security fix, and never for a major v 1. Make sure that its base is `dev`. Dependabot uses the default branch, which is `dev`. 2. Read the list of updates and the release notes that Dependabot links. -3. Wait for `CI Gate`. Then merge with a merge commit, like any other pull request. +3. Wait for `CI Gate`. Then add it to the merge queue (`gh pr merge `), like any other pull request into `dev`. Do not turn on auto-merge for Dependabot. diff --git a/docs/how-to/contribute-a-change.md b/docs/how-to/contribute-a-change.md index 0e56e4ac..b15b4303 100644 --- a/docs/how-to/contribute-a-change.md +++ b/docs/how-to/contribute-a-change.md @@ -235,7 +235,8 @@ You can also click **Update branch** on the pull request page. Do not rebase a b ### Step 9: The maintainer merges -The maintainer merges your pull request with a merge commit. You do not merge it yourself. +The maintainer adds your pull request to the merge queue. GitHub merges it onto the latest `dev`, runs CI on the +result, and lands it with a merge commit when `CI Gate` passes. You do not merge it yourself. ### Step 10: Clean up diff --git a/docs/reference/branch-protection.md b/docs/reference/branch-protection.md index 1b5fdd82..ccabab94 100644 --- a/docs/reference/branch-protection.md +++ b/docs/reference/branch-protection.md @@ -40,6 +40,7 @@ This table is a summary of them. | Required approvals | 0 | 0 | | Required status checks | `CI Gate` | `CI Gate` and `Check source branch` | | Branch up to date before merge | Required | Not required (see below) | +| Merge queue | Required: up to 5 pull requests a group, each one green, merged with a merge commit | Not used | | Conversations resolved before merge | Required | Required | | Allowed merge method | Merge commit only | Merge commit only | | Force pushes | Blocked | Blocked | @@ -52,6 +53,11 @@ Notes: - **Merge commit only.** A squash joins several Conventional Commits into one. A rebase rewrites them. A merge commit keeps each commit, so each one can be reverted alone. - **0 approvals.** A pull request is required, but an approval is not. The maintainer is the only person who merges. +- **`dev` merges through a merge queue.** `gh pr merge ` adds a pull request to the queue. GitHub then merges it + onto the latest `dev`, together with the entries ahead of it, and runs CI on that result (the `merge_group` + event). It lands only when `CI Gate` passes there, so nobody updates each branch by hand after every merge. The + queue merges as soon as one entry is ready; it builds up to 5 groups at once and waits 30 minutes for `CI Gate`. + The up-to-date rule stays on as well. - **`main` does not require an up-to-date branch.** Promotions from `dev` and the release pull request both merge into `main`. A strict rule would make each wait for, or update with, the other. - **The back-merge comes from its own branch.** `dev` requires an up-to-date branch. A pull request whose head is @@ -67,9 +73,9 @@ Both checks come from GitHub Actions (integration id `15368`). ### `CI Gate` `CI Gate` is the last job in [`.github/workflows/ci.yml`](../../.github/workflows/ci.yml). CI runs on every pull -request and on every push to `dev` and `main`. `CI Gate` needs eleven jobs and fails unless each one succeeded. -The jobs run on Ubuntu (issue #82 tracks macOS and Windows). The first six run the steps of `make check` at once; -`dev/tests/test_ci_parity.py` fails when a step of `check` has no job: +request, on every merge group of the `dev` merge queue and on every push to `dev` and `main`. `CI Gate` needs eleven +jobs and fails unless each one succeeded. The jobs run on Ubuntu (issue #82 tracks macOS and Windows). The first six +run the steps of `make check` at once; `dev/tests/test_ci_parity.py` fails when a step of `check` has no job: | Job | What it runs | | --- | --- | @@ -218,5 +224,6 @@ before that. A session of an AI agent never pushes, tags or merges in this repository (the rule is in [`AGENTS.md`](../../AGENTS.md); [ADR 0003](../../implement/docs/architecture/adrs/0003-push-and-pr-policy.md) covers -the target repositories). The maintainer pushes, opens the promotion pull request and merges. release-please opens the -release and back-merge pull requests and tags ([`RELEASE_PROCESS.md`](../../RELEASE_PROCESS.md)). +the target repositories). The maintainer pushes, opens the promotion pull request and merges; into `dev`, through +the merge queue. release-please opens the release and back-merge pull requests and tags +([`RELEASE_PROCESS.md`](../../RELEASE_PROCESS.md)). From 2309763f2efd9a462a03998334ee035f1e4322e4 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 16:34:48 +0530 Subject: [PATCH 046/111] feat(implement): stop on a report from another schema or a newer release validate_report.py now reads schema_version first. A report with another version gets one plain error that names both versions and what to do: run a new audit, or update the framework. Every tool that validates a report puts that message first. Engine reports still fail at the schema stage. compare_versions() sets the rule for the implement tools: a newer release stops (exit 2), an older one warns, and the same release at two known framework commits warns. progress.py init and check_drift.py both run it and return data.versions, so a resume sees it too. progress.framework_revision() now needs the framework root to be the top of a git working tree. A bundle inside a project recorded the project's commit. Roadmap-Step: V3 Co-Authored-By: Claude Opus 5.5 --- ...ess-state.example-service.in-progress.json | 2 +- ...iness-state.example-service.in-progress.md | 2 +- .../readiness-state.example-service.json | 2 +- .../readiness-state.example-service.md | 2 +- audit/tools/render_report.py | 1 + audit/tools/report.py | 1 + audit/tools/tests/test_validate_report.py | 116 +++++++++++++++++- audit/tools/validate_report.py | 112 ++++++++++++++++- docs/reference/tools-cli.md | 17 ++- implement/SKILL.md | 10 +- implement/state.md | 16 +-- implement/tools/check_drift.py | 9 +- implement/tools/compare_reports.py | 1 + implement/tools/progress.py | 28 ++++- implement/tools/tests/test_check_drift.py | 35 +++++- implement/tools/tests/test_progress.py | 34 ++++- 16 files changed, 359 insertions(+), 29 deletions(-) diff --git a/audit/schema/examples/readiness-state.example-service.in-progress.json b/audit/schema/examples/readiness-state.example-service.in-progress.json index b52980e5..6791d316 100644 --- a/audit/schema/examples/readiness-state.example-service.in-progress.json +++ b/audit/schema/examples/readiness-state.example-service.in-progress.json @@ -6,7 +6,7 @@ }, "recipe": { "name": "brownfield-implement", - "version": "0.2.0-dev", + "version": "0.3.2", "framework_rev": "cafecafecafecafecafecafecafecafecafecafe" }, "source_reports": [ diff --git a/audit/schema/examples/readiness-state.example-service.in-progress.md b/audit/schema/examples/readiness-state.example-service.in-progress.md index 5bb381cd..3948b4b9 100644 --- a/audit/schema/examples/readiness-state.example-service.in-progress.md +++ b/audit/schema/examples/readiness-state.example-service.in-progress.md @@ -9,7 +9,7 @@ A skipped finding is final until the owner unskips it, and every preference bind | Field | Value | |---|---| | Default branch | `main` | -| Recipe | brownfield-implement 0.2.0-dev, framework `cafecafecafecafecafecafecafecafecafecafe` | +| Recipe | brownfield-implement 0.3.2, framework `cafecafecafecafecafecafecafecafecafecafe` | | Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdeadbeefdeadbeefdeadbeefdeadbeef`, catalogue 0.4.0 | ## Summary diff --git a/audit/schema/examples/readiness-state.example-service.json b/audit/schema/examples/readiness-state.example-service.json index 6a20360e..61d576ca 100644 --- a/audit/schema/examples/readiness-state.example-service.json +++ b/audit/schema/examples/readiness-state.example-service.json @@ -6,7 +6,7 @@ }, "recipe": { "name": "brownfield-implement", - "version": "0.2.0-dev", + "version": "0.3.2", "framework_rev": "cafecafecafecafecafecafecafecafecafecafe" }, "source_reports": [ diff --git a/audit/schema/examples/readiness-state.example-service.md b/audit/schema/examples/readiness-state.example-service.md index 7495f494..f743827e 100644 --- a/audit/schema/examples/readiness-state.example-service.md +++ b/audit/schema/examples/readiness-state.example-service.md @@ -9,7 +9,7 @@ A skipped finding is final until the owner unskips it, and every preference bind | Field | Value | |---|---| | Default branch | `main` | -| Recipe | brownfield-implement 0.2.0-dev, framework `cafecafecafecafecafecafecafecafecafecafe` | +| Recipe | brownfield-implement 0.3.2, framework `cafecafecafecafecafecafecafecafecafecafe` | | Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdeadbeefdeadbeefdeadbeefdeadbeef`, catalogue 0.4.0 | ## Summary diff --git a/audit/tools/render_report.py b/audit/tools/render_report.py index 3a7e01e0..8f7da71a 100644 --- a/audit/tools/render_report.py +++ b/audit/tools/render_report.py @@ -1012,6 +1012,7 @@ def handler(argv: Sequence[str]) -> _common.Result: source = Path(args.report).expanduser() document = vr.read_report(source) outcome = vr.validate_document(document) + vr.refuse_unknown_schema(outcome) if outcome.errors: total = len(outcome.errors) raise _common.ValidationFailed( diff --git a/audit/tools/report.py b/audit/tools/report.py index 90e6c9ae..f9d0f82d 100644 --- a/audit/tools/report.py +++ b/audit/tools/report.py @@ -417,6 +417,7 @@ def read_valid_report(path: Path, *, when: str) -> dict[str, Any]: """The report at `path`, draft or final, which must be valid. `when` says what the caller was about to do.""" document = vr.read_report(path, "draft") outcome = vr.validate_document(document) + vr.refuse_unknown_schema(outcome) if outcome.errors: count = len(outcome.errors) raise _common.ValidationFailed( diff --git a/audit/tools/tests/test_validate_report.py b/audit/tools/tests/test_validate_report.py index 9fb0d028..3174effe 100644 --- a/audit/tools/tests/test_validate_report.py +++ b/audit/tools/tests/test_validate_report.py @@ -94,6 +94,118 @@ def test_the_raw_engine_reports_fail_at_the_schema_stage(self) -> None: self.assertTrue(outcome.errors) +class VersionStageTests(unittest.TestCase): + """The report's `schema_version` and framework release against these tools' (V3 of the v0.3.5 plan).""" + + def old_report(self) -> dict[str, Any]: + report = base() + provenance = report["provenance"] + del provenance["framework"] + report["provenance"] = {"recipe": {"name": "brownfield-audit", "version": "0.1.0"}, **provenance} + report["schema_version"] = "0.1" + return report + + def test_the_tools_read_the_schemas_one_version(self) -> None: + self.assertEqual(vr.known_schema_version(), "0.2") + self.assertEqual(base()["schema_version"], vr.known_schema_version()) + + def test_an_older_schema_says_run_a_new_audit_and_names_both_versions(self) -> None: + outcome = vr.validate_document(self.old_report()) + self.assertEqual(outcome.stage, "version") + self.assertEqual( + outcome.errors, + [ + "the report has schema '0.1' (made by framework v0.1.0); these tools read schema '0.2'. Run a new " + "audit with this framework (recommended), or use framework v0.1.0 to work with this report." + ], + ) + + def test_a_newer_schema_says_update_the_framework(self) -> None: + report = base() + report["schema_version"] = "0.3" + (message,) = vr.validate_document(report).errors + self.assertIn("the report has schema '0.3' (made by framework v0.3.2)", message) + self.assertIn("Update the framework to the release that made the report (recommended)", message) + + def test_a_schema_version_with_no_release_says_so(self) -> None: + report = base() + report["schema_version"] = "9" + del report["provenance"]["framework"] + (message,) = vr.validate_document(report).errors + self.assertIn("made by an unknown framework release", message) + + def test_the_cli_puts_the_plain_message_first(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "old.json" + path.write_text(json.dumps(self.old_report()), encoding="utf-8") + code, envelope, _ = call([str(path)]) + self.assertEqual(code, 2) + self.assertTrue(envelope["data"]["error"].startswith("the report has schema '0.1'")) + + +class CompareVersionsTests(unittest.TestCase): + def report(self, release: str = "0.3.2", revision: str | None = "cafecafecafe") -> dict[str, Any]: + report = base() + report["provenance"]["framework"].update(release=release, revision=revision) + return report + + def test_the_relation_compares_numbers_not_text(self) -> None: + self.assertEqual(vr.relation("0.3.2", "0.3.2"), "same") + self.assertEqual(vr.relation("0.3.10", "0.3.9"), "newer") + self.assertEqual(vr.relation("0.2.9", "0.3.0"), "older") + self.assertEqual(vr.relation("0.3.2-dev", "0.3.2"), "older") + self.assertEqual(vr.relation("0.10", "0.9"), "newer") + + def test_the_same_release_and_commit_is_quiet(self) -> None: + data, warnings = vr.compare_versions(self.report(), "0.3.2", "cafecafecafe" + "0" * 28) + self.assertEqual(warnings, []) + self.assertEqual( + data, + { + "report": {"release": "0.3.2", "revision": "cafecafecafe"}, + "tools": {"release": "0.3.2", "revision": "cafecafecafe"}, + "relation": "same", + }, + ) + + def test_an_older_release_is_a_warning(self) -> None: + data, warnings = vr.compare_versions(self.report("0.3.1"), "0.3.2", None) + self.assertEqual(data["relation"], "older") + self.assertEqual( + warnings, + [ + "the report was made by framework v0.3.1; these tools are v0.3.2. Verdicts can differ between " + "releases: run a new audit with v0.3.2 before you rely on them." + ], + ) + + def test_a_newer_release_stops(self) -> None: + with self.assertRaises(_common.ValidationFailed) as caught: + vr.compare_versions(self.report("0.4.0"), "0.3.2", None) + self.assertEqual( + str(caught.exception), + "the report was made by framework v0.4.0; these tools are v0.3.2. Update the framework to v0.4.0 or " + "later (recommended), or run a new audit with v0.3.2.", + ) + self.assertEqual( + caught.exception.errors, ["/provenance/framework/release: 0.4.0 is newer than these tools' 0.3.2"] + ) + + def test_the_same_release_at_another_commit_is_a_warning(self) -> None: + _, warnings = vr.compare_versions(self.report(), "0.3.2", "beefbeefbeef" + "0" * 28) + self.assertEqual( + warnings, + [ + "the report was made at framework commit cafecafecafe; these tools run at beefbeefbeef (both " + "v0.3.2). The checks may have changed in between: run a new audit if that matters." + ], + ) + + def test_an_unknown_commit_on_either_side_gives_no_commit_warning(self) -> None: + self.assertEqual(vr.compare_versions(self.report(revision=None), "0.3.2", "beef" * 10)[1], []) + self.assertEqual(vr.compare_versions(self.report(), "0.3.2", None)[1], []) + + class SchemaStageTests(unittest.TestCase): def test_a_schema_error_is_returned_alone_and_the_rules_do_not_run(self) -> None: report = base() @@ -111,7 +223,9 @@ def test_a_document_that_is_not_an_object_is_a_schema_error(self) -> None: def test_a_report_with_a_version_this_tool_does_not_know_is_refused(self) -> None: report = base() report["schema_version"] = "0.1" - self.assertTrue(any("/schema_version" in error for error in vr.validate_document(report).errors)) + outcome = vr.validate_document(report) + self.assertEqual(outcome.stage, "version") + self.assertEqual(len(outcome.errors), 1) def test_the_schema_errors_are_capped(self) -> None: report = base() diff --git a/audit/tools/validate_report.py b/audit/tools/validate_report.py index dd7f3d7b..6cf30e3a 100644 --- a/audit/tools/validate_report.py +++ b/audit/tools/validate_report.py @@ -62,7 +62,8 @@ @dataclass class Outcome: - """What validation found. `stage` is `schema` when the schema stopped it, and `rules` when the rules ran.""" + """What validation found. `stage` is `version` when the report's `schema_version` is one these tools do not read, + `schema` when the schema stopped it, and `rules` when the rules ran.""" errors: list[str] = field(default_factory=list) warnings: list[str] = field(default_factory=list) @@ -149,6 +150,109 @@ def schema_validator() -> subset.SubsetValidator: return subset.SubsetValidator(json.loads(REPORT_SCHEMA.read_text(encoding="utf-8"))) +# --- Versions: the report's schema and release against these tools' ------------------------------------ + +RELEASE = re.compile(r"^(\d+)\.(\d+)\.(\d+)") + + +@lru_cache(maxsize=1) +def known_schema_version() -> str: + """The one report `schema_version` these tools read: the schema's `const`.""" + schema = json.loads(REPORT_SCHEMA.read_text(encoding="utf-8")) + return str(schema["properties"]["schema_version"]["const"]) + + +def version_key(version: str) -> tuple[int, ...]: + """`MAJOR.MINOR.PATCH` (or `MAJOR.MINOR`) as numbers, for ordering; a suffix is ignored. () when unreadable.""" + match = RELEASE.match(version) or re.match(r"^(\d+)\.(\d+)", version) + return tuple(int(part) for part in match.groups()) if match else () + + +def relation(theirs: str, ours: str) -> str: + """`same`, `older` or `newer`: where `theirs` stands against `ours`. Equal numbers, other text: `older`.""" + if theirs == ours: + return "same" + return "newer" if version_key(theirs) > version_key(ours) else "older" + + +def report_release(document: Mapping[str, Any]) -> str | None: + """The framework release a report names, in any schema this tool has known; None when it names none.""" + provenance = document.get("provenance") + if not isinstance(provenance, Mapping): + return None + for holder, key in ((provenance.get("framework"), "release"), (provenance.get("recipe"), "version")): + if isinstance(holder, Mapping) and isinstance(holder.get(key), str): + return str(holder[key]) + return None + + +def schema_version_error(document: Any) -> str | None: + """The plain message for a report whose `schema_version` these tools do not read; None when they read it. + + It names both versions and what to do next, so the owner can act on it without reading the schema. An engine + report (its `provenance` names a `tool`) has a schema of its own; the schema stage refuses it, not this one. + """ + if not isinstance(document, Mapping) or "schema_version" not in document: + return None + provenance = document.get("provenance") + if isinstance(provenance, Mapping) and "tool" in provenance: + return None + found, known = document["schema_version"], known_schema_version() + if found == known: + return None + release = report_release(document) + made_by = f"framework v{release}" if release else "an unknown framework release" + head = f"the report has schema {found!r} (made by {made_by}); these tools read schema {known!r}." + if isinstance(found, str) and relation(found, known) == "newer": + return f"{head} Update the framework to the release that made the report (recommended), or run a new audit." + return f"{head} Run a new audit with this framework (recommended), or use {made_by} to work with this report." + + +def refuse_unknown_schema(outcome: "Outcome") -> None: + """Raise `ValidationFailed` with the plain version message when the version stage stopped `outcome`.""" + if outcome.stage == "version": + raise _common.ValidationFailed(outcome.errors[0], errors=outcome.errors) + + +def compare_versions( + document: Mapping[str, Any], release: str, revision: str | None +) -> tuple[dict[str, Any], list[str]]: + """The report's framework release and commit against these tools' (`release`, `revision`), for a valid report. + + Returns the `versions` data (`report`, `tools`, `relation`) and the warnings. Raises `ValidationFailed` when + the report is newer than the tools: they may not know its checks. An older report is a warning; so is the same + release at another commit, when both commits are known. + """ + framework = document["provenance"]["framework"] + theirs = str(framework["release"]) + tools_revision = revision[: len(framework["revision"])] if revision and framework["revision"] else revision + where = relation(theirs, release) + data: dict[str, Any] = { + "report": {"release": theirs, "revision": framework["revision"]}, + "tools": {"release": release, "revision": revision[:12] if revision else None}, + "relation": where, + } + if where == "newer": + raise _common.ValidationFailed( + f"the report was made by framework v{theirs}; these tools are v{release}. Update the framework to " + f"v{theirs} or later (recommended), or run a new audit with v{release}.", + errors=[f"/provenance/framework/release: {theirs} is newer than these tools' {release}"], + ) + warnings: list[str] = [] + if where == "older": + warnings.append( + f"the report was made by framework v{theirs}; these tools are v{release}. Verdicts can differ between " + f"releases: run a new audit with v{release} before you rely on them." + ) + elif framework["revision"] and tools_revision and framework["revision"] != tools_revision: + warnings.append( + f"the report was made at framework commit {framework['revision']}; these tools run at " + f"{data['tools']['revision']} (both v{release}). The checks may have changed in between: run a new " + "audit if that matters." + ) + return data, warnings + + @lru_cache(maxsize=1) def load_catalogue() -> Mapping[str, Any]: try: @@ -577,6 +681,11 @@ def validate_document( ) -> Outcome: """Validate a parsed report. The target, the catalogue and the credential patterns default to the real ones.""" out = Outcome() + version_error = schema_version_error(document) + if version_error is not None: + out.stage = "version" + out.errors = [version_error] + return out found = schema_validator().validate(document, limit=MAX_SCHEMA_ERRORS) if found: out.stage = "schema" @@ -611,6 +720,7 @@ def handler(argv: Sequence[str]) -> _common.Result: path = Path(args.report).expanduser() document = read_report(path) outcome = validate_document(document, target=target) + refuse_unknown_schema(outcome) if outcome.errors: count = len(outcome.errors) raise _common.ValidationFailed( diff --git a/docs/reference/tools-cli.md b/docs/reference/tools-cli.md index 762891a8..4ef946a5 100644 --- a/docs/reference/tools-cli.md +++ b/docs/reference/tools-cli.md @@ -131,7 +131,7 @@ rule 6: /checks/6: TOOL-03 overrides the engine's PARTIAL with PASS and gives no schema: /checks/1/verdict: expected one of ["PASS", "PARTIAL", "FAIL", "UNKNOWN", "NOT_APPLICABLE"] ``` -The tool checks in two stages. The first applies the schema, and a report that fails it is returned with its schema errors alone, up to 500. The second applies the rules, only to a report that passed the first, because the rules read fields the schema guarantees. Errors come in rule order, then in the order of the document. An error names the place with a JSON Pointer and never repeats the value it found, so a credential in a quote does not travel into the list. +The tool checks in three stages. The first reads `schema_version`. A report with another version than the schema's gets one plain error that names both versions and what to do: run a new audit (an older report), or update the framework (a newer one). Then it applies the schema, and a report that fails it is returned with its schema errors alone, up to 500. The second applies the rules, only to a report that passed the first, because the rules read fields the schema guarantees. Errors come in rule order, then in the order of the document. An error names the place with a JSON Pointer and never repeats the value it found, so a credential in a quote does not travel into the list. **Exit codes** @@ -149,6 +149,7 @@ The tool checks in two stages. The first applies the schema, and a report that f | `rule 1: ...` to `rule 12: ...` | The rule is in [Rules beyond the schema](../../audit/schema/SCHEMA.md#rules-beyond-the-schema). The message says which item to change. | | `rule 10: /summary/...: the report says X and its checks and findings give Y` | The summary is stale. It is never edited by hand: the report builder, [`report.py`](#reportpy), recomputes it on every change, so the report was changed outside the builder. Y is the correct number. | | `rule 11: ... it looks like a credential` | Shorten the quote or the output tail, or leave it out. The patterns are the engine's, so a line such as `token: abc` is flagged even when it is not a secret. | +| `the report has schema 'X' (made by framework vY); these tools read schema 'Z'. ...` | The report and these tools come from releases with different report schemas. The message says what to do; there is no conversion. | | `schema: /path: ...` | The pointer names the field. The schema is [`audit-report.schema.json`](../../audit/schema/audit-report.schema.json). | | `the file is not a report: ...` | The file is not one JSON document that this tool accepts. | @@ -840,7 +841,8 @@ python3 implement/tools/progress.py --describe **Outputs** `init` writes the JSON and the Markdown and puts their paths, the label, the item count, the count per status and -the `not_queued` count in `data`. `show` puts the summary in `data`: counts per status and per theme (`items` and +the `not_queued` count in `data`, with `versions` as [`check_drift.py`](#check_driftpy) gives it. `show` puts +the summary in `data`: counts per status and per theme (`items` and `open`, which is neither `done` nor `skipped`), the skipped finding ids, and the number of preferences, debt entries and log entries. `show --json` puts the state under `data.state`. `render` writes the Markdown and puts its path and size in `data`. Every other command checks the changed state whole, appends one log entry, replaces @@ -853,7 +855,7 @@ warning. |---|---| | `0` | Done. | | `1` | A bad argument, a missing report or target, a report outside the target, an existing state for `init`, no state for `show` or `render`, a state folder that leaves the target, or a state file that is a symlink. | -| `2` | The report is invalid, not final, or has a finding on a check with no theme; or the state fails the schema or its rules. `data.errors` lists what to fix. | +| `2` | The report is invalid, not final, from a newer framework release, or has a finding on a check with no theme; or the state fails the schema or its rules. `data.errors` lists what to fix. | | `3` | Internal error, including a write the system refused. The old file is left whole. | **Errors** @@ -861,6 +863,7 @@ warning. | Message | Fix | |---|---| | `the report is a draft, not final` | Finalise it with `report.py finalise` and pass the final report. | +| `the report was made by framework vX; these tools are vY. Update the framework ...` | The report is newer than these tools. Install release vX or later of both skills, or run a new audit with vY. | | `the report must be inside the target, so the state can point at it` | Commit the report folder to the target and pass that path. | | `a state already exists: ...; init never replaces one` | Use `show`, `render` and the transition commands. | | `F-nnn is done and done is final` | A regression is a new finding in the next audit. | @@ -918,6 +921,12 @@ change the status or the exit code: compare: the target has no state, the state predates the digest (schema `0.1`), or the state does not list this report. +- `versions` compares the framework release and commit that made the report with these tools'. It holds + `report` and `tools`, each with `release` and `revision` (12 digits, or `null` outside a checkout), and + `relation`: `same`, `older` or `newer`. `older` gives a warning that verdicts can differ between releases, and a + note in the text output. With the same release at two known commits, a warning names both. A `newer` report stops + the tool with exit 2: these tools may not know its checks. + When the audited commit is not an ancestor of `HEAD`, the diff is between the two trees and a warning says so. Long lists are bounded: `changed_paths_omitted` and `findings_omitted` count what was left out, and `truncated` is `true`. @@ -928,7 +937,7 @@ between the two trees and a warning says so. Long lists are bounded: `changed_pa |---|---| | `0` | Done, whatever the status. Read `data.status`. | | `1` | A bad argument, a missing report or target, or a target that is not a git repository. | -| `2` | The report is not valid; `data.errors` lists what to fix. | +| `2` | The report is not valid, or it comes from a newer framework release than these tools; `data.error` says what to do. | | `3` | Internal error. | ## compare_reports.py diff --git a/implement/SKILL.md b/implement/SKILL.md index b616b69a..5647c30c 100644 --- a/implement/SKILL.md +++ b/implement/SKILL.md @@ -111,8 +111,10 @@ and that a state file exists. Change nothing in the target except the state fold 4. **Validate.** Run `python3 audit/tools/validate_report.py --target `. Expect exit 0 and `data.valid` true; read `data.status` and `warnings`. If `data.status` is not `final`, or the exit code is 2, stop: tell the owner the report goes back to the [audit recipe](../audit/SKILL.md) - (`report.py finalise`). Do not repair a report here. A warning about a cited path that no longer exists is - normal; keep it for step 5. + (`report.py finalise`). Do not repair a report here. If the error starts `the report has schema`, the report + comes from a framework release with another report schema: give the owner the message as it is. It says whether + to run a new audit or update the framework. A warning about a cited path that no longer exists is normal; keep it + for step 5. 5. **Drift.** Run `python3 implement/tools/check_drift.py `, on a new run and on every resume. It exits 0 whatever it finds, so read `data.status`: - `current`: nothing moved; go on. @@ -123,6 +125,10 @@ and that a state file exists. Change nothing in the target except the state fold - Whatever the status, `data.report_changed` true means the report file was edited after `init` recorded its SHA-256 (`null` means no digest was recorded): stop and tell the owner; the text is no longer the audit's. - `sha_missing` or `no_sha`: re-verify every queued finding when its theme comes up, and say so now. + - `data.versions.relation` compares the framework release that made the report with these tools'. `older`: + give the owner the warning and go on; a new audit is the owner's call. `same` with a warning about two + framework commits: tell the owner and go on. Exit 2 with `Update the framework`: the report is newer than + these tools; stop and ask the owner to install that release of both skills. 6. **State.** On a new run, create the triage worktree first, so nothing is committed on the integration branch itself: `git -C fetch origin`, then `git -C worktree add -b agentic-readiness/-triage origin/`. Then create the state there: diff --git a/implement/state.md b/implement/state.md index 3f32a4d0..f0cba20a 100644 --- a/implement/state.md +++ b/implement/state.md @@ -42,15 +42,17 @@ The report is committed on the integration branch at `source_reports[].sha256`; `check_drift.py` recomputes it and reports `report_changed` when the file was edited since. A state made before schema `0.2` has no digest, and `report_changed` is then `null`, not `false`. -`init` records the framework commit it ran from in `recipe.framework_rev`, read with `git` from the folder that -holds the tools. From an installed skill that folder is not a framework checkout: `init` then records the commit of -any git repository the skills folder sits in, or `null` when there is none. Pass `--framework-rev ` with the +`init` records the framework commit it ran from in `recipe.framework_rev`, read with `git` from the folder above +the tools, but only when that folder is the top of a git working tree. From an installed skill it is not, even when +the skills folder sits inside another repository, so `init` records `null`. Pass `--framework-rev ` with the framework release's commit when you know it. -`init` refuses a report that is not final or that `validate_report.py` rejects, a report outside the target, and an -existing state. It queues confirmed and edited findings (an edited finding's note goes into `notes`), imports a -finding the owner deferred during the audit as `deferred` with the owner's note as its reason, and lists rejected -findings as `not_queued`. Each item gets the theme of its check from the table in the tool. +`init` refuses a report that is not final or that `validate_report.py` rejects, a report from a newer framework +release than the tools, a report outside the target, and an existing state. A report from an older release gives a +warning; `data.versions` holds both releases and commits. It queues confirmed and edited findings (an edited +finding's note goes into `notes`), imports a finding the owner deferred during the audit as `deferred` with the +owner's note as its reason, and lists rejected findings as `not_queued`. Each item gets the theme of its check +from the table in the tool. ## Item statuses diff --git a/implement/tools/check_drift.py b/implement/tools/check_drift.py index 23c2a3d4..a5713cc1 100644 --- a/implement/tools/check_drift.py +++ b/implement/tools/check_drift.py @@ -136,6 +136,7 @@ def handler(argv: Sequence[str]) -> _common.Result: target = _common.resolve_target(args.target) document = validate_report.read_report(Path(args.report).expanduser()) outcome = validate_report.validate_document(document) + validate_report.refuse_unknown_schema(outcome) if outcome.errors: count = len(outcome.errors) raise _common.ValidationFailed( @@ -146,8 +147,10 @@ def handler(argv: Sequence[str]) -> _common.Result: head = revision(target, "HEAD") if head is None: raise _common.UsageError(f"cannot read HEAD of {target}: not a git repository, or it has no commits") - data: dict[str, Any] = {"audited_sha": audited, "head_sha": head, "drifted": False} - warnings: list[str] = [] + versions, warnings = validate_report.compare_versions( + document, progress.RECIPE_VERSION, progress.framework_revision() + ) + data: dict[str, Any] = {"audited_sha": audited, "head_sha": head, "drifted": False, "versions": versions} data["dirty"] = is_dirty(target) data["report_changed"] = report_changed( target, Path(args.report).expanduser(), document["provenance"]["target"]["label"] @@ -189,6 +192,8 @@ def handler(argv: Sequence[str]) -> _common.Result: data["findings_omitted"] = omitted_findings truncated = bool(omitted_paths or omitted_findings) summary = f"check_drift: {data['status']}" + if versions["relation"] != "same": + summary += f"; the report is from an {versions['relation']} framework release" if data["status"] == "drifted": summary += f", {data['changed_count']} paths changed, {len(data['findings'])} findings affected" if data["report_changed"]: diff --git a/implement/tools/compare_reports.py b/implement/tools/compare_reports.py index ce8d4cb3..e51a3c28 100644 --- a/implement/tools/compare_reports.py +++ b/implement/tools/compare_reports.py @@ -240,6 +240,7 @@ def _counts_text(values: Mapping[str, int]) -> str: def read_valid(path: str, which: str) -> Any: document = validate_report.read_report(Path(path).expanduser(), f"{which} report") outcome = validate_report.validate_document(document) + validate_report.refuse_unknown_schema(outcome) if outcome.errors: count = len(outcome.errors) raise _common.ValidationFailed( diff --git a/implement/tools/progress.py b/implement/tools/progress.py index e10803fe..ab01dc95 100644 --- a/implement/tools/progress.py +++ b/implement/tools/progress.py @@ -257,11 +257,10 @@ def _finding_order(finding_id: str) -> tuple[int, str]: return (int(digits) if digits.isdigit() else 0, finding_id) -def framework_revision() -> str | None: - """HEAD of the framework checkout this tool runs from, read with git; None when it is not a git checkout.""" +def _framework_git(*args: str) -> str | None: try: done = subprocess.run( - ["git", "-C", str(FRAMEWORK_ROOT), "rev-parse", "HEAD"], + ["git", "-C", str(FRAMEWORK_ROOT), *args], capture_output=True, text=True, timeout=GIT_TIMEOUT_S, @@ -269,8 +268,20 @@ def framework_revision() -> str | None: ) except (OSError, subprocess.SubprocessError): return None - sha = done.stdout.strip() - return sha if done.returncode == 0 and re.fullmatch(r"[0-9a-f]{40}", sha) else None + return done.stdout.strip() if done.returncode == 0 else None + + +def framework_revision() -> str | None: + """HEAD of the framework checkout this tool runs from; None when it is not one. + + `FRAMEWORK_ROOT` must be the top of the git working tree. An installed bundle inside another repository is in a + working tree too, but that tree's HEAD is the other repository's commit, not the framework's. + """ + top = _framework_git("rev-parse", "--show-toplevel") + if top is None or Path(top).resolve() != FRAMEWORK_ROOT.resolve(): + return None + sha = _framework_git("rev-parse", "HEAD") + return sha if sha is not None and re.fullmatch(r"[0-9a-f]{40}", sha) else None def build_state( @@ -399,6 +410,7 @@ def run_init(args: Any) -> _common.Result: ) report = validate_report.read_report(report_file) outcome = validate_report.validate_document(report) + validate_report.refuse_unknown_schema(outcome) if outcome.errors: raise _common.ValidationFailed( f"the report is not valid: {len(outcome.errors)} errors; run validate_report.py on it", @@ -408,6 +420,7 @@ def run_init(args: Any) -> _common.Result: raise _common.ValidationFailed( f"the report is a {report['status']}, not final", hint="finalise it first with: report.py finalise" ) + versions, version_warnings = validate_report.compare_versions(report, RECIPE_VERSION, framework_revision()) label = report["provenance"]["target"]["label"] json_name, md_name = state_names(label) directory = state_directory(target) @@ -436,9 +449,12 @@ def run_init(args: Any) -> _common.Result: _common.atomic_write_text(json_path, encode_state(state)) _common.atomic_write_text(md_path, render(state)) counts = status_counts(state) - warnings = [f"removed {swept} temporary files an interrupted write left in {directory}"] if swept else [] + warnings = [*version_warnings] + if swept: + warnings.append(f"removed {swept} temporary files an interrupted write left in {directory}") return _common.Result( data={ + "versions": versions, "state": str(json_path), "markdown": str(md_path), "label": label, diff --git a/implement/tools/tests/test_check_drift.py b/implement/tools/tests/test_check_drift.py index 2b42bd57..87bb8854 100644 --- a/implement/tools/tests/test_check_drift.py +++ b/implement/tools/tests/test_check_drift.py @@ -40,11 +40,44 @@ def setUp(self) -> None: self.audited = _git.run_git(self.repo, "rev-parse", "HEAD") self.report_path = base / "report.json" - def write_report(self, sha: str | None) -> None: + def write_report(self, sha: str | None, release: str | None = None) -> None: report = json.loads(REPORT.read_text(encoding="utf-8")) report["provenance"]["target"]["git_sha"] = sha + if release is not None: + report["provenance"]["framework"]["release"] = release self.report_path.write_text(json.dumps(report), encoding="utf-8") + def test_the_versions_of_the_report_and_the_tools_are_in_the_data(self) -> None: + self.write_report(self.audited) + code, envelope, _ = self.run_tool() + self.assertEqual(code, 0) + versions = envelope["data"]["versions"] + self.assertEqual(versions["report"]["release"], "0.3.2") + self.assertEqual(versions["tools"]["release"], progress.RECIPE_VERSION) + self.assertIn(versions["relation"], ("same", "older", "newer")) + + def test_a_report_from_an_older_release_warns_and_goes_on(self) -> None: + self.write_report(self.audited, release="0.0.1") + code, envelope, stderr = self.run_tool() + self.assertEqual(code, 0) + self.assertEqual(envelope["data"]["versions"]["relation"], "older") + self.assertTrue(any("made by framework v0.0.1" in warning for warning in envelope["warnings"])) + self.assertIn("the report is from an older framework release", stderr) + + def test_a_report_from_a_newer_release_stops(self) -> None: + self.write_report(self.audited, release="99.0.0") + code, envelope, _ = self.run_tool() + self.assertEqual(code, 2) + self.assertIn("Update the framework to v99.0.0 or later", envelope["data"]["error"]) + + def test_a_report_with_another_schema_stops_with_the_plain_message(self) -> None: + report = json.loads(REPORT.read_text(encoding="utf-8")) + report["schema_version"] = "0.1" + self.report_path.write_text(json.dumps(report), encoding="utf-8") + code, envelope, _ = self.run_tool() + self.assertEqual(code, 2) + self.assertTrue(envelope["data"]["error"].startswith("the report has schema '0.1'")) + def commit(self, name: str, text: str) -> None: (self.repo / name).parent.mkdir(parents=True, exist_ok=True) (self.repo / name).write_text(text, encoding="utf-8") diff --git a/implement/tools/tests/test_progress.py b/implement/tools/tests/test_progress.py index 0973865f..e2800e1f 100644 --- a/implement/tools/tests/test_progress.py +++ b/implement/tools/tests/test_progress.py @@ -24,6 +24,7 @@ from unittest import mock import _implement_paths +import _git import _common import progress @@ -42,7 +43,7 @@ EXAMPLE_ARGS = ("--date", "2026-10-29", "--framework-rev", "cafecafecafecafecafecafecafecafecafecafe") # The recipe version it was made with. release-please bumps `progress.RECIPE_VERSION` each release; the checked-in # example keeps the version that made it, the same way it keeps its date. -EXAMPLE_RECIPE_VERSION = "0.2.0-dev" +EXAMPLE_RECIPE_VERSION = "0.3.2" def call(argv: Sequence[str]) -> tuple[int, dict[str, Any], str]: @@ -120,7 +121,38 @@ def test_no_subcommand_is_a_usage_error(self) -> None: self.assertEqual((code, envelope["ok"]), (1, False)) +class FrameworkRevisionTests(unittest.TestCase): + def test_a_bundle_inside_another_repository_has_no_framework_revision(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + project = Path(tmp).resolve() / "project" + _git.make_repo(project) + bundle = project / ".claude" / "skills" / "agentic-readiness-implement" + bundle.mkdir(parents=True) + with mock.patch.object(progress, "FRAMEWORK_ROOT", bundle): + self.assertIsNone(progress.framework_revision()) + with mock.patch.object(progress, "FRAMEWORK_ROOT", project): + self.assertEqual(progress.framework_revision(), _git.run_git(project, "rev-parse", "HEAD")) + + class InitTests(TargetCase): + def with_release(self, release: str) -> None: + report = json.loads(self.report.read_text(encoding="utf-8")) + report["provenance"]["framework"]["release"] = release + self.report.write_text(json.dumps(report), encoding="utf-8") + + def test_init_refuses_a_report_from_a_newer_release_and_writes_nothing(self) -> None: + self.with_release("99.0.0") + code, envelope, _ = self.init(*EXAMPLE_ARGS) + self.assertEqual(code, 2) + self.assertIn("Update the framework to v99.0.0 or later", envelope["data"]["error"]) + self.assertFalse(self.state_dir.exists()) + + def test_init_warns_about_a_report_from_an_older_release(self) -> None: + self.with_release("0.0.1") + envelope = self.init_ok() + self.assertEqual(envelope["data"]["versions"]["relation"], "older") + self.assertTrue(any("made by framework v0.0.1" in warning for warning in envelope["warnings"])) + def test_init_stamps_the_release_version(self) -> None: self.init_ok() state = json.loads(self.json_path.read_text(encoding="utf-8")) From 3de9d91aa4bf38bea507637237cbd97a233ecf0a Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 16:43:24 +0530 Subject: [PATCH 047/111] docs: record the dev merge queue in Current state The dev ruleset (id 26787626) was updated on 2026-11-09 from dev-protection.json: the merge queue is on, and strict "up to date" stays on. The page also records the Linux-only CI and the parallel jobs. Roadmap-Step: CS3.4 Co-Authored-By: Claude Opus 5.5 --- docs/reference/branch-protection.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/reference/branch-protection.md b/docs/reference/branch-protection.md index ccabab94..e8843b8d 100644 --- a/docs/reference/branch-protection.md +++ b/docs/reference/branch-protection.md @@ -160,10 +160,13 @@ The other checks score 10, or do not apply. The full, current list is on the | Ruleset | Target | State on GitHub | | --- | --- | --- | | `main protection` (id 26750008) | `refs/heads/main` | Active since 2026-11-07. Updated to [`main-protection.json`](../../.github/rulesets/main-protection.json) on 2026-11-08 | -| `dev protection` (id 26787626) | `refs/heads/dev` | Active since 2026-11-08, from [`dev-protection.json`](../../.github/rulesets/dev-protection.json) | +| `dev protection` (id 26787626) | `refs/heads/dev` | Active since 2026-11-08. Updated to [`dev-protection.json`](../../.github/rulesets/dev-protection.json) on 2026-11-09: the merge queue | On 2026-11-08 the default branch changed from `main` to `dev`. +On 2026-11-09 CI moved to Linux only (issue #82 tracks macOS and Windows), `make check` became parallel jobs, and +the `dev` merge queue was turned on. Pull requests into `dev` now land with `gh pr merge `. + On 2026-11-07 these settings were also turned on: private vulnerability reporting, secret scanning, push protection and Dependabot security updates (see [`SECURITY.md`](../../SECURITY.md)). From d7ad4290cfcf525468f62da994ec118744c61530 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 17:16:22 +0530 Subject: [PATCH 048/111] feat(engine): TOOL-06 also reads the commit rule in CI TOOL-06 only read a local commit-message rule, so a hook that --no-verify or a missing install skips was a PASS, and a pull-request title check alone was a FAIL. It now also reads every workflow file for a step that checks commit messages: commitlint, cz check, gitlint, the pre-commit or lefthook commit-msg stage, a pull-request title action, a repository script the local commit-msg stage also runs, or a runner target whose command does one of those. Install lines and comments do not count. Local and CI is PASS, one of the two is PARTIAL, neither is FAIL. TOOL-06 stays COSMETIC. The runner-target parser moves from the CI pack to _common so both packs share it. Ruleset 2026-11-09.4; REQ-SH-032 adds the CI step. Two one-defect fixtures with eval files; tool-ok-node gains a commitlint workflow, which moves the facts golden and the stack-detection row it held. 13 in-process tests. Self-audit: TOOL-06 PASS. Coverage 91% (802 missed). Roadmap-Step: C4 Co-Authored-By: Claude Opus 5.5 --- .secrets.baseline | 24 +- audit/checks/05-tooling.md | 39 +-- audit/engine/audit/__init__.py | 2 +- audit/engine/audit/rules/checks/_common.py | 18 ++ audit/engine/audit/rules/checks/ci.py | 21 +- audit/engine/audit/rules/checks/tool.py | 145 ++++++++++- .../02-check-catalogue.md | 2 +- audit/engine/docs/reference/changelog.md | 1 + audit/engine/tests/fixtures/README.md | 4 +- .../.github/workflows/pr-title.yml | 11 + .../.pre-commit-config.yaml | 8 + .../.prettierrc.json | 3 + .../eslint.config.js | 1 + .../tool-06-ci-only-commit-rule/package.json | 10 + .../tool-06-ci-only-commit-rule/tsconfig.json | 5 + .../.pre-commit-config.yaml | 13 + .../.prettierrc.json | 3 + .../commitlint.config.js | 1 + .../eslint.config.js | 1 + .../package.json | 10 + .../tsconfig.json | 5 + .../.github/workflows/commits.yml | 14 ++ audit/engine/tests/test_fixture_matrix.py | 2 + .../tests/test_tool06_commit_rule_in_ci.py | 231 ++++++++++++++++++ audit/engine/tests/test_tool_pack.py | 2 + audit/schema/examples/facts.golden.git.json | 16 +- audit/tools/tests/test_facts_stack.py | 2 +- .../fixtures/tool-06-ci-only-commit-rule.json | 14 ++ .../tool-06-local-only-commit-rule.json | 14 ++ dev/eval/expected/fixtures/tool-ok-node.json | 4 +- docs/reference/audit-master.md | 3 +- spec/readiness-requirements.md | 3 +- 32 files changed, 571 insertions(+), 61 deletions(-) create mode 100644 audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/.github/workflows/pr-title.yml create mode 100644 audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/.pre-commit-config.yaml create mode 100644 audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/.prettierrc.json create mode 100644 audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/eslint.config.js create mode 100644 audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/package.json create mode 100644 audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/tsconfig.json create mode 100644 audit/engine/tests/fixtures/tool-06-local-only-commit-rule/.pre-commit-config.yaml create mode 100644 audit/engine/tests/fixtures/tool-06-local-only-commit-rule/.prettierrc.json create mode 100644 audit/engine/tests/fixtures/tool-06-local-only-commit-rule/commitlint.config.js create mode 100644 audit/engine/tests/fixtures/tool-06-local-only-commit-rule/eslint.config.js create mode 100644 audit/engine/tests/fixtures/tool-06-local-only-commit-rule/package.json create mode 100644 audit/engine/tests/fixtures/tool-06-local-only-commit-rule/tsconfig.json create mode 100644 audit/engine/tests/fixtures/tool-ok-node/.github/workflows/commits.yml create mode 100644 audit/engine/tests/test_tool06_commit_rule_in_ci.py create mode 100644 dev/eval/expected/fixtures/tool-06-ci-only-commit-rule.json create mode 100644 dev/eval/expected/fixtures/tool-06-local-only-commit-rule.json diff --git a/.secrets.baseline b/.secrets.baseline index b49ceb1e..c819e3a3 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -270,7 +270,7 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/facts.golden.git.json", - "hashed_secret": "b203f82e7569fb976aef75bf62f09850e6fa0d03", + "hashed_secret": "ab03a43c030d306a41b4dad11caa40429d3f76a0", "is_verified": false, "line_number": 5 } @@ -1409,6 +1409,24 @@ "line_number": 4 } ], + "dev/eval/expected/fixtures/tool-06-ci-only-commit-rule.json": [ + { + "type": "Hex High Entropy String", + "filename": "dev/eval/expected/fixtures/tool-06-ci-only-commit-rule.json", + "hashed_secret": "95fda17799f0315d0404bbf62e8a12fa506a6236", + "is_verified": false, + "line_number": 4 + } + ], + "dev/eval/expected/fixtures/tool-06-local-only-commit-rule.json": [ + { + "type": "Hex High Entropy String", + "filename": "dev/eval/expected/fixtures/tool-06-local-only-commit-rule.json", + "hashed_secret": "af89bb14c7cbc83db4e802247480abc133d72f98", + "is_verified": false, + "line_number": 4 + } + ], "dev/eval/expected/fixtures/tool-06-no-commit-rule.json": [ { "type": "Hex High Entropy String", @@ -1422,7 +1440,7 @@ { "type": "Hex High Entropy String", "filename": "dev/eval/expected/fixtures/tool-ok-node.json", - "hashed_secret": "fabd60da48166178c638970a98c60c25e3336345", + "hashed_secret": "d626ea1b547da9a9b1ab661c73aacb88a909cc32", "is_verified": false, "line_number": 4 } @@ -1519,5 +1537,5 @@ } ] }, - "generated_at": "2026-11-09T08:43:23Z" + "generated_at": "2026-11-09T11:43:43Z" } diff --git a/audit/checks/05-tooling.md b/audit/checks/05-tooling.md index 6d2cc7fe..eda311b1 100644 --- a/audit/checks/05-tooling.md +++ b/audit/checks/05-tooling.md @@ -194,30 +194,35 @@ Why it matters Release automation reads commit messages; without an enforced c and changelog generation silently stop working ([greenfield/README.md § 2.3 Git Hooks](../../greenfield/README.md#23-git-hooks--husky--lint-staged--commitlint)). Severity COSMETIC -Engine `TOOL-06`. Reads for a commitlint config file (`commitlint.config.*`, `.commitlintrc*`, or - `package.json`'s `"commitlint"` key), a `commit-msg` hook file under `.husky/` or - `.githooks/`, or a `commit-msg`/`commitlint`/`conventional` reference inside - `.pre-commit-config.yaml`/`lefthook.yml`/`.yaml`. Any found is `PASS`; none found, with a - detected ecosystem or any hook wiring already present, is `FAIL`. Distrust a `FAIL` when - the convention is enforced through a server-side hook or a CI-only check this check does - not read (CI enforcement is `TOOL-06`'s own blind spot, not a reason to call the finding - wrong without checking CI first) — verify by reading the CI workflow. +Engine `TOOL-06`. Reads two places. Locally: a commitlint config file (`commitlint.config.*`, + `.commitlintrc*`, or `package.json`'s `"commitlint"` key), a `commit-msg` hook file under + `.husky/` or `.githooks/`, or a `commit-msg`/`commitlint`/`conventional` reference inside + `.pre-commit-config.yaml`/`lefthook.yml`/`.yaml`. In CI (every workflow file of a detected + provider): a line that runs commitlint (`npx commitlint`, `wagoid/commitlint-github-action`), + `cz check`, `gitlint`, `pre-commit run --hook-stage commit-msg` or `lefthook run commit-msg`; a + pull-request title check (`amannn/action-semantic-pull-request`); a repository script that the + local `commit-msg` stage also runs; or a runner target (`make commit-lint`, `npm run …`) whose + command does one of those. An install line (`@commitlint/cli`) and a comment do not count. + Distrust a `PARTIAL` or `FAIL` when CI checks commits through a reusable workflow, a + server-side hook or a script the local stage does not name — verify by reading the CI + workflow. The engine does not read whether the job runs on pull requests. Method automated Look for The commitlint config file or `package.json` key; the hook files under `.husky/` and - `.githooks/`; the hook-manager config's own text for a commit-message stage. -Rubric PASS: a commitlint config, a `commit-msg` hook file, or an equivalent pre-commit/lefthook - stage is found. - PARTIAL: not used by this check (the engine's outcome is binary: the rule exists locally - or it does not). - FAIL: none of the above is found and the repository has a detected ecosystem, or any hook - chain (wired or merely configured) is already present. + `.githooks/`; the hook-manager config's own text for a commit-message stage; the workflow line + that runs the same rule. +Rubric PASS: a local rule (commitlint config, `commit-msg` hook file or equivalent pre-commit/lefthook + stage) and a CI step that checks commit messages or the pull-request title are both found. + PARTIAL: only one of the two is found. Local only: `--no-verify` or a missing hook install + skips the rule. CI only: an agent learns of a bad subject only after it pushes. + FAIL: neither is found and the repository has a detected ecosystem, or any hook chain (wired + or merely configured) is already present. NOT_APPLICABLE: never. UNKNOWN when: no recognised ecosystem and no hook configuration exists at all, so a commit convention cannot be judged either way. Evidence to cite A `file` evidence item citing the commitlint config, the `commit-msg` hook file, or the - hook-manager config line that matched. + hook-manager config line that matched; for CI only, the workflow line and what it runs. Absence proof A FAIL cites every runner manifest or detected ecosystem signal as the evidence that no - commitlint config, `commit-msg` hook, or equivalent stage was found. + commitlint config, `commit-msg` hook, equivalent stage or CI step was found. Remediation `add-hooks-and-commit-rules` Pitfalls `COSMETIC` severity means this check never raises a repository's audit to BLOCKER on its own — do not treat a `TOOL-06` `FAIL` as equivalent in urgency to a `DEGRADER` finding diff --git a/audit/engine/audit/__init__.py b/audit/engine/audit/__init__.py index ff1f2955..d1a16cce 100644 --- a/audit/engine/audit/__init__.py +++ b/audit/engine/audit/__init__.py @@ -24,7 +24,7 @@ #: Bumped whenever a rule pack, a check, or the instruction-variant table changes. #: Recorded in every report's provenance block so a report can be attributed to an #: exact catalogue revision (docs: Scan Engine & Tiering, § 7 Determinism). -__ruleset_revision__ = "2026-11-09.3" +__ruleset_revision__ = "2026-11-09.4" #: Highest framework phase this build evaluates. Phase 1 only in v1. __phase_ceiling__ = 1 diff --git a/audit/engine/audit/rules/checks/_common.py b/audit/engine/audit/rules/checks/_common.py index 15311ae3..848540d3 100644 --- a/audit/engine/audit/rules/checks/_common.py +++ b/audit/engine/audit/rules/checks/_common.py @@ -183,6 +183,24 @@ def makefile_errors(inventory: Inventory) -> tuple[MakeError, ...]: _PROMPT_RE = re.compile(r"^\s*(?:\$|>|%|❯|PS>)\s+") +#: Runner indirection: how a command names a verb/script through its runner. +_NPM_RE = re.compile(r"\b(?:npm|pnpm|bun)\s+(?:run\s+)?([A-Za-z0-9:_.-]+)") +_YARN_RE = re.compile(r"\byarn\s+(?:run\s+)?([A-Za-z0-9:_.-]+)") +_TASK_RE = re.compile(r"\b(?:make|task|just|mise)\s+(?:-\S+\s+)*([A-Za-z0-9:_.-]+)") +_GRADLE_RE = re.compile(r"\b(?:\./)?(?:gradlew|gradle)\s+([A-Za-z0-9:_.-]+)") +_RUNNER_TARGET_RES = (_NPM_RE, _YARN_RE, _TASK_RE, _GRADLE_RE) + + +def runner_target(command: str) -> str | None: + """The script or target a command runs through its runner (``npm run x``, ``make x``, ``./gradlew x``), else + ``None``.""" + for pattern in _RUNNER_TARGET_RES: + match = pattern.search(command) + if match: + return match.group(1) + return None + + def strip_shell_comment(line: str) -> str: """``line`` without a trailing shell comment. diff --git a/audit/engine/audit/rules/checks/ci.py b/audit/engine/audit/rules/checks/ci.py index d5c4eebe..cce3a9a1 100644 --- a/audit/engine/audit/rules/checks/ci.py +++ b/audit/engine/audit/rules/checks/ci.py @@ -57,7 +57,7 @@ from audit.evaluate import CheckOutcome from audit.findings import Evidence, Finding, Verdict, statement from audit.probes import MUTATING_VERBS -from audit.rules.checks._common import resolve_verbs, runner_commands +from audit.rules.checks._common import resolve_verbs, runner_commands, runner_target from audit.stack import CI_PROVIDERS, VERBS from audit.rules.payloads import Payload from audit.scan import Kind @@ -78,13 +78,6 @@ _BLOCK_SCALAR_VALUES = frozenset({"|", "|-", "|+", ">", ">-", ">+", ""}) _COMMAND_SPLIT_RE = re.compile(r"\s*(?:&&|\|\||;|\|)\s*") -#: Runner indirection: how a command names a verb/script through its runner. -_NPM_RE = re.compile(r"\b(?:npm|pnpm|bun)\s+(?:run\s+)?([A-Za-z0-9:_.-]+)") -_YARN_RE = re.compile(r"\byarn\s+(?:run\s+)?([A-Za-z0-9:_.-]+)") -_TASK_RE = re.compile(r"\b(?:make|task|just|mise)\s+(?:-\S+\s+)*([A-Za-z0-9:_.-]+)") -_GRADLE_RE = re.compile(r"\b(?:\./)?(?:gradlew|gradle)\s+([A-Za-z0-9:_.-]+)") -_RUNNER_TARGET_RES = (_NPM_RE, _YARN_RE, _TASK_RE, _GRADLE_RE) - #: Direct tool invocations, mapped to the framework verb they satisfy. `format:check` precedes #: `format` and `security` precedes the rest so the more specific form wins. _DIRECT_VERB_PATTERNS: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple( @@ -378,17 +371,9 @@ def _iter_run_commands(text: str) -> list[tuple[int, str]]: return commands -def _runner_target(command: str) -> str | None: - for pattern in _RUNNER_TARGET_RES: - match = pattern.search(command) - if match: - return match.group(1) - return None - - def _command_verb(command: str) -> str | None: """The framework verb a command invokes, by runner indirection or direct tool, else ``None``.""" - target = _runner_target(command) + target = runner_target(command) if target in VERBS: return target for verb, patterns in _DIRECT_VERB_PATTERNS: @@ -517,7 +502,7 @@ def check_ci03(*, spec, target, inventory, stack, components, session) -> CheckO for rel, line, command in _all_run_commands(inventory, _workflows(inventory)): if _command_verb(command) is not None: continue # CI-02's subject, not repeated here - target = _runner_target(command) + target = runner_target(command) if target is None or target in _PM_BUILTINS or target in local_targets: continue findings.append( diff --git a/audit/engine/audit/rules/checks/tool.py b/audit/engine/audit/rules/checks/tool.py index a45bf6dd..80c35839 100644 --- a/audit/engine/audit/rules/checks/tool.py +++ b/audit/engine/audit/rules/checks/tool.py @@ -28,8 +28,12 @@ * **TOOL-05** — a wired hook chain: `.pre-commit-config.yaml`, `lefthook.yml`, a declared `core.hooksPath`, or `.husky/` referenced by a `prepare`/install script. Present but unwired ⇒ `PARTIAL`. -* **TOOL-06** — a commit-message rule: a commitlint config, a `commit-msg` hook, or an equivalent - pre-commit/lefthook stage. Absent ⇒ `FAIL` (`COSMETIC`). +* **TOOL-06** — a commit-message rule in two places. Locally: a commitlint config, a `commit-msg` hook, or an + equivalent pre-commit/lefthook stage. In CI: a workflow line that runs a commit-message linter (commitlint, + commitizen's `cz check`, gitlint, the pre-commit or lefthook `commit-msg` stage), a pull-request title check, a + script the local `commit-msg` stage also runs, or a runner target whose command does one of those. Both ⇒ + `PASS`; one of the two ⇒ `PARTIAL`; neither ⇒ `FAIL` (`COSMETIC`). Whether the CI job runs on pull requests is + not read. Deliberate limits, recorded rather than hidden. **Config detection is filename-based** against the documented set below; a config that lives somewhere unexpected is not seen. **Verb resolution is @@ -45,9 +49,9 @@ from audit.evaluate import CheckOutcome from audit.findings import Evidence, Finding, Verdict, statement -from audit.rules.checks._common import resolve_verbs, runner_commands +from audit.rules.checks._common import resolve_verbs, runner_commands, runner_target from audit.scan import Kind -from audit.stack import extract_npm_scripts +from audit.stack import CI_PROVIDERS, extract_npm_scripts from audit.rules.payloads import Payload #: Formatter configuration filenames, across the ecosystems the catalogue names. @@ -131,6 +135,24 @@ #: Hook managers whose mere presence is a wired chain. _WIRED_HOOK_FILES: tuple[str, ...] = (".pre-commit-config.yaml", "lefthook.yml", "lefthook.yaml") +#: TOOL-06: a CI line that checks commit messages or the pull-request title, by tool. ``@commitlint/cli`` (an +#: install) is not a run; ``wagoid/commitlint-github-action`` is. +_CI_COMMIT_TOOLS: tuple[tuple[str, re.Pattern[str]], ...] = tuple( + (name, re.compile(pattern)) + for name, pattern in ( + ("commitlint", r"(? Chec # =========================================================================== +def _commit_msg_text(inventory) -> list[str]: + """The lines of every local `commit-msg` stage: the hook files, lefthook's `commit-msg:` block, and each + pre-commit hook item whose text names the `commit-msg` stage.""" + lines: list[str] = [] + for rel in (".husky/commit-msg", ".githooks/commit-msg"): + lines.extend((inventory.read(rel) or "").splitlines()) + for rel in ("lefthook.yml", "lefthook.yaml"): + inside = False + for line in (inventory.read(rel) or "").splitlines(): + if line and not line[0].isspace() and not line.startswith("#"): + inside = line.split(":", 1)[0].strip().strip("'\"") == "commit-msg" + continue + if inside: + lines.append(line) + item: list[str] = [] + for line in [*(inventory.read(".pre-commit-config.yaml") or "").splitlines(), "- id: "]: + if re.match(r"^\s*-\s+(?:id|repo)\s*:", line): + if any("commit-msg" in text for text in item): + lines.extend(item) + item = [] + item.append(line) + return lines + + +def _commit_msg_scripts(inventory) -> list[str]: + """Repository scripts a local `commit-msg` stage runs, by path, so CI can be seen to run the same rule.""" + paths = set(inventory.paths()) + scripts: set[str] = set() + for line in _commit_msg_text(inventory): + for token in _PATH_TOKEN_RE.findall(line): + rel = token.removeprefix("./") + if rel in paths and rel.endswith(_SCRIPT_SUFFIXES): + scripts.add(rel) + return sorted(scripts) + + +def _commit_rule_in(text: str, scripts: list[str]) -> str | None: + """What in ``text`` checks commit messages: a known tool, or a script the local stage runs.""" + for name, pattern in _CI_COMMIT_TOOLS: + if pattern.search(text): + return name + for rel in scripts: + if re.search(r"(? list[tuple[str, int, str]]: + """``(workflow, line, what)`` for each workflow line that checks commit messages or the pull-request title, + directly or through a runner target (``make commit-lint``, ``npm run commitlint``).""" + targets: dict[str, str] = {} + for _runner, commands in reversed(runner_commands(inventory)): + targets.update(commands) + hits: list[tuple[str, int, str]] = [] + for globs in CI_PROVIDERS.values(): + for rel in sorted(inventory.match(*globs)): + for number, line in enumerate((inventory.read(rel) or "").splitlines(), start=1): + if line.lstrip().startswith("#"): + continue + what = _commit_rule_in(line, scripts) + target = runner_target(line) + if what is None and target in targets: + via = _commit_rule_in(targets[target], scripts) + what = f"{target} ({via})" if via else None + if what: + hits.append((rel, number, what)) + return hits + + def check_tool06(*, spec, target, inventory, stack, components, session) -> CheckOutcome: - rules = _commit_rule(inventory) + rules = sorted(set(_commit_rule(inventory))) + ci_steps = _ci_commit_steps(inventory, _commit_msg_scripts(inventory)) + ci_seen = ", ".join(sorted({f"{what} ({rel})" for rel, _line, what in ci_steps})) + detail = f"local: {', '.join(rules) or 'none'}; CI: {ci_seen or 'none'}" + + if rules and ci_steps: + return _outcome(spec, Verdict.PASS, detail) if rules: - return _outcome(spec, Verdict.PASS, f"commit rule: {', '.join(sorted(set(rules)))}") + return _outcome( + spec, + Verdict.PARTIAL, + detail, + [ + _finding( + spec, + "rely on every commit following the convention", + "the commit-message rule runs only in a local hook, which `--no-verify` or a missing install " + "skips, and no CI step checks it", + Verdict.PARTIAL, + [Evidence(rel.split(" ")[0].split(":")[0]) for rel in rules], + "Run the same commit-message rule in CI over the commits a pull request adds (or check the " + "pull-request title when the repository squash-merges).", + ) + ], + ) + if ci_steps: + return _outcome( + spec, + Verdict.PARTIAL, + detail, + [ + _finding( + spec, + "learn that a commit message breaks the convention before it pushes", + "the commit-message rule runs only in CI; no local `commit-msg` stage runs it", + Verdict.PARTIAL, + [Evidence(rel, line=line, note=what) for rel, line, what in ci_steps], + "Run the same rule in a local `commit-msg` hook, with the type list CI uses.", + ) + ], + ) wired, configured = _hook_wiring(inventory) if stack.ecosystems or wired or configured: @@ -756,10 +885,10 @@ def check_tool06(*, spec, target, inventory, stack, components, session) -> Chec _finding( spec, "rely on a consistent commit-message convention", - "no commitlint config, commit-msg hook, or equivalent exists", + "no commitlint config, commit-msg hook, equivalent stage or CI step exists", Verdict.FAIL, _manifest_evidence(inventory), - "Add a commitlint config or a commit-msg hook so commit messages follow one convention.", + "Add a commit-message rule to a commit-msg hook and run the same rule in CI.", ) ], ) diff --git a/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md b/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md index 2baef3a2..bacbf6c3 100644 --- a/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md +++ b/audit/engine/docs/architecture/contributor-deep-dive/02-check-catalogue.md @@ -117,7 +117,7 @@ suspicious verdict. | `TOOL-03` | Type checker configured | A | DEGRADER | Language-appropriate strict configuration (`tsconfig`, `mypy`/`pyright`, `global.json` nullable settings, etc.) | Implemented | | `TOOL-04` | Dependency / security scan verb | A | DEGRADER | A resolvable verb performing a dependency-audit check | Implemented | | `TOOL-05` | Local hook chain active | B | DEGRADER | Hook manager configured and wired (`core.hooksPath`, `.pre-commit-config.yaml`, `lefthook.yml`) | Implemented | -| `TOOL-06` | Commit convention enforced | A | COSMETIC | A commit-message hook or equivalent exists | Implemented | +| `TOOL-06` | Commit convention enforced | A | COSMETIC | A commit-message rule runs in a local `commit-msg` stage and in CI | Implemented | --- diff --git a/audit/engine/docs/reference/changelog.md b/audit/engine/docs/reference/changelog.md index 8e6c8ceb..256a7649 100644 --- a/audit/engine/docs/reference/changelog.md +++ b/audit/engine/docs/reference/changelog.md @@ -13,6 +13,7 @@ the record of how; git has the rest. | Revision | Date | Scoreable checks | What it is | |---|---|---|---| +| `2026-11-09.4` | 2026-11-09 | 57 | v0.3.3 C4: `TOOL-06` (commit convention, COSMETIC) also reads CI. A local rule and a CI step (commitlint, `cz check`, gitlint, the pre-commit or lefthook `commit-msg` stage, a pull-request title check, the local stage's own script, or a runner target that runs one) ⇒ `PASS`; only one of the two ⇒ `PARTIAL`; neither ⇒ `FAIL`. A repository with only a local rule drops from `PASS` to `PARTIAL`; one with only a CI check rises from `FAIL` to `PARTIAL` | | `2026-11-09.3` | 2026-11-09 | 57 | v0.3.3 C3: new check `EXEC-08` (tool versions visible to dependency automation, DEGRADER). It reads the command runners, the CI workflows and the hook files for a tool version written as a literal (`uvx ruff@0.5.0`, `npx x@1.2.3`, `pip install x==1.2`, `go install pkg@v1.2`, `cargo install --version`, a tool-installer action's `version` input): none ⇒ `PASS`, every literal also in a manifest ⇒ `PARTIAL`, any literal no manifest holds ⇒ `FAIL`, no such file ⇒ `UNKNOWN`. A file a Renovate custom manager reads is visible. Every repository with an inline tool version gains an `EXEC-08` finding | | `2026-11-09.2` | 2026-11-09 | 56 | v0.3.3 C2: `HYG-09` (informational, unscored) reads the Dependabot or Renovate file. `PARTIAL` when a detected ecosystem (from a manifest, plus `github-actions` with GitHub CI) has no entry, or when a Dependabot entry sets `target-branch` together with `ignore` or `groups`; `UNKNOWN` when the file cannot be read. The update policy itself is the owner's choice and never a verdict. No score moves | | `2026-11-09.1` | 2026-11-09 | 56 | v0.3.3 C1: new check `CI-05` (workflow files are linted, DEGRADER). It reads the linters each CI provider supports (actionlint, zizmor, check-jsonschema schemas, `glab ci lint`, `circleci config validate`): local ⇒ `PASS`, CI only ⇒ `PARTIAL`, none ⇒ `FAIL`, no workflow or Jenkins only ⇒ `UNKNOWN`. Every repository with workflow files and no linter gains a `CI-05` finding | diff --git a/audit/engine/tests/fixtures/README.md b/audit/engine/tests/fixtures/README.md index 90065bb9..8fc0a075 100644 --- a/audit/engine/tests/fixtures/README.md +++ b/audit/engine/tests/fixtures/README.md @@ -90,7 +90,9 @@ fixtures/ tool-04-no-audit-verb/ node project, no dependency-audit verb -> TOOL-04 FAIL tool-05-unwired-hooks/ .husky/ present but never referenced -> TOOL-05 PARTIAL tool-06-no-commit-rule/ wired hooks, no commit-message rule -> TOOL-06 FAIL - tool-ok-node/ the smallest node repository passing every TOOL check + tool-06-local-only-commit-rule/ commit-msg hook, no CI step -> TOOL-06 PARTIAL + tool-06-ci-only-commit-rule/ PR-title check in CI, no hook -> TOOL-06 PARTIAL + tool-ok-node/ the smallest node repository passing every TOOL check (hook and CI step) ci-01-no-pr-trigger/ workflow without a pull-request trigger -> CI-01 PARTIAL ci-02-parity-break/ CI runs a framework verb with no local script -> CI-02 FAIL ci-03-ci-only-step/ CI runs `npm run deploy` with no local script -> CI-03 FAIL diff --git a/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/.github/workflows/pr-title.yml b/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/.github/workflows/pr-title.yml new file mode 100644 index 00000000..8739639d --- /dev/null +++ b/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/.github/workflows/pr-title.yml @@ -0,0 +1,11 @@ +name: pr-title +on: + pull_request: + types: [opened, edited, synchronize] +permissions: + pull-requests: read +jobs: + title: + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v5 diff --git a/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/.pre-commit-config.yaml b/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/.pre-commit-config.yaml new file mode 100644 index 00000000..c11041f7 --- /dev/null +++ b/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/.pre-commit-config.yaml @@ -0,0 +1,8 @@ +repos: + - repo: local + hooks: + - id: prettier + name: prettier + entry: npx prettier --check . + language: system + types: [text] diff --git a/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/.prettierrc.json b/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/.prettierrc.json new file mode 100644 index 00000000..544138be --- /dev/null +++ b/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/.prettierrc.json @@ -0,0 +1,3 @@ +{ + "singleQuote": true +} diff --git a/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/eslint.config.js b/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/eslint.config.js new file mode 100644 index 00000000..d6d1738d --- /dev/null +++ b/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/eslint.config.js @@ -0,0 +1 @@ +export default []; diff --git a/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/package.json b/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/package.json new file mode 100644 index 00000000..3e0fc289 --- /dev/null +++ b/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/package.json @@ -0,0 +1,10 @@ +{ + "name": "tool-06-ci-only-commit-rule", + "version": "0.1.0", + "private": true, + "scripts": { + "format": "prettier --write .", + "lint": "eslint .", + "security": "npm audit --audit-level=high" + } +} diff --git a/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/tsconfig.json b/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/tsconfig.json new file mode 100644 index 00000000..aee0ec94 --- /dev/null +++ b/audit/engine/tests/fixtures/tool-06-ci-only-commit-rule/tsconfig.json @@ -0,0 +1,5 @@ +{ + "compilerOptions": { + "strict": true + } +} diff --git a/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/.pre-commit-config.yaml b/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/.pre-commit-config.yaml new file mode 100644 index 00000000..93c5803e --- /dev/null +++ b/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: + - repo: local + hooks: + - id: prettier + name: prettier + entry: npx prettier --check . + language: system + types: [text] + - id: commitlint + name: commitlint + entry: npx commitlint --edit + language: system + stages: [commit-msg] diff --git a/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/.prettierrc.json b/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/.prettierrc.json new file mode 100644 index 00000000..544138be --- /dev/null +++ b/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/.prettierrc.json @@ -0,0 +1,3 @@ +{ + "singleQuote": true +} diff --git a/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/commitlint.config.js b/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/commitlint.config.js new file mode 100644 index 00000000..3f5e287f --- /dev/null +++ b/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/commitlint.config.js @@ -0,0 +1 @@ +export default { extends: ['@commitlint/config-conventional'] }; diff --git a/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/eslint.config.js b/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/eslint.config.js new file mode 100644 index 00000000..d6d1738d --- /dev/null +++ b/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/eslint.config.js @@ -0,0 +1 @@ +export default []; diff --git a/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/package.json b/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/package.json new file mode 100644 index 00000000..f535a021 --- /dev/null +++ b/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/package.json @@ -0,0 +1,10 @@ +{ + "name": "tool-06-local-only-commit-rule", + "version": "0.1.0", + "private": true, + "scripts": { + "format": "prettier --write .", + "lint": "eslint .", + "security": "npm audit --audit-level=high" + } +} diff --git a/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/tsconfig.json b/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/tsconfig.json new file mode 100644 index 00000000..aee0ec94 --- /dev/null +++ b/audit/engine/tests/fixtures/tool-06-local-only-commit-rule/tsconfig.json @@ -0,0 +1,5 @@ +{ + "compilerOptions": { + "strict": true + } +} diff --git a/audit/engine/tests/fixtures/tool-ok-node/.github/workflows/commits.yml b/audit/engine/tests/fixtures/tool-ok-node/.github/workflows/commits.yml new file mode 100644 index 00000000..5a9bf827 --- /dev/null +++ b/audit/engine/tests/fixtures/tool-ok-node/.github/workflows/commits.yml @@ -0,0 +1,14 @@ +name: commits +on: + pull_request: +permissions: + contents: read +jobs: + commits: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: npm ci + - run: npx commitlint --from "${{ github.event.pull_request.base.sha }}" --to "${{ github.event.pull_request.head.sha }}" diff --git a/audit/engine/tests/test_fixture_matrix.py b/audit/engine/tests/test_fixture_matrix.py index 94e8d395..56c56290 100644 --- a/audit/engine/tests/test_fixture_matrix.py +++ b/audit/engine/tests/test_fixture_matrix.py @@ -100,6 +100,8 @@ "tool-04-no-audit-verb": {"TOOL-04": "FAIL"}, "tool-05-unwired-hooks": {"TOOL-05": "PARTIAL"}, "tool-06-no-commit-rule": {"TOOL-06": "FAIL"}, + "tool-06-local-only-commit-rule": {"TOOL-06": "PARTIAL"}, + "tool-06-ci-only-commit-rule": {"TOOL-06": "PARTIAL"}, "tool-ok-node": { "TOOL-01": "PASS", "TOOL-02": "PASS", diff --git a/audit/engine/tests/test_tool06_commit_rule_in_ci.py b/audit/engine/tests/test_tool06_commit_rule_in_ci.py new file mode 100644 index 00000000..b7bcfc8a --- /dev/null +++ b/audit/engine/tests/test_tool06_commit_rule_in_ci.py @@ -0,0 +1,231 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# Copyright (C) 2026 K. C. Ramakrishna + +"""`TOOL-06`: the commit-message rule runs in a local `commit-msg` stage and in CI. + +Both ⇒ `PASS`; only one of the two ⇒ `PARTIAL`; neither ⇒ `FAIL`. CI is seen through a known tool on a workflow +line, a pull-request title check, a script the local stage also runs, or a runner target whose command does one +of those. The check runs in-process here, so coverage traces it. +""" + +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import support # noqa: E402 + +from audit.rules.checks.tool import check_tool06 as CHECK # noqa: E402 +from audit.rules.registry import REGISTRY_BY_ID # noqa: E402 +from audit.scan import build_inventory # noqa: E402 +from audit.stack import detect_stack # noqa: E402 + +CHECK_ID = "TOOL-06" + +PACKAGE = '{"name": "x", "version": "0.1.0", "private": true}\n' +SCRIPT = "import sys\nsys.exit(0)\n" + +LEFTHOOK_WITH_SCRIPT = """pre-commit: + commands: + format: + run: scripts/format_check.py {staged_files} +commit-msg: + commands: + subject: + run: python3 scripts/commit_lint.py --message-file {1} +""" + + +def workflow(*runs: str) -> str: + steps = "".join(f" - run: {run}\n" for run in runs) + return ( + "name: ci\non:\n pull_request:\njobs:\n commits:\n runs-on: ubuntu-latest\n steps:\n" + " - uses: actions/checkout@v4\n" + steps + ) + + +class Tool06CommitRuleInCiTests(unittest.TestCase): + def check(self, files: dict[str, str]) -> dict: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for rel, text in files.items(): + support.write(root, rel, text) + inventory = build_inventory(root) + outcome = CHECK( + spec=REGISTRY_BY_ID[CHECK_ID], + target=None, + inventory=inventory, + stack=detect_stack(inventory), + components=None, + session=None, + ) + return { + "verdict": outcome.verdict.value, + "summary": outcome.summary, + "findings": [finding.to_dict() for finding in outcome.findings], + } + + # -- PASS: the rule runs in both places ----------------------------------------------------------- + + def test_the_hook_script_run_again_in_ci_passes(self): + result = self.check( + { + "package.json": PACKAGE, + "lefthook.yml": LEFTHOOK_WITH_SCRIPT, + "scripts/commit_lint.py": SCRIPT, + "scripts/format_check.py": SCRIPT, + ".github/workflows/ci.yml": workflow('python3 scripts/commit_lint.py --range "$BASE..$HEAD"'), + } + ) + self.assertEqual(result["verdict"], "PASS") + self.assertIn("CI: scripts/commit_lint.py (.github/workflows/ci.yml)", result["summary"]) + + def test_a_make_target_that_runs_the_hook_script_passes(self): + result = self.check( + { + "Makefile": "commit-lint:\n\tpython3 scripts/commit_lint.py --range $(RANGE)\n", + "lefthook.yml": LEFTHOOK_WITH_SCRIPT, + "scripts/commit_lint.py": SCRIPT, + "scripts/format_check.py": SCRIPT, + ".github/workflows/ci.yml": workflow("make commit-lint RANGE=origin/main..HEAD"), + } + ) + self.assertEqual(result["verdict"], "PASS") + self.assertIn("commit-lint (scripts/commit_lint.py)", result["summary"]) + + def test_an_npm_script_that_runs_commitlint_passes(self): + result = self.check( + { + "package.json": '{"name": "x", "scripts": {"lint:commits": "commitlint --from origin/main"}}\n', + ".husky/commit-msg": 'npx --no -- commitlint --edit "$1"\n', + "commitlint.config.js": "export default { extends: ['@commitlint/config-conventional'] };\n", + ".github/workflows/ci.yml": workflow("npm run lint:commits"), + } + ) + self.assertEqual(result["verdict"], "PASS") + self.assertIn("lint:commits (commitlint)", result["summary"]) + + def test_gitlint_in_gitlab_ci_with_a_pre_commit_hook_passes(self): + result = self.check( + { + "pyproject.toml": '[project]\nname = "x"\nversion = "0.1.0"\n', + ".pre-commit-config.yaml": ( + "repos:\n - repo: https://github.com/jorisroovers/gitlint\n rev: v0.19.1\n hooks:\n" + " - id: gitlint\n stages: [commit-msg]\n" + ), + ".gitlab-ci.yml": ( + "commits:\n rules:\n - if: $CI_PIPELINE_SOURCE == 'merge_request_event'\n script:\n" + ' - gitlint --commits "$CI_MERGE_REQUEST_DIFF_BASE_SHA..$CI_COMMIT_SHA"\n' + ), + } + ) + self.assertEqual(result["verdict"], "PASS") + self.assertIn("gitlint (.gitlab-ci.yml)", result["summary"]) + + def test_commitizen_check_in_ci_passes(self): + result = self.check( + { + "pyproject.toml": '[project]\nname = "x"\nversion = "0.1.0"\n', + ".pre-commit-config.yaml": ( + "repos:\n - repo: https://github.com/commitizen-tools/commitizen\n rev: v4.0.0\n" + " hooks:\n - id: commitizen\n stages: [commit-msg]\n" + ), + ".github/workflows/ci.yml": workflow('cz check --rev-range "$BASE..$HEAD"'), + } + ) + self.assertEqual(result["verdict"], "PASS") + + # -- PARTIAL: only one of the two ------------------------------------------------------------------ + + def test_a_local_hook_with_no_ci_step_is_partial(self): + result = self.check({"package.json": PACKAGE, ".husky/commit-msg": 'npx --no -- commitlint --edit "$1"\n'}) + self.assertEqual(result["verdict"], "PARTIAL") + self.assertIn("runs only in a local hook", result["findings"][0]["statement"]) + self.assertEqual([e["path"] for e in result["findings"][0]["evidence"]], [".husky/commit-msg"]) + + def test_installing_commitlint_in_ci_is_not_running_it(self): + result = self.check( + { + "package.json": PACKAGE, + ".husky/commit-msg": 'npx --no -- commitlint --edit "$1"\n', + ".github/workflows/ci.yml": workflow("npm install --no-save @commitlint/cli"), + } + ) + self.assertEqual(result["verdict"], "PARTIAL") + + def test_a_commented_out_ci_step_does_not_count(self): + result = self.check( + { + "package.json": PACKAGE, + ".husky/commit-msg": 'npx --no -- commitlint --edit "$1"\n', + ".github/workflows/ci.yml": workflow("npm test") + " # - run: npx commitlint --from main\n", + } + ) + self.assertEqual(result["verdict"], "PARTIAL") + + def test_a_script_from_another_hook_stage_does_not_count(self): + result = self.check( + { + "package.json": PACKAGE, + "lefthook.yml": LEFTHOOK_WITH_SCRIPT, + "scripts/commit_lint.py": SCRIPT, + "scripts/format_check.py": SCRIPT, + ".github/workflows/ci.yml": workflow("python3 scripts/format_check.py"), + } + ) + self.assertEqual(result["verdict"], "PARTIAL") + + def test_a_pre_commit_item_outside_the_commit_msg_stage_does_not_count(self): + result = self.check( + { + "package.json": PACKAGE, + ".pre-commit-config.yaml": ( + "repos:\n - repo: local\n hooks:\n - id: fmt\n entry: python3 scripts/fmt.py\n" + " language: system\n - id: subject\n entry: python3 scripts/subject.py\n" + " language: system\n stages: [commit-msg]\n" + ), + "scripts/fmt.py": SCRIPT, + "scripts/subject.py": SCRIPT, + ".github/workflows/ci.yml": workflow("python3 scripts/fmt.py"), + } + ) + self.assertEqual(result["verdict"], "PARTIAL") + self.assertIn("CI: none", result["summary"]) + + def test_a_ci_step_with_no_local_rule_is_partial(self): + result = self.check( + { + "package.json": PACKAGE, + ".github/workflows/title.yml": ( + "name: title\non:\n pull_request:\njobs:\n title:\n runs-on: ubuntu-latest\n" + " steps:\n - uses: amannn/action-semantic-pull-request@v5\n" + ), + } + ) + self.assertEqual(result["verdict"], "PARTIAL") + finding = result["findings"][0] + self.assertIn("runs only in CI", finding["statement"]) + self.assertEqual( + [(e["path"], e["line"], e["note"]) for e in finding["evidence"]], + [(".github/workflows/title.yml", 8, "pull-request title check")], + ) + + # -- FAIL and UNKNOWN: unchanged ------------------------------------------------------------------ + + def test_no_rule_anywhere_fails(self): + result = self.check({"package.json": PACKAGE, ".github/workflows/ci.yml": workflow("npm test")}) + self.assertEqual(result["verdict"], "FAIL") + self.assertIn("CI step", result["findings"][0]["statement"]) + + def test_nothing_to_judge_is_unknown(self): + result = self.check({"notes.txt": "hello\n"}) + self.assertEqual(result["verdict"], "UNKNOWN") + + +if __name__ == "__main__": + unittest.main() diff --git a/audit/engine/tests/test_tool_pack.py b/audit/engine/tests/test_tool_pack.py index 3c126ce7..c67a41d5 100644 --- a/audit/engine/tests/test_tool_pack.py +++ b/audit/engine/tests/test_tool_pack.py @@ -31,6 +31,8 @@ "tool-04-no-audit-verb": ("TOOL-04", "FAIL"), "tool-05-unwired-hooks": ("TOOL-05", "PARTIAL"), "tool-06-no-commit-rule": ("TOOL-06", "FAIL"), + "tool-06-local-only-commit-rule": ("TOOL-06", "PARTIAL"), + "tool-06-ci-only-commit-rule": ("TOOL-06", "PARTIAL"), } diff --git a/audit/schema/examples/facts.golden.git.json b/audit/schema/examples/facts.golden.git.json index ab8991d4..280dc134 100644 --- a/audit/schema/examples/facts.golden.git.json +++ b/audit/schema/examples/facts.golden.git.json @@ -2,12 +2,12 @@ "facts_version": "0.1", "label": "git-example", "git": { - "sha": "d0acc3aa52552759970e658d6cde4c5cb9897e85", + "sha": "26704539c7b7c54205f98ad260a919494bcc6c03", "dirty": true }, "inventory": { - "files": 7, - "bytes": 666, + "files": 8, + "bytes": 1006, "by_kind": { "manifest": 1, "build": 0, @@ -15,7 +15,7 @@ "source": 2, "test": 0, "doc": 1, - "ci": 0, + "ci": 1, "script": 0, "generated": 0, "binary": 0, @@ -32,7 +32,9 @@ ], "source_only": [], "package_managers": [], - "ci_providers": [], + "ci_providers": [ + "github" + ], "test_frameworks": [], "hook_managers": [ "pre-commit" @@ -49,8 +51,8 @@ }, "size": { "class": "small", - "files": 7, - "bytes": 666, + "files": 8, + "bytes": 1006, "truncated": false, "thresholds": { "small_below": 1000, diff --git a/audit/tools/tests/test_facts_stack.py b/audit/tools/tests/test_facts_stack.py index c2547487..0e539719 100644 --- a/audit/tools/tests/test_facts_stack.py +++ b/audit/tools/tests/test_facts_stack.py @@ -94,7 +94,7 @@ "tool-04-no-audit-verb": ("hook_managers", "husky"), "tool-05-unwired-hooks": ("hook_managers", "husky"), "tool-06-no-commit-rule": ("hook_managers", "pre-commit"), - "tool-ok-node": ("hook_managers", "pre-commit"), + "tool-06-local-only-commit-rule": ("hook_managers", "pre-commit"), } # Files that look like configuration of a CI provider, a test framework or a hook manager, and are not what the diff --git a/dev/eval/expected/fixtures/tool-06-ci-only-commit-rule.json b/dev/eval/expected/fixtures/tool-06-ci-only-commit-rule.json new file mode 100644 index 00000000..e9d38919 --- /dev/null +++ b/dev/eval/expected/fixtures/tool-06-ci-only-commit-rule.json @@ -0,0 +1,14 @@ +{ + "expected_version": "0.1", + "repo_label": "tool-06-ci-only-commit-rule", + "repo_sha": "77b64665fce027f489e1d7fe56fe10468e245347", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-06-ci-only-commit-rule' (audit/engine/tests/fixtures/tool-06-ci-only-commit-rule), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "checks": [ + { + "id": "TOOL-06", + "acceptable_verdicts": [ + "PARTIAL" + ] + } + ] +} diff --git a/dev/eval/expected/fixtures/tool-06-local-only-commit-rule.json b/dev/eval/expected/fixtures/tool-06-local-only-commit-rule.json new file mode 100644 index 00000000..24bd1ac3 --- /dev/null +++ b/dev/eval/expected/fixtures/tool-06-local-only-commit-rule.json @@ -0,0 +1,14 @@ +{ + "expected_version": "0.1", + "repo_label": "tool-06-local-only-commit-rule", + "repo_sha": "504396dc259eb5cce4d045ba697c3362fac34836", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-06-local-only-commit-rule' (audit/engine/tests/fixtures/tool-06-local-only-commit-rule), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "checks": [ + { + "id": "TOOL-06", + "acceptable_verdicts": [ + "PARTIAL" + ] + } + ] +} diff --git a/dev/eval/expected/fixtures/tool-ok-node.json b/dev/eval/expected/fixtures/tool-ok-node.json index 7625889b..207d6922 100644 --- a/dev/eval/expected/fixtures/tool-ok-node.json +++ b/dev/eval/expected/fixtures/tool-ok-node.json @@ -1,8 +1,8 @@ { "expected_version": "0.1", "repo_label": "tool-ok-node", - "repo_sha": "93289273697f4baade88c7547b89aa1d36f21a6c", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-ok-node' (brownfield-recipes/deterministic-audit/tests/fixtures/tool-ok-node), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "repo_sha": "851b8228d85e76be03f1c65b015f267f978e728f", + "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-ok-node' (audit/engine/tests/fixtures/tool-ok-node), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-01", diff --git a/docs/reference/audit-master.md b/docs/reference/audit-master.md index 38a73bc0..f43c7c85 100644 --- a/docs/reference/audit-master.md +++ b/docs/reference/audit-master.md @@ -366,7 +366,8 @@ Can an agent run one command that formats, lints, type-checks and tests, and tru `COSMETIC` · scored · engine check · fixed by [`add-hooks-and-commit-rules`](implement-master.md#add-hooks-and-commit-rules) -- **Passes when:** a commitlint config, a `commit-msg` hook file, or an equivalent pre-commit/lefthook stage is found. +- **Passes when:** a local rule (commitlint config, `commit-msg` hook file or equivalent pre-commit/lefthook stage) and + a CI step that checks commit messages or the pull-request title are both found. - **Why it matters:** Release automation reads commit messages; without an enforced convention, version bumps and changelog generation silently stop working ([greenfield/README.md § 2.3 Git Hooks](../../greenfield/README.md#23-git-hooks--husky--lint-staged--commitlint)). diff --git a/spec/readiness-requirements.md b/spec/readiness-requirements.md index 344c7828..5e664cdb 100644 --- a/spec/readiness-requirements.md +++ b/spec/readiness-requirements.md @@ -272,7 +272,8 @@ The repository MUST wire a local Git hook chain, through a hook manager or `core - **Checks:** TOOL-05 ### REQ-SH-032 — Commit convention enforced -The repository SHOULD enforce its commit-message convention with a `commit-msg` hook or an equivalent stage. +The repository SHOULD enforce its commit-message convention with a `commit-msg` hook or an equivalent stage, and +SHOULD check the same rule in CI. - **Phase:** 1 - **Verification:** automated From f429ae740b41d912bd3b9c5050259ceb4bd8758a Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 17:20:45 +0530 Subject: [PATCH 049/111] docs(implement): run the commit rule in a local stage and in CI add-hooks-and-commit-rules now writes one commit tool in two places with one type list, by stack: commitlint for Node, commitizen (or an existing gitlint) for Python, and a commit-lint script for mixed or Java repositories. It gives the CI step for GitHub Actions and GitLab CI, a trimmed copy of dev/commit_lint.py and a POSIX shell hook as templates, a CI verification row and the preference commit.tool. The title-only option reads PARTIAL now, not FAIL. conventional-pre-commit has no range form, so it is not the CI half. The Python, Node and Java profiles name the CI step. Roadmap-Step: C4 Co-Authored-By: Claude Opus 5.5 --- audit/stacks/java.md | 3 +- audit/stacks/python.md | 4 +- audit/stacks/typescript-node.md | 3 +- docs/reference/implement-master.md | 7 +- .../playbooks/add-hooks-and-commit-rules.md | 166 ++++++++++++++++-- 5 files changed, 163 insertions(+), 20 deletions(-) diff --git a/audit/stacks/java.md b/audit/stacks/java.md index 3d416b19..11bf08ab 100644 --- a/audit/stacks/java.md +++ b/audit/stacks/java.md @@ -43,7 +43,8 @@ Dependency-Check Gradle plugin (`org.owasp.dependencycheck`), registering `depen task name that does not match `security` literally, so a repository applying the plugin still reads `CMD-01` `FAIL` for that verb unless a `security` task delegates to it (see Pitfalls). Hook manager: none is standard; a committed `.githooks/` with `core.hooksPath` set by a build task, or lefthook (no JVM plugin -needed). Commit rule: a `commit-msg` script or lefthook stage. Dependency updates: Dependabot +needed). Commit rule: a `commit-msg` script or lefthook stage, with +the same script run over the pull request's commits in CI (`TOOL-06`). Dependency updates: Dependabot `package-ecosystem: gradle` or `maven`; Renovate's `gradle` and `maven` managers. ## Execution determinism diff --git a/audit/stacks/python.md b/audit/stacks/python.md index a57261a2..45bbbec0 100644 --- a/audit/stacks/python.md +++ b/audit/stacks/python.md @@ -33,7 +33,9 @@ Flake8/Pylint on an older one; config in `pyproject.toml`'s `[tool.ruff]`/`[tool `mypy.ini`'s `strict = true`. Test runner: pytest. Dependency scanner: `pip-audit` (works against any of pip/uv/poetry's resolved environment) or `safety check`. Hook manager: pre-commit (`.pre-commit-config.yaml`); lefthook in a mixed repository. Commit rule: a pre-commit `commit-msg` hook -(conventional-pre-commit, commitizen or gitlint). Dependency updates: Dependabot `package-ecosystem: pip` +(conventional-pre-commit, commitizen or gitlint), and the same rule in CI (`TOOL-06`): +`cz check --rev-range` or `gitlint --commits` over the pull request's commits (conventional-pre-commit has no +range form). Dependency updates: Dependabot `package-ecosystem: pip` (pip, pipenv, poetry, pip-compile) or `uv` (a `uv.lock`); Renovate's `pep621`, `pip_requirements`, `poetry` and `pipenv` managers. diff --git a/audit/stacks/typescript-node.md b/audit/stacks/typescript-node.md index 647b6cb3..f158aeeb 100644 --- a/audit/stacks/typescript-node.md +++ b/audit/stacks/typescript-node.md @@ -43,7 +43,8 @@ seen across the corpus (§Verified) — a third option, Node's own built-in test Dependency scanner: the package manager's own `audit` command; `audit-ci` or Snyk are alternatives that pin a fixed severity threshold in config rather than a CLI flag. Hook manager: Husky + lint-staged (`prepare: husky`), or simple-git-hooks; lefthook when there is no root `package.json`. Commit rule: -commitlint in `.husky/commit-msg`. Dependency updates: Dependabot `package-ecosystem: npm` (npm, Yarn and +commitlint in `.husky/commit-msg`, and `npx commitlint --from --to ` in CI +(`TOOL-06`). Dependency updates: Dependabot `package-ecosystem: npm` (npm, Yarn and pnpm lockfiles) or `bun`; Renovate's `npm` manager. ## Execution determinism diff --git a/docs/reference/implement-master.md b/docs/reference/implement-master.md index 1cdf3873..751bf8af 100644 --- a/docs/reference/implement-master.md +++ b/docs/reference/implement-master.md @@ -208,13 +208,14 @@ Full playbook: [`add-ci-parity.md`](../../implement/playbooks/add-ci-parity.md). ### `add-hooks-and-commit-rules` A commit runs a fast local hook chain before it is created, with one committed configuration and a documented -one-time install. The commit-message convention is either enforced locally or explicitly recorded as enforced -elsewhere (the pull-request title) by the owner's decision. +one-time install. The commit-message convention runs in two places with one type list: a local `commit-msg` stage, +so an agent learns of a bad subject before it pushes, and a CI step, which no `--no-verify` can skip. The owner may +instead record the pull-request title check as enough. | Check | What the playbook does | |---|---| | [`TOOL-05` Local hook chain active](audit-master.md#tool-05-local-hook-chain-active) | Commits a hook-manager configuration the engine reads as wired, with fast changed-files hooks | -| [`TOOL-06` Commit convention enforced](audit-master.md#tool-06-commit-convention-enforced) | Offers three options; the owner may skip when the title check is enough (see Commit rule) | +| [`TOOL-06` Commit convention enforced](audit-master.md#tool-06-commit-convention-enforced) | Writes the commit rule as a local `commit-msg` stage and a CI step (`PASS`); the owner may skip when the title check is enough (see Commit rule) | **What it never does:** diff --git a/implement/playbooks/add-hooks-and-commit-rules.md b/implement/playbooks/add-hooks-and-commit-rules.md index fc92b5c3..3f6d2906 100644 --- a/implement/playbooks/add-hooks-and-commit-rules.md +++ b/implement/playbooks/add-hooks-and-commit-rules.md @@ -6,15 +6,16 @@ ## Goal A commit runs a fast local hook chain before it is created, with one committed configuration and a documented -one-time install. The commit-message convention is either enforced locally or explicitly recorded as enforced -elsewhere (the pull-request title) by the owner's decision. +one-time install. The commit-message convention runs in two places with one type list: a local `commit-msg` stage, +so an agent learns of a bad subject before it pushes, and a CI step, which no `--no-verify` can skip. The owner may +instead record the pull-request title check as enough. ## Checks resolved | Check | Rubric | What this playbook does for it | |---|---|---| | TOOL-05 | [05-tooling.md](../../audit/checks/05-tooling.md#tool-05--local-hook-chain-active) | Commits a hook-manager configuration the engine reads as wired, with fast changed-files hooks | -| TOOL-06 | [05-tooling.md](../../audit/checks/05-tooling.md#tool-06--commit-convention-enforced) | Offers three options; the owner may skip when the title check is enough (see Commit rule) | +| TOOL-06 | [05-tooling.md](../../audit/checks/05-tooling.md#tool-06--commit-convention-enforced) | Writes the commit rule as a local `commit-msg` stage and a CI step (`PASS`); the owner may skip when the title check is enough (see Commit rule) | ## Preconditions @@ -25,9 +26,11 @@ elsewhere (the pull-request title) by the owner's decision. ([`define-command-surface`](define-command-surface.md), [`configure-quality-tools`](configure-quality-tools.md)). A hook calls those verbs; it never defines its own commands. - Preferences read, when present: `format.mass_reformat` (a `no` means hooks check changed files only, never the - full tree), `lint.mode`, `layout.command_runner`, `hooks.manager`, `commit.convention`. + full tree), `lint.mode`, `layout.command_runner`, `hooks.manager`, `commit.convention`, `commit.tool`. - **Drift** ([Stage 3](../SKILL.md#stage-3-apply), step 2): a finding saying "no hook manager" can be stale; look for `lefthook.yml`, `.pre-commit-config.yaml`, `.husky/`, `.githooks/` and `core.hooksPath` on the current default branch. + A `TOOL-06` `PARTIAL` names which half is missing: "only in a local hook" needs the CI step, "only in CI" needs + the local stage. Keep the half that exists and add the other with the same tool and type list. ## Inputs to ask @@ -48,16 +51,30 @@ elsewhere (the pull-request title) by the owner's decision. not archived, and it is the most used manager (checked 2026-11-09). Reject a tool only when it is archived or has an open security advisory. Installing a binary is a developer step the owner approves, not a repository dependency. 2. **Hook content.** Default below; the owner may remove, never add a slow hook silently. -3. **Commit rule** (skip when already skipped in the state file): +3. **Commit rule** (skip when already skipped in the state file). One tool, run in both places, so the two + cannot disagree. Detect first: keep a tool the repository already runs in either place. Otherwise recommend by + stack (preference `commit.tool`). Versions checked 2026-11-09 (npm registry, PyPI, GitHub API); re-check when + applied. + + | Stack | Tool | Local `commit-msg` stage | CI step (over the commits the pull request adds) | + |---|---|---|---| + | Node with a root manifest | commitlint (`@commitlint/cli` 21.2.4, 2026-10-18), a dev dependency | `.husky/commit-msg`: `npx --no -- commitlint --edit "$1"` | `npx commitlint --from "$BASE" --to "$HEAD" --verbose` | + | Python | commitizen (4.19.3, 2026-10-24), or gitlint (0.19.1, 2023-03-10) when the repository already uses it | pre-commit hook `commitizen` (or `gitlint`) with `stages: [commit-msg]` | `cz check --rev-range "$BASE..$HEAD"` (or `gitlint --commits "$BASE..$HEAD"`) | + | Mixed, Java, Node with no root manifest, or no profile | a commit-lint script in the repository ([template](#commit-lint-script-template)) | lefthook `commit-msg` stage, or `.githooks/commit-msg`, running the script with `--message-file` | the same script with `--range "$BASE..$HEAD"` | + + conventional-pre-commit (4.5.0, 2026-10-29) checks one message file only, with no range form, so it cannot + be the CI half; keep it where it exists and run its type list through commitizen or the script in CI. + The engine sees the CI step by the tool's name, by the same script path in the hook and the workflow, or + through a runner target (`make commit-lint`, `npm run lint:commits`) whose command runs either. | Option | What | Engine reads it as | |---|---|---| - | A. Document the PR-title rule | The repository squash-merges and a CI check rejects non-conventional PR titles; the PR title is what lands on the default branch. Write this in the contributing doc and record the finding `skipped` with that reason | TOOL-06 stays `FAIL` (the engine does not read CI; "TOOL-06's own blind spot"); owner-accepted | - | B. A commit-message rule in a `commit-msg` stage, matched to the stack | Node: commitlint (`@commitlint/cli` 21.2.4) in `.husky/commit-msg`. Python: a pre-commit `commit-msg` hook (conventional-pre-commit, commitizen or gitlint). Others: the manager's `commit-msg` stage | `PASS`. commitlint needs Node; with no root manifest its version cannot be pinned by a lockfile, so prefer the Python or lefthook route there | - | C. Both | A and B | `PASS`; carries B's costs | + | A. The pull-request title check only | The repository squash-merges, and a CI check (`amannn/action-semantic-pull-request`, v6.1.2, 2026-11-05) rejects a non-conventional title, which becomes the squash commit's subject. Write this in the contributing doc and record the finding `skipped` with that reason | `PARTIAL` (CI only); owner-accepted | + | B. The commit rule, local and in CI | The tool above in the `commit-msg` stage and as a CI step | `PASS` | + | C. Both | A and B, with one type list | `PASS`; carries B's costs | - Recommend A where squash merges and a title check exist (the history that matters is the squash commit); - otherwise B. Only the owner skips; record the owner's words. + Recommend B, or C where squash merges and a title check already exist. Only the owner skips; record the + owner's words. ## Steps @@ -74,8 +91,9 @@ elsewhere (the pull-request title) by the owner's decision. binary install line too. Say that CI is the real gate. Never advise `--no-verify`; when a hook fails, fix the cause. 4. **Commit rule** per the owner's option. For A: the contributing doc names the PR-title workflow and the type - list it enforces. For B or C: add the config and the `commit-msg` stage, with the same type list as the - title rule so the two cannot disagree. + list it enforces. For B or C: add the config and the `commit-msg` stage, then the CI step for the provider + (see [Commit rule in CI](#commit-rule-in-ci)), with the same type list as the title rule so they cannot + disagree. Commit the local stage and the CI step separately, so either reverts alone. 5. **Verify** with a deliberately bad staged file in a scratch worktree (see Verification). Commit config, docs and rule separately. @@ -111,6 +129,123 @@ With lefthook (IR07.3): - Fetch the test binary into a scratch directory and compare its SHA-256 with the digest GitHub publishes for the release asset; do not put it on the owner's `PATH`. +### Commit rule in CI + +The step checks every non-merge commit the pull request adds. It needs the full history (a shallow clone has no +base commit) and must pass, not fail, when there is no base (a new branch). Put it in the workflow that already +runs the gate, in a job of its own. + +GitHub Actions. `BASE` covers a pull request, a merge-queue group and a push: + +```yaml + commits: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ # the repository's existing checkout pin + with: + persist-credentials: false + fetch-depth: 0 + - name: Check the commit subjects + env: + BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }} + run: | + if [[ -z "$BASE" || "$BASE" =~ ^0+$ ]] || ! git cat-file -e "$BASE^{commit}" 2>/dev/null; then + echo "No base commit to compare with (a new branch); nothing to check." + exit 0 + fi + +``` + +When the step runs a Node or Python tool, set it up as the gate's other jobs do (`actions/setup-node` and +`npm ci`, or `astral-sh/setup-uv`), at the version the repository already pins. + +GitLab CI. Merge-request pipelines carry the base in `CI_MERGE_REQUEST_DIFF_BASE_SHA`: + +```yaml +commits: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + variables: + GIT_DEPTH: "0" + script: + - +``` + +Another provider: the same three parts (full history, the pull request's base and head, a pass with no base). + +### Commit-lint script template + +For the mixed or Java route, or when the owner wants no new tool. A trimmed form of this framework's own +`dev/commit_lint.py`; set `TYPES` to the repository's convention (`commit.convention`) and put the script where +the repository keeps its scripts. Standard library only; needs Python 3.9 or newer. + +```python +"""Check that commit subjects follow Conventional Commits: ``type(scope)!: subject``. + + commit_lint.py --range .. every non-merge commit in the range (CI) + commit_lint.py --message-file the first line of one message (the commit-msg hook) + +Exit 0 when every subject passes, 1 when one fails, 2 on a usage or git error. +""" + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +TYPES = ("feat", "fix", "docs", "test", "chore", "ci", "refactor", "perf", "build", "style", "revert") +SUBJECT = re.compile(r"^(?:" + "|".join(TYPES) + r")(?:\([a-z0-9._-]+\))?!?: \S.*$") + + +def subjects(revision_range): + out = subprocess.run( + ["git", "log", "--no-merges", "--format=%h%x00%s", revision_range], + capture_output=True, + text=True, + check=False, + ) + if out.returncode != 0: + sys.exit(f"commit_lint: {out.stderr.strip() or 'git log failed'}") + return [tuple(line.split("\x00", 1)) for line in out.stdout.splitlines()] + + +def main(): + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--range", dest="revision_range") + group.add_argument("--message-file", type=Path) + args = parser.parse_args() + if args.message_file: + lines = [l for l in args.message_file.read_text(encoding="utf-8").splitlines() if not l.startswith("#")] + pairs = [("message", lines[0].rstrip() if lines else "")] + else: + pairs = subjects(args.revision_range) + bad = [(sha, subject) for sha, subject in pairs if not SUBJECT.match(subject)] + for sha, subject in bad: + print(f"{sha}: not a Conventional Commits subject: {subject!r}") + print(f"allowed types: {', '.join(TYPES)}" if bad else f"commit_lint: {len(pairs)} subject(s) checked") + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main()) +``` + +With no Python on the machines (some Java repositories), a POSIX shell hook does the local half, and the CI step +loops over `git log --no-merges --format=%s "$BASE..$HEAD"` with the same pattern: + +```sh +#!/bin/sh +# .githooks/commit-msg: the first non-comment line must be a Conventional Commits subject. +subject=$(grep -v '^#' "$1" | head -n 1) +printf '%s\n' "$subject" | grep -Eq '^(feat|fix|docs|test|chore|ci|refactor|perf|build|style|revert)(\([a-z0-9._-]+\))?!?: [^ ]' || { + echo "commit-msg: not a Conventional Commits subject: $subject" >&2 + exit 1 +} +``` + ## Continue on error - If the changed-files verb reveals existing debt in a staged file, the hook blocks only on files the commit @@ -125,9 +260,10 @@ With lefthook (IR07.3): |---|---|---| | TOOL-05 | `test -f ` (`.husky/pre-commit`, `.pre-commit-config.yaml`, `lefthook.yml`) and `git config core.hooksPath` or the install step in the docs | The engine's wiring file exists; the install step is documented | | TOOL-05 (behaviour) | In a scratch worktree, stage a file with a deliberate format violation and a fake secret, run the hook | The commit is refused with a clear message; a clean staged change passes in a few seconds | -| TOOL-06 | Option A: the contributing doc names the title check and the state records `skipped` with the owner's reason. Option B or C: the config and the `commit-msg` stage exist; a bad message is refused | As stated | +| TOOL-06 | Option A: the contributing doc names the title check and the state records `skipped` with the owner's reason. Option B or C: the config and the `commit-msg` stage exist; a bad message is refused locally | As stated | +| TOOL-06 (CI) | Option B or C: push a scratch branch with one bad subject and open a draft pull request only with the owner's approval; or run the CI command locally over `..HEAD` | The CI step fails on the bad subject and passes on a clean range; the engine reads `TOOL-06` `PASS` | -The engine reads TOOL-06 as a file or stage check; option A leaves it `FAIL` by design. +The engine reads `TOOL-06` from files only: option A gives `PARTIAL` (CI only) by design. ## Rollback @@ -151,6 +287,8 @@ Beyond the shared lines in [Stage 3](../SKILL.md#stage-3-apply) (`in_progress`, ```bash python3 implement/tools/progress.py pref set hooks.manager \ --target --phase --reason "" --by owner +python3 implement/tools/progress.py pref set commit.tool \ + --target --phase --reason "" --by owner python3 implement/tools/progress.py skip F-nnn --target --phase --by owner \ --reason "" python3 implement/tools/progress.py set-status F-nnn done --target --phase \ From 23b354cd7d24e8fc7bdd9f19c9a494cbe5612473 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Mon, 9 Nov 2026 17:22:32 +0530 Subject: [PATCH 050/111] feat(implement): record in the state which framework created and last wrote it State schema 0.3 replaces recipe with framework {release, revision, install}, in the report's shape, and adds last_framework {release, revision}. Every write sets last_framework and, when it changed, logs "Framework moved from v0.3.2 at to v0.3.3 at ." progress.py show returns data.versions (created, last, tools). The tools read state 0.3 only and refuse an older state with a message that names both versions. Each source report now records its framework release and revision, sha256 is required, and the commits V4 writes are 12 digits, as in the report. compare_reports.py warns when the two reports come from different releases, from one release at two commits, or from different catalogues. The framework detection moves into validate_report.py, which both bundles ship, so report.py and progress.py share it. progress.py RECIPE_VERSION is now FRAMEWORK_RELEASE. The three state examples move to 0.3 and their Markdown is re-rendered. The secrets baseline gains two entries (a commit id and a digest in the server worked-example state; accepted by the owner), and the markdownlint baseline drops to 697. Roadmap-Step: V4 Co-Authored-By: Claude Opus 5.5 --- .quality-baseline.json | 2 +- .secrets.baseline | 44 ++++--- RELEASE_PROCESS.md | 7 +- audit/schema/SCHEMA.md | 14 +- ...ess-state.example-service.in-progress.json | 18 ++- ...iness-state.example-service.in-progress.md | 5 +- .../readiness-state.example-service.json | 18 ++- .../readiness-state.example-service.md | 5 +- ...ssistant-server-agentic-readiness-STATE.md | 9 +- ...istant-server-agentic-readiness-state.json | 21 ++- audit/schema/readiness-state.schema.json | 64 +++++---- audit/tools/report.py | 39 +----- audit/tools/tests/test_report.py | 4 +- audit/tools/validate_report.py | 52 ++++++++ dev/tests/test_release_config.py | 5 +- docs/reference/tools-cli.md | 31 +++-- implement/state.md | 23 ++-- implement/tools/check_drift.py | 2 +- implement/tools/compare_reports.py | 53 +++++++- implement/tools/progress.py | 123 +++++++++++------- implement/tools/tests/test_check_drift.py | 2 +- implement/tools/tests/test_compare_reports.py | 37 ++++++ implement/tools/tests/test_progress.py | 57 +++++--- .../tools/tests/test_progress_transitions.py | 55 ++++++++ .../tests/test_readiness_state_schema.py | 36 ++++- 25 files changed, 518 insertions(+), 208 deletions(-) diff --git a/.quality-baseline.json b/.quality-baseline.json index b848058c..fbd9f259 100644 --- a/.quality-baseline.json +++ b/.quality-baseline.json @@ -15,7 +15,7 @@ "verb": "make typecheck" }, "markdownlint": { - "count": 698, + "count": 697, "what": "markdownlint-cli2 issues over every tracked Markdown file (the DOCS_LINT_PATHS ratchet is a subset and clean)", "command": "git ls-files -z '*.md' | xargs -0 npx markdownlint-cli2@", "verb": "make docs-lint" diff --git a/.secrets.baseline b/.secrets.baseline index c8aa6ed5..d40fec8d 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -281,7 +281,7 @@ "filename": "audit/schema/examples/readiness-state.example-service.in-progress.json", "hashed_secret": "03179aaeae53aacc8a97dd50d94f55a4a770ba5f", "is_verified": false, - "line_number": 17 + "line_number": 23 } ], "audit/schema/examples/readiness-state.example-service.json": [ @@ -290,7 +290,7 @@ "filename": "audit/schema/examples/readiness-state.example-service.json", "hashed_secret": "03179aaeae53aacc8a97dd50d94f55a4a770ba5f", "is_verified": false, - "line_number": 17 + "line_number": 23 } ], "audit/schema/examples/realistic/agentic-tdd.audit-draft.20261010T065738Z.json": [ @@ -446,72 +446,86 @@ { "type": "Hex High Entropy String", "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json", - "hashed_secret": "94519421c0c055727e8ea27de76e9af8d56a9c56", + "hashed_secret": "48861ebf759be08f154c2da7c992bb95133a8a2b", "is_verified": false, - "line_number": 10 + "line_number": 9 }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json", - "hashed_secret": "dba1a39fdb7930692431327ef0589c9f6c1047b2", + "hashed_secret": "07090dcf641aeedaa37a0d45df7eacf895a2733a", "is_verified": false, - "line_number": 15 + "line_number": 19 + }, + { + "type": "Hex High Entropy String", + "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json", + "hashed_secret": "b9bbeca69c7ec12a4b808e759fa7048dfd406c08", + "is_verified": false, + "line_number": 22 + }, + { + "type": "Hex High Entropy String", + "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json", + "hashed_secret": "6f1ed551d7a8a3b7ada7f52f5a288fa9cbdb9ddb", + "is_verified": false, + "line_number": 23 }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json", "hashed_secret": "22c6d53af390efedc67eec3c60cc451cd9d74143", "is_verified": false, - "line_number": 501 + "line_number": 508 }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json", "hashed_secret": "2ea801dde1cdd75bbb0cfad7fabaf57410e5386d", "is_verified": false, - "line_number": 510 + "line_number": 517 }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json", "hashed_secret": "362c7d0200854d6e2455e9888e3cc03883bec778", "is_verified": false, - "line_number": 519 + "line_number": 526 }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json", "hashed_secret": "367cedbefc6e73c2fd8273525cc884b5ef4c756d", "is_verified": false, - "line_number": 528 + "line_number": 535 }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json", "hashed_secret": "5f7060afa80145dcbed1423d145ea8be5cf3bcf6", "is_verified": false, - "line_number": 537 + "line_number": 544 }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json", "hashed_secret": "5cb283eac2f48fd294515d80715b9c61c7b72f36", "is_verified": false, - "line_number": 546 + "line_number": 553 }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json", "hashed_secret": "91e6255a54491ece7a52cd6303a9ece391ecc8c1", "is_verified": false, - "line_number": 555 + "line_number": 562 }, { "type": "Hex High Entropy String", "filename": "audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json", "hashed_secret": "3e98cf30e8840c3af584f2020ff17bbf6dd4085e", "is_verified": false, - "line_number": 564 + "line_number": 571 } ], "audit/schema/examples/server-worked-example/digital-assistant-server-first-audit-20261028T131409Z.json": [ @@ -1484,5 +1498,5 @@ } ] }, - "generated_at": "2026-11-09T10:22:05Z" + "generated_at": "2026-11-09T11:38:29Z" } diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index ce8c4e6c..4fb1b0cc 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -71,7 +71,8 @@ The version is written in five places, and release-please changes all five in th - `__version__` in [`audit/engine/audit/__init__.py`](audit/engine/audit/__init__.py): what `audit --version` and the report's `provenance.engine.version` print. - `FRAMEWORK_RELEASE` in [`audit/tools/report.py`](audit/tools/report.py): the report's `provenance.framework.release`. -- `RECIPE_VERSION` in [`implement/tools/progress.py`](implement/tools/progress.py): the state's `recipe.version`. +- `FRAMEWORK_RELEASE` in [`implement/tools/progress.py`](implement/tools/progress.py): the state's + `framework.release`. - The new entry in [`CHANGELOG.md`](CHANGELOG.md). Each of the three code lines carries the `x-release-please-version` marker and is an `extra-files` entry. @@ -193,8 +194,8 @@ The push to `main` starts `release-please.yml`. A minute later the release pull gh pr list --base main --head release-please--branches--main ``` -2. Read its diff. It changes `.release-please-manifest.json`, `__version__` in the engine, `FRAMEWORK_RELEASE` - in `report.py`, `RECIPE_VERSION` in `progress.py` and `CHANGELOG.md`. A second commit, +2. Read its diff. It changes `.release-please-manifest.json`, `__version__` in the engine, the two + `FRAMEWORK_RELEASE` lines (`report.py` and `progress.py`) and `CHANGELOG.md`. A second commit, `style(changelog): ...`, holds the markdownlint fixes. 3. Check the version. To force a different one, see [Choose the version](#choose-the-version). 4. Without the App token, close and reopen the pull request to start CI: diff --git a/audit/schema/SCHEMA.md b/audit/schema/SCHEMA.md index 31129743..888698e1 100644 --- a/audit/schema/SCHEMA.md +++ b/audit/schema/SCHEMA.md @@ -14,7 +14,7 @@ | [`audit-intake.schema.json`](audit-intake.schema.json) | The answers to the audit's Stage 0 questions, given in a file. `intake_version` `0.1`. | | [`catalogue.schema.json`](catalogue.schema.json) | The check catalogue, [`../catalogue.json`](../catalogue.json): every check in one ID namespace. | | [`facts.schema.json`](facts.schema.json) | The file `facts.py` writes: counts and names that describe a repository, with no verdict. `facts_version` `0.1`. See [Facts file](#facts-file). | -| [`readiness-state.schema.json`](readiness-state.schema.json) | The implement recipe's per-repository state, which `progress.py` writes inside the target. `schema_version` `0.2`. See [Readiness state](#readiness-state). | +| [`readiness-state.schema.json`](readiness-state.schema.json) | The implement recipe's per-repository state, which `progress.py` writes inside the target. `schema_version` `0.3`. See [Readiness state](#readiness-state). | | [`examples/minimal.json`](examples/minimal.json) | The smallest valid report: one agent check, no findings. | | [`examples/facts.minimal.json`](examples/facts.minimal.json) | A valid facts file for a small made-up service. | | [`examples/realistic/`](examples/realistic/) | The engine's real reports on three open-source repositories, each beside its conversion: `agentic-tdd`, `Digital-Assistant-SDK` and `Digital-Assistant-Server`. They were scanned on 2026-10-10 at the commits named in each report's `provenance.target`, with the engine at ruleset revision `2026-09-28.13`. A `*.engine-report.*.json` file is the engine's output, untouched. A `*.audit-draft.*.json` file is that report converted by the mapping below and nothing else: all 69 checks, every finding pending, no name anonymised and nothing invented, because the repositories are open source. In all three the recomputed phase-1 score equals the engine's (0.7311, 0.6513 and 0.4244). Each pair is the input and the expected output of the engine adapter: a test imports each raw report into a draft made from the golden draft's own `init` fields and requires the draft beside it, byte for byte. When the mapping changes, the drafts are regenerated from the raw reports and the raw reports are left alone. | @@ -34,11 +34,14 @@ The Markdown report, which [`tools/render_report.py`](../tools/render_report.py) | `schema_version` | the report | The shape of the report. `0.2` today; see the note below the table. | | `intake_version` | the intake file | The shape of the intake file. `0.1` today. | | `facts_version` | the facts file | The shape of the facts file. `0.1` today. | -| `schema_version` | the readiness state | The shape of the state file, a separate schema from the report's. `0.2` today; `0.2` added the optional `source_reports[].sha256`, and `0.1` states stay valid. | +| `schema_version` | the readiness state | The shape of the state file, a separate schema from the report's. `0.3` today; see the note below the table. | | `catalogue_version` | `provenance` | The check catalogue the report was built against, `MAJOR.MINOR.PATCH`, as in [`catalogue.json`](../catalogue.json). The bump rules are in [`catalogue.schema.json`](catalogue.schema.json). | | `provenance.framework.release` | `provenance` | The framework release that produced the report. | | `provenance.engine.version`, `ruleset_hash` | `provenance` | The deterministic engine and its ruleset. | +State schema `0.3` replaced `recipe` with `framework` and `last_framework`, added `release` and `revision` to each +source report, and made `sha256` required. The tools do not read an older state. + Report schema `0.2` replaced `provenance.recipe` and `provenance.framework_revision` with `provenance.framework`. The tools do not read a `0.1` report. The release is the version in [`.release-please-manifest.json`](../../.release-please-manifest.json). @@ -263,10 +266,11 @@ the skip, defer and preference rules are in the [state contract](../../implement | Section | Holds | |---|---| -| `schema_version` | `0.2` (a `0.1` state is still read; it has no `sha256`) | +| `schema_version` | `0.3`; the tools read no other | | `repo` | `label` (the report's `provenance.target.label`, which names the files) and `default_branch` | -| `recipe` | `brownfield-implement`, its version, and `framework_rev`, the framework commit the state was created from or null | -| `source_reports` | Each final report's path relative to the target, its `git_sha` and `catalogue_version`, copied as given, and `sha256`, the SHA-256 of the report file when `init` read it (optional; `0.2`) | +| `framework` | The framework that created the state: `release`, `revision` (12 digits or null) and `install`, as in a report | +| `last_framework` | The framework that last wrote the state: `release` and `revision`. A write that changes it logs the move | +| `source_reports` | Each final report's path relative to the target, its `git_sha` (12 digits), `catalogue_version`, `release` and `revision`, and `sha256`, the SHA-256 of the report file when `init` read it | | `preferences` | The owner's choices: `key`, `value`, `reason`, `by: owner`, `date`, `phase` | | `themes` | T1 to T8 in order: `title`, `status` (`not_started`, `branch_open`, `pr_open`, `merged`), `phase`, `branch`, `pr`, `merged_sha` | | `items` | One per confirmed, edited or deferred finding: `finding`, `check`, `severity`, `theme`, `statement`, `status`, `by`, `date`, `reason`, `commits`, `verification`, `debt`, `notes` | diff --git a/audit/schema/examples/readiness-state.example-service.in-progress.json b/audit/schema/examples/readiness-state.example-service.in-progress.json index 6791d316..5e294587 100644 --- a/audit/schema/examples/readiness-state.example-service.in-progress.json +++ b/audit/schema/examples/readiness-state.example-service.in-progress.json @@ -1,19 +1,25 @@ { - "schema_version": "0.2", + "schema_version": "0.3", "repo": { "label": "example-service", "default_branch": "main" }, - "recipe": { - "name": "brownfield-implement", - "version": "0.3.2", - "framework_rev": "cafecafecafecafecafecafecafecafecafecafe" + "framework": { + "release": "0.3.2", + "revision": "cafecafecafe", + "install": "checkout" + }, + "last_framework": { + "release": "0.3.2", + "revision": "cafecafecafe" }, "source_reports": [ { "path": "docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json", - "git_sha": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + "git_sha": "deadbeefdead", "catalogue_version": "0.4.0", + "release": "0.3.2", + "revision": "cafecafecafe", "sha256": "51e8e6afb0125d46d0fa12d60657929f89ba8ffde27894a59366aacdd0b94b98" } ], diff --git a/audit/schema/examples/readiness-state.example-service.in-progress.md b/audit/schema/examples/readiness-state.example-service.in-progress.md index 3948b4b9..b67efa5a 100644 --- a/audit/schema/examples/readiness-state.example-service.in-progress.md +++ b/audit/schema/examples/readiness-state.example-service.in-progress.md @@ -9,8 +9,9 @@ A skipped finding is final until the owner unskips it, and every preference bind | Field | Value | |---|---| | Default branch | `main` | -| Recipe | brownfield-implement 0.3.2, framework `cafecafecafecafecafecafecafecafecafecafe` | -| Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdeadbeefdeadbeefdeadbeefdeadbeef`, catalogue 0.4.0 | +| Created by | `v0.3.2` at `cafecafecafe` (checkout) | +| Last written by | `v0.3.2` at `cafecafecafe` | +| Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdead`, catalogue 0.4.0, made by `v0.3.2` at `cafecafecafe` | ## Summary diff --git a/audit/schema/examples/readiness-state.example-service.json b/audit/schema/examples/readiness-state.example-service.json index 61d576ca..a074d8ba 100644 --- a/audit/schema/examples/readiness-state.example-service.json +++ b/audit/schema/examples/readiness-state.example-service.json @@ -1,19 +1,25 @@ { - "schema_version": "0.2", + "schema_version": "0.3", "repo": { "label": "example-service", "default_branch": "main" }, - "recipe": { - "name": "brownfield-implement", - "version": "0.3.2", - "framework_rev": "cafecafecafecafecafecafecafecafecafecafe" + "framework": { + "release": "0.3.2", + "revision": "cafecafecafe", + "install": "checkout" + }, + "last_framework": { + "release": "0.3.2", + "revision": "cafecafecafe" }, "source_reports": [ { "path": "docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json", - "git_sha": "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + "git_sha": "deadbeefdead", "catalogue_version": "0.4.0", + "release": "0.3.2", + "revision": "cafecafecafe", "sha256": "51e8e6afb0125d46d0fa12d60657929f89ba8ffde27894a59366aacdd0b94b98" } ], diff --git a/audit/schema/examples/readiness-state.example-service.md b/audit/schema/examples/readiness-state.example-service.md index f743827e..4ec0f740 100644 --- a/audit/schema/examples/readiness-state.example-service.md +++ b/audit/schema/examples/readiness-state.example-service.md @@ -9,8 +9,9 @@ A skipped finding is final until the owner unskips it, and every preference bind | Field | Value | |---|---| | Default branch | `main` | -| Recipe | brownfield-implement 0.3.2, framework `cafecafecafecafecafecafecafecafecafecafe` | -| Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdeadbeefdeadbeefdeadbeefdeadbeef`, catalogue 0.4.0 | +| Created by | `v0.3.2` at `cafecafecafe` (checkout) | +| Last written by | `v0.3.2` at `cafecafecafe` | +| Source report | `docs/agentic-readiness-report/example-service.audit-report.20261010T094207Z.json` at `deadbeefdead`, catalogue 0.4.0, made by `v0.3.2` at `cafecafecafe` | ## Summary diff --git a/audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-STATE.md b/audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-STATE.md index 44af9803..acd36682 100644 --- a/audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-STATE.md +++ b/audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-STATE.md @@ -9,8 +9,9 @@ A skipped finding is final until the owner unskips it, and every preference bind | Field | Value | |---|---| | Default branch | `dev` | -| Recipe | brownfield-implement 0.2.0-dev, framework `a7e5e023654d48620e4439168bee4158f4a7deaf` | -| Source report | `docs/agentic-readiness-report/digital-assistant-server-first-audit-20261028T131409Z.json` at `4a48e1976e424f3f424d2510049413c1c3f38dd6`, catalogue 0.2.2 | +| Created by | `v0.3.2` at `a7e5e023654d` (checkout) | +| Last written by | `v0.3.2` at `a7e5e023654d` | +| Source report | `docs/agentic-readiness-report/digital-assistant-server-first-audit-20261028T131409Z.json` at `4a48e1976e42`, catalogue 0.2.2, made by `v0.3.2` at `b4bf27df7915` | ## Summary @@ -292,8 +293,8 @@ A skipped finding is final until the owner unskips it, and every preference bind - **F-018**: commits: `eaf8fab`; verification: `./gradlew jacocoTestCoverageVerification -x test (minimum 0.70 then 0.71)` gave 0.70 passes (70.6 percent measured, two identical runs); 0.71 fails Rule violated - **F-038**: commits: `eaf8fab`, `b98c4f8`; verification: `./gradlew test jacocoTestReport (local stack up, - flock )` gave 167 tests 0 failures 0 skipped; report build/reports/jacoco/test; Quarkus - own report agrees (396 missed 951 covered lines) + flock )` gave 167 tests 0 failures 0 skipped; report build/reports/jacoco/test; Quarkus own + report agrees (396 missed 951 covered lines) ### T6 Docs and navigation diff --git a/audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json b/audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json index e6c3a4ef..26da306e 100644 --- a/audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json +++ b/audit/schema/examples/server-worked-example/digital-assistant-server-agentic-readiness-state.json @@ -1,19 +1,26 @@ { - "schema_version": "0.1", + "schema_version": "0.3", "repo": { "label": "digital-assistant-server", "default_branch": "dev" }, - "recipe": { - "name": "brownfield-implement", - "version": "0.2.0-dev", - "framework_rev": "a7e5e023654d48620e4439168bee4158f4a7deaf" + "framework": { + "release": "0.3.2", + "revision": "a7e5e023654d", + "install": "checkout" + }, + "last_framework": { + "release": "0.3.2", + "revision": "a7e5e023654d" }, "source_reports": [ { "path": "docs/agentic-readiness-report/digital-assistant-server-first-audit-20261028T131409Z.json", - "git_sha": "4a48e1976e424f3f424d2510049413c1c3f38dd6", - "catalogue_version": "0.2.2" + "git_sha": "4a48e1976e42", + "catalogue_version": "0.2.2", + "release": "0.3.2", + "revision": "b4bf27df7915", + "sha256": "934f8c8d295f7420f941bfa520c694b2868c582f0f1249e8cc514332cadc809e" } ], "preferences": [ diff --git a/audit/schema/readiness-state.schema.json b/audit/schema/readiness-state.schema.json index 501c3602..ca33841e 100644 --- a/audit/schema/readiness-state.schema.json +++ b/audit/schema/readiness-state.schema.json @@ -1,15 +1,15 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:codebase-agentic-readiness-framework:readiness-state:0.2", + "$id": "urn:codebase-agentic-readiness-framework:readiness-state:0.3", "title": "Readiness state", "description": "The implement recipe's per-repository state: which audit findings are queued, done, partial, skipped or deferred, the owner's preferences, each theme's branch and PR, and a log. It lives in the target at docs/agentic-readiness/