Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
# suite of `test` has no job here. The steps that are cheap share one job, because each job pays about 15 s of
# setup. Python 3.11 runs every step; Python 3.14 runs the runtime code: the suites and the schema cross-check.
static:
name: static (format, typecheck, rules, recipe lint, masters, workflow lint, spell)
name: static (format, typecheck, rules, recipe lint, ADR lint, masters, workflow lint, spell)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand All @@ -53,7 +53,7 @@ jobs:
with:
go-version-file: dev/tools/actionlint/go.mod
cache-dependency-path: dev/tools/actionlint/go.sum
- run: make format-check typecheck verify-rules recipe-lint masters-check eval-expected-check workflow-lint spell
- run: make format-check typecheck verify-rules recipe-lint adr-lint masters-check eval-expected-check workflow-lint spell

crosscheck:
name: crosscheck (Python ${{ matrix.python }})
Expand Down
11 changes: 10 additions & 1 deletion .secrets.baseline

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ make lint # read-only lint
make typecheck # mypy, strict, on audit/tools, implement/tools and dev, and the engine's build.py
make test # the engine's, the audit tools', the implement tools' and the dev checks' suites (at once on Linux)
make check # THE gate: format:check, typecheck, test, the engine's --verify-rules, the schema
# cross-check, recipe-lint, masters-check, eval-expected-check, workflow-lint (actionlint,
# cross-check, recipe-lint, adr-lint, masters-check, eval-expected-check, workflow-lint (actionlint,
# zizmor, check-jsonschema), spell (typos)
make security # secrets scan, then the dependency audit (osv-scanner; needs Go and the network)
make commit-lint # Conventional Commits subjects on this branch (RANGE=origin/dev..HEAD); CI runs it on each PR
Expand Down
10 changes: 7 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,13 @@ endif
ifndef TEST_SUITES_SUBMAKE
.NOTPARALLEL:
endif
.PHONY: help setup format format-check format\:check lint typecheck test $(TEST_SUITES) test-engine-shard test-audit-tools-shard test-dev-shard test-part coverage coverage-engine coverage-engine-shard coverage-audit-tools-shard coverage-part coverage-audit-tools coverage-implement-tools coverage-combine verify-rules crosscheck recipe-lint masters-check eval-expected-check update-masters check security security-secrets security-deps update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint spell spell-fix
.PHONY: help setup format format-check format\:check lint typecheck test $(TEST_SUITES) test-engine-shard test-audit-tools-shard test-dev-shard test-part coverage coverage-engine coverage-engine-shard coverage-audit-tools-shard coverage-part coverage-audit-tools coverage-implement-tools coverage-combine verify-rules crosscheck recipe-lint adr-lint masters-check eval-expected-check update-masters check security security-secrets security-deps update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint spell spell-fix

help:
@echo "verbs: format format:check (format-check) lint typecheck test check security"
@echo "parts: test-engine test-audit-tools test-implement-tools test-dev coverage-engine coverage-audit-tools coverage-implement-tools coverage-combine"
@echo "ci: test-part SHARD=<i> coverage-part SHARD=<i> coverage-combine SHARDED=1 (and test-<suite>-shard, coverage-<suite>-shard)"
@echo "extra: setup (checks prerequisites, installs nothing) coverage self-audit update-audit-baseline verify-rules crosscheck recipe-lint masters-check eval-expected-check update-masters workflow-lint commit-lint spell spell-fix update-secrets-baseline validate-skills"
@echo "extra: setup (checks prerequisites, installs nothing) coverage self-audit update-audit-baseline verify-rules crosscheck recipe-lint adr-lint masters-check eval-expected-check update-masters workflow-lint commit-lint spell spell-fix update-secrets-baseline validate-skills"
@echo "docs: docs-lint docs-lint-all docs-prose [FILES=...] docs-mermaid FILES=... changelog-fix"

# There is nothing to install into the project: the runtime code is standard-library Python and the dev tools run
Expand Down Expand Up @@ -292,6 +292,10 @@ crosscheck:
recipe-lint:
$(PYTHON) dev/recipe_lint.py .

# Every ADR follows docs/ADR_POLICY.md: no lifecycle fields or tombstones, the four sections, a row in the root index.
adr-lint:
$(PYTHON) dev/adr_lint.py .

# The audit and implement master lists (docs/reference/*-master.md) are generated from the catalogue, the check files
# and the playbooks. masters-check fails when either page is stale; update-masters rewrites them.
masters-check:
Expand Down Expand Up @@ -326,7 +330,7 @@ RANGE ?= origin/dev..HEAD
commit-lint:
$(PYTHON) dev/commit_lint.py --range $(RANGE)

check: format-check typecheck test verify-rules crosscheck recipe-lint masters-check eval-expected-check workflow-lint spell
check: format-check typecheck test verify-rules crosscheck recipe-lint adr-lint masters-check eval-expected-check workflow-lint spell

# No third-party runtime dependencies exist to audit, so the gate is a secrets scan. The baseline
# lists the deliberate key-shaped strings in the fixtures and tests; a new hit fails the scan.
Expand Down
16 changes: 11 additions & 5 deletions audit/checks/03-documentation-contract.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Documentation Contract (`DOC`)

> Follows the anatomy in [`README.md`](README.md). Five checks, all `method: automated`. See

Check notice on line 3 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 38 words (limit 25)
> [Why These Checks §4](../engine/docs/architecture/user-overview/03-why-these-checks.md#4-documentation-contract-doc)
> for the consequence behind this family: without a declared, canonical home for permanent knowledge,
> documentation scatters and nothing is canonical — an agent cannot tell a settled architecture decision
Expand All @@ -14,7 +14,7 @@

## DOC-01 — `docs/` exists

Why it matters `docs/` is the declared home for permanent documentation; without it on day one,

Check notice on line 17 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 28 words (limit 25)

Check notice on line 17 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 28 words (limit 25)

Check notice on line 17 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 29 words (limit 25)

Check notice on line 17 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 36 words (limit 25)

Check notice on line 17 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 31 words (limit 25)

Check notice on line 17 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 26 words (limit 25)
documentation has nowhere canonical to start
([greenfield/README.md § 5.1 Diátaxis Framework](../../greenfield/README.md#51-diátaxis-framework)).
Severity DEGRADER
Expand Down Expand Up @@ -45,7 +45,7 @@

## DOC-02 — `docs/STYLE_GUIDE.md` exists

Why it matters Agents need authoring rules they can cite instead of restating per page, or rules get

Check warning on line 48 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 49 words (limit 25)

Check notice on line 48 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 40 words (limit 25)

Check notice on line 48 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 36 words (limit 25)

Check notice on line 48 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 30 words (limit 25)
restated and drift
([greenfield/README.md § 5.2 Templates & Guides](../../greenfield/README.md#52-templates--guides)).
Severity DEGRADER
Expand Down Expand Up @@ -80,7 +80,7 @@

## DOC-03 — `artefacts/` exists and is ignored

Why it matters Transient scratch must be separable and off-limits, or an agent reads work-in-progress

Check warning on line 83 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 61 words (limit 25)

Check warning on line 83 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 41 words (limit 25)

Check warning on line 83 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 57 words (limit 25)

Check warning on line 83 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 42 words (limit 25)
notes as the current, settled requirement
([greenfield/README.md § 6.2 Agent Scratchpad](../../greenfield/README.md#62-agent-scratchpad--artefacts)).
Severity DEGRADER
Expand Down Expand Up @@ -133,7 +133,7 @@

## DOC-04 — ADR directory and index

Why it matters Architectural decisions need a location and a registry, or the rationale lives only in

Check warning on line 136 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 44 words (limit 25)

Check warning on line 136 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 43 words (limit 25)

Check warning on line 136 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 43 words (limit 25)

Check warning on line 136 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 47 words (limit 25)

Check warning on line 136 in audit/checks/03-documentation-contract.md

View workflow job for this annotation

GitHub Actions / docs

PL001

sentence has 59 words (limit 25)
chat and pull requests, and agents re-litigate settled decisions
([greenfield/README.md § 5.2 Templates & Guides](../../greenfield/README.md#52-templates--guides)).
Severity COSMETIC
Expand All @@ -144,23 +144,29 @@
index (a `README.md`/`index.md`/`_index.md` beside them, or the parent directory's own
README) is `PARTIAL`; an index that lists some but not all entries (matched by filename or
the first four characters of the ADR number) is `PARTIAL`; every entry listed is `PASS`.
An entry that is a tombstone is `PARTIAL` even when the index lists every entry: a
status line or field, or a `## Status` section, that says superseded, deprecated,
rejected or obsolete, or a line that opens with "superseded by", "replaced by" or
"deprecated by / in favour of". The word inside a sentence or an alternatives table
does not count, and `Status: Accepted` is not flagged.
Distrust a `PARTIAL` on "index omits some" when the index links to entries by a title that
does not embed the filename or number fragment the engine's matcher looks for — verify by
reading the index for a link to each ADR's actual path.
Method automated
Look for `docs/architecture/adrs/` first (the convention this project's own docs use), else
`docs/adr/`; the index file beside the entries, or the architecture README that lists them
instead.
Rubric PASS: a numbered-ADR directory exists, holds at least one entry, and an index lists every
entry.
Rubric PASS: a numbered-ADR directory exists, holds at least one entry, an index lists every
entry, and no entry is a tombstone.
PARTIAL: the directory exists with entries but no index, or an index that omits some, or
the directory exists with no numbered entry at all.
the directory exists with no numbered entry at all, or an entry is marked superseded,
deprecated, rejected or obsolete (an agent loads it as a current rule).
FAIL: no ADR directory (at either known location) exists.
NOT_APPLICABLE: never — `REQ-SH-019` is a `SHOULD`, but the catalogue still scores it as
`DOC-04`, at `COSMETIC` severity, so it is never excused as not applicable.
UNKNOWN when: never — directory and file presence are always knowable from the inventory.
Evidence to cite A `file` evidence item per unlisted entry (for the omits-some case), or the directory path
alone (for the no-directory or no-entry cases).
Evidence to cite A `file` evidence item per unlisted entry (for the omits-some case), the file and line of
each tombstone, or the directory path alone (for the no-directory or no-entry cases).
Absence proof A FAIL records a `glob` search for `docs/architecture/adrs/*.md` and `docs/adr/*.md`, each
with a `0` hit count.
Remediation Playbook: `establish-docs-contract`. Add `docs/architecture/adrs/` with
Expand Down
2 changes: 1 addition & 1 deletion audit/engine/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ The framework's verbs are the contract. The runner is the **root `Makefile`** of
| `lint` | `make lint` | ruff lint rules over `audit`, `implement` and `dev`, read-only. |
| `typecheck` | `make typecheck` | mypy strict, on `audit/tools`, `implement/tools`, `dev` and this directory's `build.py`. The rest of the engine joins later through a ratchet. |
| `test` | `make test` | `python3 -m unittest discover -s tests` in this directory, then the audit tools' suite in `audit/tools/tests`, the implement tools' in `implement/tools/tests`, then the dev checks' in `dev/tests`. |
| `check` | `make check` | `format:check`, `typecheck`, `test`, `--verify-rules`, the schema cross-check (`crosscheck`) and `recipe-lint`. |
| `check` | `make check` | `format:check`, `typecheck`, `test`, `--verify-rules`, the schema cross-check (`crosscheck`), `recipe-lint` and `adr-lint`. |
| `security` | `make security` | Secrets scan against the committed baseline, then osv-scanner on the dev tools' lockfiles. The engine itself has no dependencies to audit. |

The dev tools (ruff, mypy, detect-secrets) are pinned in `dev/tools/pyproject.toml` and run through `uvx`. They
Expand Down
2 changes: 1 addition & 1 deletion audit/engine/audit/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
#: Bumped whenever a rule pack, a check, or the instruction-variant table changes.
#: Recorded in every report's provenance block so a report can be attributed to an
#: exact catalogue revision (docs: Scan Engine & Tiering, § 7 Determinism).
__ruleset_revision__ = "2026-11-10.4"
__ruleset_revision__ = "2026-11-10.5"

#: Highest framework phase this build evaluates. Phase 1 only in v1.
__phase_ceiling__ = 1
122 changes: 87 additions & 35 deletions audit/engine/audit/rules/checks/doc.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@
rather than infer it (the same degradation `SEC-01` uses).
* **DOC-04** — an ADR directory (`docs/architecture/adrs/` or `docs/adr/`) with numbered entries and
an index (a `README.md`/`index.md` beside them, or the directory's parent README) that lists them.
No directory is `FAIL`; entries without an index, or an index that omits some, is `PARTIAL`.
No directory is `FAIL`; entries without an index, or an index that omits some, is `PARTIAL`. An entry that
is a tombstone (:func:`_adr_tombstone`: marked superseded, deprecated, rejected or obsolete) is `PARTIAL`
too: an agent loads it as a current instruction.
* **DOC-05** — the plain-language measures of the prose in `docs/` (``audit.rules.checks._prose``): the share of
sentences over 25 words (20 in a numbered step), vague words per 1,000 words, and explanation or tutorial pages
without a Mermaid diagram. Within every pass level is `PASS`; a long-sentence share over
Expand Down Expand Up @@ -311,6 +313,38 @@ def _adr_entries(inventory: Inventory, directory: str) -> list[str]:
)


#: A status line or field that retires the record: `Status: Superseded`, `* **Status:** Deprecated`,
#: `| Status | Rejected |` or front matter `status: superseded by ADR-0005`.
_TOMBSTONE_STATUS_RE = re.compile(
r"^[ \t>*_|#-]*status[ \t*_:|-]*(superseded|deprecated|rejected|obsolete)\b", re.IGNORECASE | re.MULTILINE
)
#: A line that opens by pointing at the record that replaced this one.
_TOMBSTONE_POINTER_RE = re.compile(
r"^[ \t>*_-]*(superseded\s+by|replaced\s+by|deprecated\s+(by|in\s+favou?r\s+of))\b",
re.IGNORECASE | re.MULTILINE,
)
#: The MADR form: a `## Status` heading whose first line of text retires the record.
_STATUS_SECTION_RE = re.compile(
r"^#{1,6}\s*status\s*\n(?:[ \t]*\n)*[ \t>*_-]*(superseded|deprecated|rejected|obsolete)\b",
re.IGNORECASE | re.MULTILINE,
)


def _adr_tombstone(text: str) -> int | None:
"""The 1-based line that marks this ADR as retired, or ``None`` for a current record.

Only a status line or field, a `## Status` section, or a line that opens with "superseded by",
"replaced by" or "deprecated by / in favour of" counts. A word like "superseded" inside a sentence or a
table of alternatives does not, so an ADR may still discuss a rejected option.
"""
hits = [
m.start() for rx in (_TOMBSTONE_STATUS_RE, _TOMBSTONE_POINTER_RE, _STATUS_SECTION_RE) for m in rx.finditer(text)
]
if not hits:
return None
return text.count("\n", 0, min(hits)) + 1


def _adr_index(inventory: Inventory, directory: str) -> str | None:
"""A README/index beside the ADRs, falling back to the parent directory's README."""
for candidate in (f"{directory}/README.md", f"{directory}/index.md", f"{directory}/_index.md"):
Expand Down Expand Up @@ -350,55 +384,73 @@ def check_doc04(*, spec, target, inventory, stack, components, session) -> Check
return _outcome(spec, Verdict.PARTIAL, f"{directory}/ exists but holds no numbered ADR")

index = _adr_index(inventory, directory)
findings: list[Finding] = []
summaries: list[str] = []

if index is None:
return _outcome(
spec,
Verdict.PARTIAL,
f"{len(entries)} ADR(s) present but no index file",
[
summaries.append(f"{len(entries)} ADR(s) present but no index file")
findings.append(
Finding(
check=spec.id,
severity=spec.severity,
phase=spec.phase,
verdict=Verdict.PARTIAL,
statement=statement(
"discover the decisions of record from one page",
f"{directory}/ holds {len(entries)} ADR(s) but no index lists them",
),
evidence=[Evidence(rel) for rel in entries[:12]],
remediation="Add a README (or index) beside the ADRs, or list them from the architecture README.",
)
)
else:
text = inventory.read(index) or ""
listed = [
rel
for rel in entries
if rel.rsplit("/", 1)[-1] in text or re.search(rf"\b{rel.rsplit('/', 1)[-1][:4]}\b", text)
]
if len(listed) < len(entries):
summaries.append(f"{len(listed)}/{len(entries)} ADR(s) listed in {index}")
findings.append(
Finding(
check=spec.id,
severity=spec.severity,
phase=spec.phase,
verdict=Verdict.PARTIAL,
statement=statement(
"discover the decisions of record from one page",
f"{directory}/ holds {len(entries)} ADR(s) but no index lists them",
"find every decision of record",
f"{index} lists {len(listed)} of {len(entries)} ADRs in {directory}/",
),
evidence=[Evidence(rel) for rel in entries[:12]],
remediation="Add a README (or index) beside the ADRs, or list them from the architecture README.",
),
],
)

text = inventory.read(index) or ""
listed = [
rel
for rel in entries
if rel.rsplit("/", 1)[-1] in text or re.search(rf"\b{rel.rsplit('/', 1)[-1][:4]}\b", text)
]
if len(listed) == len(entries):
return _outcome(spec, Verdict.PASS, f"{len(entries)} ADR(s) indexed in {index}")

return _outcome(
spec,
Verdict.PARTIAL,
f"{len(listed)}/{len(entries)} ADR(s) listed in {index}",
[
evidence=[Evidence(index)],
remediation="Add a row to the ADR index for each entry, in the same change set that adds the ADR.",
)
)

tombstones = [(rel, line) for rel in entries if (line := _adr_tombstone(inventory.read(rel) or "")) is not None]
if tombstones:
summaries.append(f"{len(tombstones)} ADR(s) marked superseded, deprecated or rejected")
findings.append(
Finding(
check=spec.id,
severity=spec.severity,
phase=spec.phase,
verdict=Verdict.PARTIAL,
statement=statement(
"find every decision of record",
f"{index} lists {len(listed)} of {len(entries)} ADRs in {directory}/",
"trust that every ADR it loads is a current decision",
f"{len(tombstones)} ADR(s) in {directory}/ are marked superseded, deprecated or rejected, "
"and an agent can apply them as current rules",
),
evidence=[Evidence(index)],
remediation="Add a row to the ADR index for each entry, in the same change set that adds the ADR.",
),
],
)
evidence=[Evidence(rel, line, note="retired record") for rel, line in tombstones[:12]],
remediation="Delete each retired ADR, with its index row and every reference to it; git keeps its "
"history. When a decision changed, overwrite the current ADR in place and list the old choice under "
"Alternatives Considered.",
)
)

if findings:
return _outcome(spec, Verdict.PARTIAL, "; ".join(summaries), findings)
return _outcome(spec, Verdict.PASS, f"{len(entries)} ADR(s) indexed in {index}")


# ===========================================================================
Expand Down
Loading
Loading