From 6eabee17a87e6676cd0642d076f25fa68ebc1008 Mon Sep 17 00:00:00 2001 From: darthrootbeer Date: Mon, 26 Oct 2026 20:40:40 -0400 Subject: [PATCH] ci: check docs-pipeline checkpoints and code examples Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Bihc4e3Pg6Ago5K1xfMbe5 --- .github/scripts/check_examples.py | 72 +++++++++++++++++++ .github/workflows/ci.yml | 14 ++++ .../docs-pipeline/tests/check_pipeline.py | 27 ++++++- 3 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/check_examples.py diff --git a/.github/scripts/check_examples.py b/.github/scripts/check_examples.py new file mode 100644 index 0000000..19e7a8c --- /dev/null +++ b/.github/scripts/check_examples.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Check that the code examples in the Markdown docs are well formed. + +Run from the repo root: python3 .github/scripts/check_examples.py + +For every fenced block tagged json, yaml/yml or bash/sh, it checks that the block +parses (json.loads, yaml.safe_load, bash -n). Blocks tagged with anything else are +skipped. Saved model outputs under tests/ are skipped: they quote examples on purpose. +A block that holds placeholders on purpose can opt out with an HTML comment on the +line above the fence: + +Exit code 0 means clean, 1 means problems were printed. +""" +import json +import pathlib +import re +import subprocess +import sys + +import yaml + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SKIP_DIRS = {".git", "tests", "node_modules"} +FENCE = re.compile(r"^(?P\s*)```(?P[A-Za-z0-9_+-]*)\s*$") + +problems = [] +checked = 0 + + +def check(lang, body): + if lang == "json": + json.loads(body) + elif lang in ("yaml", "yml"): + list(yaml.safe_load_all(body)) + elif lang in ("bash", "sh", "shell"): + r = subprocess.run(["bash", "-n"], input=body, text=True, capture_output=True) + if r.returncode: + raise ValueError(r.stderr.strip().splitlines()[-1] if r.stderr.strip() else "bash -n failed") + + +for md in sorted(ROOT.rglob("*.md")): + rel = md.relative_to(ROOT) + if SKIP_DIRS & set(rel.parts): + continue + lines = md.read_text(encoding="utf-8").splitlines() + i = 0 + while i < len(lines): + m = FENCE.match(lines[i]) + if m and m.group("lang"): + lang, start = m.group("lang").lower(), i + skip = i > 0 and "example-check: skip" in lines[i - 1] + j = i + 1 + while j < len(lines) and not FENCE.match(lines[j]): + j += 1 + ind = len(m.group("ind")) + body = "\n".join(l[ind:] if l[:ind].isspace() else l for l in lines[i + 1:j]) + "\n" + if lang in {"json", "yaml", "yml", "bash", "sh", "shell"} and not skip: + checked += 1 + try: + check(lang, body) + except Exception as e: # any parse error is a finding + problems.append(f"{rel}:{start + 1}: {lang} block does not parse: {str(e).splitlines()[0]}") + i = j + 1 + else: + i += 1 + +if problems: + print(f"check_examples: {len(problems)} problem(s)") + for p in problems: + print(" " + p) + sys.exit(1) +print(f"check_examples: clean ({checked} blocks checked)") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11071e8..6f594cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,20 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + docs-checks: + name: docs checks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + - run: pip install pyyaml + - name: The docs pipeline is complete and every stage ends in a checkpoint + run: python3 pipelines/docs-pipeline/tests/check_pipeline.py + - name: Code examples in the Markdown docs parse + run: python3 .github/scripts/check_examples.py + tests: name: tests (${{ matrix.os }}, python ${{ matrix.python }}) runs-on: ${{ matrix.os }} diff --git a/pipelines/docs-pipeline/tests/check_pipeline.py b/pipelines/docs-pipeline/tests/check_pipeline.py index 8c56a66..6f6d8a7 100644 --- a/pipelines/docs-pipeline/tests/check_pipeline.py +++ b/pipelines/docs-pipeline/tests/check_pipeline.py @@ -3,11 +3,16 @@ Run from anywhere: python3 pipelines/docs-pipeline/tests/check_pipeline.py -It needs no model and no network. It checks four things: +It needs no model and no network. It checks six things: 1. Every `./_knowledge/...` path a skill file names exists in this folder. 2. Every skill file has the frontmatter name the install loop relies on. 3. No file mentions a private tool, home path, or leftover setup. 4. The sample doc, the .gitignore template and the product knowledge starters ship. +5. The orchestrator names every stage skill, so no stage can be dropped from the chain unnoticed. +6. Every stage summary in the orchestrator ends with a "Proceed to ..." checkpoint, so no stage hands off without the human deciding to continue. + +The checkpoint is a prompt to the person running the pipeline, not code that blocks a stage. +This script checks that the checkpoints are written; it cannot check that a model obeys them. Exit code 0 means clean, 1 means problems were printed. """ @@ -63,6 +68,26 @@ if not (READABILITY / "README.md").is_file(): problems.append("missing skills/docs-readability-check/README.md") +# 5. orchestrator names every stage skill +ORCH = HERE / "docs-pipeline.md" +orch_text = ORCH.read_text(encoding="utf-8") +for f in skill_files: + if f.name in {"docs-pipeline.md", "docs-workspace-setup.md"}: + continue + name = "docs-readability-check" if f.name == "SKILL.md" else f.stem + if f"/{name}" not in orch_text: + problems.append(f"docs-pipeline.md: never calls /{name}") + +# 6. every stage summary ends with a checkpoint +sections = re.split(r"(?m)^(?=#{2,3} Stage )", orch_text) +for sec in sections: + heading = sec.split("\n", 1)[0] + if not heading.startswith("#"): + continue + ends_stage = re.search(r"(?i)complete|written", heading) or re.match(r"### Stage 4[a-c]", heading) + if ends_stage and "Proceed to" not in sec: + problems.append(f"docs-pipeline.md: '{heading.lstrip('# ')}' has no 'Proceed to' checkpoint") + if problems: print(f"check_pipeline: {len(problems)} problem(s)") for p in problems: