ci: check docs-pipeline checkpoints and code examples [PANL-015]

.github/scripts/check_examples.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Check that the code examples in the Markdown docs are well formed.
+
+Run from the repo root:  python3 .github/scripts/check_examples.py
+
+For every fenced block tagged json, yaml/yml or bash/sh, it checks that the block
+parses (json.loads, yaml.safe_load, bash -n). Blocks tagged with anything else are
+skipped. Saved model outputs under tests/ are skipped: they quote examples on purpose.
+A block that holds placeholders on purpose can opt out with an HTML comment on the
+line above the fence:  <!-- example-check: skip -->
+
+Exit code 0 means clean, 1 means problems were printed.
+"""
+import json
+import pathlib
+import re
+import subprocess
+import sys
+
+import yaml
+
+ROOT = pathlib.Path(__file__).resolve().parents[2]
+SKIP_DIRS = {".git", "tests", "node_modules"}
+FENCE = re.compile(r"^(?P<ind>\s*)```(?P<lang>[A-Za-z0-9_+-]*)\s*$")
+
+problems = []
+checked = 0
+
+
+def check(lang, body):
+    if lang == "json":
+        json.loads(body)
+    elif lang in ("yaml", "yml"):
+        list(yaml.safe_load_all(body))
+    elif lang in ("bash", "sh", "shell"):
+        r = subprocess.run(["bash", "-n"], input=body, text=True, capture_output=True)
+        if r.returncode:
+            raise ValueError(r.stderr.strip().splitlines()[-1] if r.stderr.strip() else "bash -n failed")
+
+
+for md in sorted(ROOT.rglob("*.md")):
+    rel = md.relative_to(ROOT)
+    if SKIP_DIRS & set(rel.parts):
+        continue
+    lines = md.read_text(encoding="utf-8").splitlines()
+    i = 0
+    while i < len(lines):
+        m = FENCE.match(lines[i])
+        if m and m.group("lang"):
+            lang, start = m.group("lang").lower(), i
+            skip = i > 0 and "example-check: skip" in lines[i - 1]
+            j = i + 1
+            while j < len(lines) and not FENCE.match(lines[j]):
+                j += 1
+            ind = len(m.group("ind"))
+            body = "\n".join(l[ind:] if l[:ind].isspace() else l for l in lines[i + 1:j]) + "\n"
+            if lang in {"json", "yaml", "yml", "bash", "sh", "shell"} and not skip:
+                checked += 1
+                try:
+                    check(lang, body)
+                except Exception as e:  # any parse error is a finding
+                    problems.append(f"{rel}:{start + 1}: {lang} block does not parse: {str(e).splitlines()[0]}")
+            i = j + 1
+        else:
+            i += 1
+
+if problems:
+    print(f"check_examples: {len(problems)} problem(s)")
+    for p in problems:
+        print("  " + p)
+    sys.exit(1)
+print(f"check_examples: clean ({checked} blocks checked)")

.github/workflows/ci.yml

@@ -38,6 +38,20 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+  docs-checks:
+    name: docs checks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.12'
+      - run: pip install pyyaml
+      - name: The docs pipeline is complete and every stage ends in a checkpoint
+        run: python3 pipelines/docs-pipeline/tests/check_pipeline.py
+      - name: Code examples in the Markdown docs parse
+        run: python3 .github/scripts/check_examples.py
+
   tests:
     name: tests (${{ matrix.os }}, python ${{ matrix.python }})
     runs-on: ${{ matrix.os }}

pipelines/docs-pipeline/tests/check_pipeline.py

@@ -3,11 +3,16 @@
 
 Run from anywhere:  python3 pipelines/docs-pipeline/tests/check_pipeline.py
 
-It needs no model and no network. It checks four things:
+It needs no model and no network. It checks six things:
 1. Every `./_knowledge/...` path a skill file names exists in this folder.
 2. Every skill file has the frontmatter name the install loop relies on.
 3. No file mentions a private tool, home path, or leftover setup.
 4. The sample doc, the .gitignore template and the product knowledge starters ship.
+5. The orchestrator names every stage skill, so no stage can be dropped from the chain unnoticed.
+6. Every stage summary in the orchestrator ends with a "Proceed to ..." checkpoint, so no stage hands off without the human deciding to continue.
+
+The checkpoint is a prompt to the person running the pipeline, not code that blocks a stage.
+This script checks that the checkpoints are written; it cannot check that a model obeys them.
 
 Exit code 0 means clean, 1 means problems were printed.
 """
@@ -63,6 +68,26 @@
 if not (READABILITY / "README.md").is_file():
     problems.append("missing skills/docs-readability-check/README.md")
 
+# 5. orchestrator names every stage skill
+ORCH = HERE / "docs-pipeline.md"
+orch_text = ORCH.read_text(encoding="utf-8")
+for f in skill_files:
+    if f.name in {"docs-pipeline.md", "docs-workspace-setup.md"}:
+        continue
+    name = "docs-readability-check" if f.name == "SKILL.md" else f.stem
+    if f"/{name}" not in orch_text:
+        problems.append(f"docs-pipeline.md: never calls /{name}")
+
+# 6. every stage summary ends with a checkpoint
+sections = re.split(r"(?m)^(?=#{2,3} Stage )", orch_text)
+for sec in sections:
+    heading = sec.split("\n", 1)[0]
+    if not heading.startswith("#"):
+        continue
+    ends_stage = re.search(r"(?i)complete|written", heading) or re.match(r"### Stage 4[a-c]", heading)
+    if ends_stage and "Proceed to" not in sec:
+        problems.append(f"docs-pipeline.md: '{heading.lstrip('# ')}' has no 'Proceed to' checkpoint")
+
 if problems:
     print(f"check_pipeline: {len(problems)} problem(s)")
     for p in problems: