From cbab47a51fb60d21db954cf3cee7391ddc24893e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Mon, 5 Oct 2026 09:00:00 +0200 Subject: [PATCH 01/13] feat(notify-umbrella): send submodule-updated on every push, docs-updated on a docs diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MIP-0076 §5.5 step 1's half for notify-umbrella.yml. Until now every repo's notify-umbrella sent only submodule-docs-updated, and only because the caller's paths: filter made that the only trigger — pointer-sync.yml listens for submodule-updated too, but nothing has ever sent it (MIP-0076 §2, confirmed 2026-10-03). The workflow now checks out the caller's own repo and diffs github.event.before..github.sha itself to decide: submodule-updated goes on every call, submodule-docs-updated goes too when that range touches README.md or docs/** — so a caller drops its paths: filter and runs on every push. A zero or unreachable before (a branch's first push, or history rewritten out from under it) counts as a docs change, so both events go rather than silently dropping one. event-type stays declared, now unused, instead of being dropped: GitHub fails a reusable-workflow call outright if the caller still passes an input the called workflow no longer declares, and today's five callers are updated to drop paths: in later rows of this MIP, not this one. Closes #27 Tested: actionlint ok (repo-wide); shellcheck --severity=error ok; ruff check/format ok (uvx ruff 0.16.9, no nix); bash tests/self-tests.sh all ok; scripts/agents-check.sh ok; python3 scripts/docs_lint.py clean; claude plugin validate . passed Cost: ~$5.14 est. · ~2.6M tokens est. (diff-size model, 88 lines, no session log, IQR 0.6M-13.5M tokens for this diff from 75 calibrated commits) · scripts/cost-split.py --estimate 2026-10-05 Co-Authored-By: Claude --- .github/workflows/notify-umbrella.yml | 72 +++++++++++++++++++-------- docs/4-reference_workflows.md | 16 ++++-- 2 files changed, 62 insertions(+), 26 deletions(-) diff --git a/.github/workflows/notify-umbrella.yml b/.github/workflows/notify-umbrella.yml index b4a7f54..205c7f0 100644 --- a/.github/workflows/notify-umbrella.yml +++ b/.github/workflows/notify-umbrella.yml @@ -1,14 +1,15 @@ -# notify-umbrella — tell the umbrella a repo's docs changed, so it rebuilds within minutes -# instead of waiting for its daily cron (MIP-0070 §5.5). Modelled on h0ffmann/nix-config's -# profile-ping.yml reusable workflow — a plain repository_dispatch, no checkout needed on either -# side. +# notify-umbrella — tell the umbrella every push to main (`submodule-updated`), so +# pointer-sync.yml can move pointers on event instead of its daily cron, and that a repo's docs +# changed too (`submodule-docs-updated`) when the push touches README.md or docs/**, so +# docs.yml rebuilds within minutes (MIP-0070 §5.5, MIP-0076 §5.5 step 1). Modelled on +# h0ffmann/nix-config's profile-ping.yml reusable workflow. # -# A consumer adds about three lines: +# A consumer adds about three lines, no `paths:` filter — this workflow decides for itself which +# event types a given push earns: # # on: # push: # branches: [main] -# paths: [README.md, docs/**] # jobs: # notify: # uses: marola-dev/marola-devkit/.github/workflows/notify-umbrella.yml@v0.5.0 @@ -28,9 +29,9 @@ on: type: string default: marola-dev/marola event-type: - description: "repository_dispatch event_type the umbrella's docs workflow listens for." + description: "Deprecated, ignored (MIP-0076): every call now sends submodule-updated, plus submodule-docs-updated on a docs change. Kept declared so a caller still passing it doesn't fail GitHub's reusable-workflow input check." type: string - default: submodule-docs-updated + default: "" runner: description: "Runner label for the dispatch job — a reusable workflow's runs-on resolves in the caller's repository." type: string @@ -40,33 +41,62 @@ on: description: "Fine-grained PAT with Contents: read & write on the umbrella." required: false -permissions: {} +permissions: + contents: read jobs: dispatch: runs-on: ${{ inputs.runner }} - timeout-minutes: 2 + timeout-minutes: 5 steps: - - name: repository_dispatch ${{ inputs.event-type }} -> ${{ inputs.umbrella }} + # Full history: github.event.before can be arbitrarily far back, and the diff below needs it. + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Did this push touch README.md or docs/**? + id: docs + env: + BEFORE: ${{ github.event.before }} + SHA: ${{ github.sha }} + run: | + set -euo pipefail + changed=false + if [ -z "${BEFORE:-}" ] || [ "$BEFORE" = "0000000000000000000000000000000000000000" ]; then + changed=true # first push on the branch: no prior ref to diff against + elif ! git cat-file -e "$BEFORE^{commit}" 2>/dev/null; then + changed=true # before rewritten out of history (e.g. a force-push) + elif [ -n "$(git diff --name-only "$BEFORE" "$SHA" -- README.md docs)" ]; then + changed=true + fi + echo "changed=$changed" >>"$GITHUB_OUTPUT" + + - name: repository_dispatch -> ${{ inputs.umbrella }} env: TOKEN: ${{ secrets.token }} UMBRELLA: ${{ inputs.umbrella }} - EVENT_TYPE: ${{ inputs.event-type }} REPO: ${{ github.repository }} SHA: ${{ github.sha }} + DOCS_CHANGED: ${{ steps.docs.outputs.changed }} run: | set -euo pipefail if [ -z "$TOKEN" ]; then echo "::notice::no umbrella-dispatch token set in $REPO — skipping; the umbrella's daily cron still picks this up" exit 0 fi - body="$(jq -nc --arg repo "$REPO" --arg sha "$SHA" --arg event "$EVENT_TYPE" \ - '{event_type: $event, client_payload: {repo: $repo, sha: $sha}}')" - code="$(curl -sS -o "$RUNNER_TEMP/notify-umbrella-response.txt" -w '%{http_code}' -X POST \ - -H "Authorization: Bearer $TOKEN" -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/$UMBRELLA/dispatches" -d "$body")" - if [ "$code" != 204 ]; then - echo "::warning::dispatch to $UMBRELLA answered HTTP $code: $(head -c 300 "$RUNNER_TEMP/notify-umbrella-response.txt")" - exit 0 + events=(submodule-updated) + if [ "$DOCS_CHANGED" = true ]; then + events+=(submodule-docs-updated) fi - echo "dispatched $EVENT_TYPE for $REPO@$SHA to $UMBRELLA" + for event in "${events[@]}"; do + body="$(jq -nc --arg repo "$REPO" --arg sha "$SHA" --arg event "$event" \ + '{event_type: $event, client_payload: {repo: $repo, sha: $sha}}')" + code="$(curl -sS -o "$RUNNER_TEMP/notify-umbrella-response.txt" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer $TOKEN" -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$UMBRELLA/dispatches" -d "$body")" + if [ "$code" != 204 ]; then + echo "::warning::dispatch of $event to $UMBRELLA answered HTTP $code: $(head -c 300 "$RUNNER_TEMP/notify-umbrella-response.txt")" + else + echo "dispatched $event for $REPO@$SHA to $UMBRELLA" + fi + done diff --git a/docs/4-reference_workflows.md b/docs/4-reference_workflows.md index a58dc8f..32054da 100644 --- a/docs/4-reference_workflows.md +++ b/docs/4-reference_workflows.md @@ -114,16 +114,19 @@ No secrets. ## notify-umbrella -Tells the umbrella a repo's docs changed via `repository_dispatch`, so it rebuilds within minutes -instead of at its next daily cron (MIP-0070 §5.5). Modelled on h0ffmann/nix-config's -`profile-ping.yml` — no checkout on either side, ~3 lines to call. +Tells the umbrella every push to `main` via `repository_dispatch` (`submodule-updated`), so +pointer-sync.yml can move pointers on the event instead of waiting for its daily cron, and that a +repo's docs changed too (`submodule-docs-updated`) when the push touched `README.md` or `docs/**`, +so docs.yml rebuilds within minutes (MIP-0070 §5.5, MIP-0076 §5.5 step 1). Modelled on +h0ffmann/nix-config's `profile-ping.yml`. The caller needs no `paths:` filter: this workflow +checks out the caller's own repo and diffs `github.event.before..github.sha` itself to decide +which event types a given push earns. ```yaml name: notify umbrella on: push: branches: [main] - paths: [README.md, docs/**] jobs: notify: uses: marola-dev/marola-devkit/.github/workflows/notify-umbrella.yml@v0.5.0 @@ -134,7 +137,7 @@ jobs: | Input | Default | Notes | |---|---|---| | `umbrella` | `marola-dev/marola` | MAROLA_UMBRELLA, MIP-0070 §5.6 | -| `event-type` | `submodule-docs-updated` | what the umbrella's docs workflow listens for | +| `event-type` | `""` | deprecated, ignored (MIP-0076) — kept so a caller still passing it doesn't fail GitHub's input check | | `runner` | `ubuntu-latest` | resolved in the *caller's* repo | **Secret** `token` (optional): every repo passes the org secret `MAROLA_CROSS_REPO_PAT`, a @@ -142,6 +145,9 @@ fine-grained PAT with Contents: read & write on the umbrella (`repository_dispat `GITHUB_TOKEN` cannot reach another repo). Unset is a notice, not a failure — the umbrella's daily cron still catches the change. +A zero or unreachable `github.event.before` (a branch's first push, or history rewritten out from +under it) counts as a docs change, so both events go rather than silently dropping one. + ## labels-sync Reconciles the caller repo's labels against `scripts/issues.sh labels sync`'s own default manifest From 81646923240d885953293cc707d3ffab71950a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Mon, 5 Oct 2026 09:03:36 +0200 Subject: [PATCH 02/13] feat(wiring): extract the org's cross-repo wiring into four tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/wiring.py, MIP-0076 §5.1 without the gate: it reads .gitmodules, the devkit's tree under .devkit (else its own), every repo's workflows, pin files, and the scripts, justfiles, Dockerfiles, docker-compose*.yml and build.sbt naming a pin or image. It renders four tables (artifact, dispatch, pin bump, deploy) between a file's wiring:start/wiring:end markers. Beyond §5.1's sources, per the tasks file's decisions: branch artifacts (api-docs, site-data) from the `-push.sh` idiom, read by any file fetching the branch (Decision 1); compose `image:` lines as image readers (Decision 2); a workflow step opening a PR in or pushing to another marola-dev repo (Decision 3). A devkit reusable workflow's effects are attributed to each caller with the call's ref, and its inputs resolved from the caller's `with:`. Closes #25 Tested: wiring --self-test (13 cases, red against a stubbed scan, green after); a run over a fresh --recurse-submodules umbrella clone at 0145e12 Cost: ~$19.79 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- scripts/wiring.py | 422 ++++++++++++++++++++++++++++++++++++++++++++ tests/self-tests.sh | 2 +- 2 files changed, 423 insertions(+), 1 deletion(-) create mode 100755 scripts/wiring.py diff --git a/scripts/wiring.py b/scripts/wiring.py new file mode 100755 index 0000000..05c58f6 --- /dev/null +++ b/scripts/wiring.py @@ -0,0 +1,422 @@ +#!/usr/bin/env python3 +"""The org's cross-repo wiring, parsed from every repo's workflows, pins and scripts (MIP-0076 §5.1). + +Run in an umbrella checkout with its submodules; the devkit's tree is `.devkit`, else this script's. + +wiring [--root DIR] [FILE] print the block, or rewrite it between FILE's wiring markers +wiring --self-test +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +ORG = "marola-dev" +SELF = Path(__file__) # its fixture names every idiom, so it is never a reader +PINS = ("marola-image", "corpus.version", "resources.version") +START, END = "", "" +READERS = ("scripts/**/*", "justfile", "Dockerfile*", "docker-compose*.yml", "build.sbt") +USES = re.compile(rf"^{ORG}/([\w.-]+)/\.github/workflows/([\w.-]+)@([\w.-]+)$") +IMAGE = re.compile(rf"ghcr\.io/{ORG}/[\w.-]+") +RELEASE = re.compile(r"gh release (?:upload|create)\s+\S+\s+(\S+)") +ASSIGN = re.compile(r"""^\s*(\w+)=["']?([^"'\s]+)""", re.M) +SHVAR = re.compile(r"\$\{?(\w+)\}?") +EXPR = re.compile(r"\$\{\{\s*(env|inputs|github)\.([\w-]+)\s*\}\}") + + +class Wiring: + def __init__(self) -> None: + self.artifacts: dict[str, dict] = {} + self.dispatch: dict[str, dict[str, list[str]]] = {} + self.bumps: dict[str, list[str]] = {} + self.deploys: list[tuple[str, str]] = [] + self.devkit: dict[str, list[str]] = {} + self.calls: dict[str, dict[str, list[str]]] = {} + + def art(self, key: str, repo: str = "", match: str = "") -> dict: + a = self.artifacts.setdefault(key, {"pub": [], "pin": [], "read": [], "repos": set()}) + a["repos"].add(repo) + a.setdefault("match", match) + return a + + +def add(xs: list[str], x: str) -> None: + if x not in xs: + xs.append(x) + + +def sub(s: str, env: dict, inputs: dict) -> str: + def expr(m: re.Match) -> str: + ctx, name = m.groups() + if ctx == "github": + return ORG if name == "repository_owner" else m[0] + return str((env if ctx == "env" else inputs).get(name, m[0])) + + s = EXPR.sub(expr, str(s)) + return SHVAR.sub(lambda m: str(env.get(m[1], m[0])), s) + + +def lines(text: str) -> str: + return "\n".join(x for x in text.splitlines() if not x.lstrip().startswith(("#", "//"))) + + +def read(p: Path) -> str: + try: + return p.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return "" + + +def repos(root: Path) -> list[tuple[str, Path]]: + url = subprocess.run( + ["git", "-C", str(root), "config", "--get", "remote.origin.url"], + capture_output=True, + text=True, + check=False, + ).stdout.strip() + out = [(re.sub(r"\.git$", "", url.rsplit("/", 1)[-1]) or "marola", root)] + for path in re.findall(r"^\s*path\s*=\s*(\S+)", read(root / ".gitmodules"), re.M): + out.append((Path(path).name, root / path)) + devkit = root / ".devkit" + out.append( + ("marola-devkit", devkit if devkit.is_dir() else Path(__file__).resolve().parents[1]) + ) + return [(n, p) for n, p in out if p.is_dir()] + + +def triggers(doc: dict) -> dict: + on = doc.get("on", doc.get(True)) or {} + if isinstance(on, str | list): + on = {k: None for k in ([on] if isinstance(on, str) else on)} + return {k: v or {} for k, v in on.items()} + + +def when(on: dict) -> str: + push = on.get("push", {}) + if push.get("tags"): + return f"on a `{push['tags'][0]}` tag" + if "push" in on: + paths = push.get("paths") or [] + touching = " touching " + ", ".join(f"`{p}`" for p in paths) if 0 < len(paths) <= 3 else "" + return f"on a push to `{(push.get('branches') or ['main'])[0]}`{touching}" + if "schedule" in on: + return "on a schedule" + return "by hand" if set(on) == {"workflow_dispatch"} else "" + + +def site_of(repo_dir: Path, text: str) -> str: + if m := re.search(r"""echo\s+["']?([\w.-]+\.[a-z]+)["']?\s*>\s*\S*CNAME""", text): + return m[1] + for f in ("mkdocs/mkdocs.yml", "mkdocs.yml"): + if m := re.search(r"^site_url:\s*https?://([^/\s]+)", read(repo_dir / f), re.M): + return m[1] + return "?" + + +def effects(doc: dict, repo: str, inputs: dict, workflows: dict) -> list[tuple]: + """(kind, value, via) for what a workflow sends, publishes, deploys or writes elsewhere.""" + out: list[tuple] = [] + text = yaml.safe_dump(doc) + for job in (doc.get("jobs") or {}).values(): + env = {**(doc.get("env") or {}), **(job.get("env") or {})} + if m := USES.match(str(job.get("uses", ""))): + target = workflows.get((m[1], m[2])) + if target: + defaults = triggers(target).get("workflow_call", {}).get("inputs") or {} + given = {k: v.get("default", "") for k, v in defaults.items()} + given |= {k: sub(v, env, inputs) for k, v in (job.get("with") or {}).items()} + via = f"`{m[2]}@{m[3]}`" + out += [(k, v, via) for k, v, _ in effects(target, repo, given, workflows)] + out.append(("ref", (m[2], m[3]), "")) + continue + checkouts: dict[str, tuple[str, str]] = {} + for step in job.get("steps") or []: + senv = {**env, **(step.get("env") or {})} + uses, w = str(step.get("uses", "")), step.get("with") or {} + if uses.startswith("actions/checkout") and w.get("repository"): + name = sub(w["repository"], senv, inputs).rsplit("/", 1)[-1] + checkouts[str(w.get("path", "."))] = (name, str(w.get("ref", ""))) + if uses.startswith("docker/build-push-action") and ( + m := IMAGE.search(sub(env.get("IMAGE", ""), senv, inputs)) + ): + out.append(("image", m[0], "")) + if uses.startswith("actions/deploy-pages"): + out.append(("deploy", text, "")) + run = step.get("run") + if not run: + continue + assigned = dict(ASSIGN.findall(run)) + out += [("send", t, "") for t in re.findall(r"event_type=([\w-]+)", run)] + if "/dispatches" in run: + out += [("send", sub(v, {}, inputs), "") for k, v in senv.items() if "EVENT" in k] + for asset in RELEASE.findall(run): + name = sub(asset.strip("\"'"), assigned, {}).rsplit("/", 1)[-1] + out.append(("asset", SHVAR.sub("", name), "")) + if branches := re.findall(r"([\w-]+)-push\.sh\b", run): + for b in branches: + owner = next((r for r, ref in checkouts.values() if ref == b), repo) + out.append(("branch", (b, owner), "")) + continue + if m := re.search(r"gh pr create\b.*?--repo\s+(\S+)", run, re.S): + files = sorted( + {Path(f).name for f in " ".join(re.findall(r"git add\s+(.+)", run)).split()} + ) + out.append( + ("pr", (sub(m[1].strip("\"'"), senv, inputs).rsplit("/", 1)[-1], files), "") + ) + elif m := re.search(r"(?:git -C (\S+) |cd (\S+)[^\n]*\n(?:.*\n)*?\s*git )push\b", run): + target = checkouts.get((m[1] or m[2]).strip("\"'")) + if target: + out.append(("push", target[0], "")) + return [(k, v, via) for k, v, via in out if k not in ("pr", "push") or v[0] != repo] + + +def scan(root: Path) -> Wiring: + w, rs = Wiring(), repos(root) + docs: dict[tuple[str, str], dict] = {} + for name, d in rs: + for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): + docs[(name, f.name)] = yaml.safe_load(read(f)) or {} + for (repo, file), doc in docs.items(): + on = triggers(doc) + if set(on) == {"workflow_call"}: + continue + label = f"{repo} `{file}`" + for t in on.get("repository_dispatch", {}).get("types") or []: + add(w.dispatch.setdefault(t, {"sent": [], "listen": []})["listen"], label) + for pin in set(PINS + ("flake.lock",)) & set(on.get("push", {}).get("paths") or []): + add(w.bumps.setdefault(f"{repo} `{pin}`", []), label) + repo_dir = dict(rs)[repo] + for kind, v, via in effects(doc, repo, {}, docs): + parts = [p for p in (via, when(on)) if p] + pub = label + (f" ({', '.join(parts)})" if parts else "") + if kind == "send": + add(w.dispatch.setdefault(v, {"sent": [], "listen": []})["sent"], pub) + elif kind == "image": + add(w.art(f"`{v}` image", repo, v)["pub"], pub) + elif kind == "asset": + add(w.art(f"`{v}` release asset", repo)["pub"], pub) + elif kind == "branch": + b, owner = v + key = f"`{b}` branch" + (f" of {owner}" if owner != repo else "") + add(w.art(key, owner, b)["pub"], pub) + elif kind == "pr": + a = w.art(f"PRs into {v[0]}: " + ", ".join(f"`{f}`" for f in v[1]), v[0]) + add(a["pub"], pub), add(a["read"], v[0]) + elif kind == "push": + a = w.art(f"pushes to {v}", v) + add(a["pub"], pub), add(a["read"], v) + elif kind == "deploy": + w.deploys.append((label, site_of(repo_dir, v))) + elif kind == "ref": + add(w.calls.setdefault(v[0], {}).setdefault(v[1], []), repo) + for repo, d in rs: + files = sorted( + {f for g in READERS for f in d.glob(g) if f.is_file() and f.name != SELF.name} + ) + texts = {f.relative_to(d).as_posix(): lines(read(f)) for f in files} + for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): + texts[f".github/workflows/{f.name}"] = lines(read(f)) + for rel, text in texts.items(): + for img in re.findall(r"(?:image:|FROM)\s+(" + IMAGE.pattern + ")", text): + if (key := f"`{img}` image") in w.artifacts: + add(w.artifacts[key]["read"], f"{repo} `{Path(rel).name}`") + for key, a in w.artifacts.items(): + b = a["match"] + if " branch" not in key or rel.endswith(f"{b}-push.sh"): + continue + names = {b} | {v for v, x in ASSIGN.findall(text) if x == b} + if any( + "fetch" in ln + and any(re.search(rf"(? str: + def cell(xs) -> str: + return ", ".join(xs) or "—" + + out = ["| Artifact | Published by | Pinned in | Read by |", "|---|---|---|---|"] + kinds = (" image", "release asset", " branch", "PRs into", "pushes to") + for key, a in sorted( + w.artifacts.items(), key=lambda kv: ([k in kv[0] for k in kinds].index(True), kv[0]) + ): + out.append(f"| {key} | {cell(a['pub'])} | {cell(a['pin'])} | {cell(a['read'])} |") + pins = "; ".join(f"{r} {', '.join(v)}" for r, v in w.devkit.items()) or "—" + calls = "; ".join( + f"`{f}` " + ", ".join(f"{ref} ({' '.join(rs)})" for ref, rs in sorted(refs.items())) + for f, refs in sorted(w.calls.items()) + ) + out += [f"| a marola-devkit tag | marola-devkit | {pins} | {calls or '—'} |", ""] + out += ["| Dispatch | Sent by | Triggers |", "|---|---|---|"] + for t, d in sorted(w.dispatch.items()): + out.append(f"| `{t}` | {cell(d['sent'])} | {cell(d['listen'])} |") + out += ["", "| Pin bump | Workflow |", "|---|---|"] + out += [f"| {p} | {cell(wf)} |" for p, wf in sorted(w.bumps.items())] + out += ["", "| Deploy | Site |", "|---|---|"] + out += [f"| {wf} | {site} |" for wf, site in w.deploys] + return "\n".join(out) + "\n" + + +def write_block(path: Path, block: str) -> None: + text = path.read_text(encoding="utf-8") + head, sep, rest = text.partition(START) + _, sep2, tail = rest.partition(END) + if not (sep and sep2): + raise SystemExit(f"wiring: {path} has no {START} … {END} markers") + path.write_text(f"{head}{START}\n\n{block}\n{END}{tail}", encoding="utf-8") + + +CO = " - uses: actions/checkout@v7\n with: {repository: %s, path: %s%s}\n" +FIXTURE = { + ".gitmodules": "".join( + f'[submodule "{r}"]\n\tpath = {r}\n' + for r in ("marola-app", "marola-site", "marola-corpus", "marola-ml") + ), + "mkdocs/mkdocs.yml": "site_name: x\nsite_url: https://docs.marola.dev/\n", + "scripts/fetch-api-docs.sh": 'API_DOCS_BRANCH="api-docs"\ngit -C "$tmp" fetch -q "$url" "$API_DOCS_BRANCH"\n', + ".github/workflows/pointer-sync.yml": "on:\n repository_dispatch:\n types: [submodule-updated, submodule-docs-updated]\njobs: {}\n", + ".github/workflows/docs.yml": "on: {push: {paths: [flake.lock]}}\njobs:\n d:\n steps:\n - uses: actions/deploy-pages@v5\n", + ".devkit/.github/workflows/notify-umbrella.yml": "on:\n workflow_call:\n inputs:\n event-type: {type: string, default: submodule-docs-updated}\njobs:\n dispatch:\n steps:\n - env: {EVENT_TYPE: '${{ inputs.event-type }}'}\n run: curl https://api.github.com/repos/$UMBRELLA/dispatches -d x\n", + ".devkit/.github/workflows/api-docs.yml": "on: {workflow_call: {inputs: {devkit-ref: {type: string}}}}\njobs:\n publish:\n steps:\n - run: bash .devkit-checkout/scripts/api-docs-push.sh out url sha\n", + "marola-site/.github/workflows/notify-umbrella.yml": "on: {push: {branches: [main], paths: [README.md, 'docs/**']}}\njobs:\n n:\n uses: marola-dev/marola-devkit/.github/workflows/notify-umbrella.yml@v0.3.1\n", + "marola-site/.github/workflows/site.yml": 'on:\n push: {paths: [marola-image]}\n repository_dispatch: {types: [site-data-updated]}\njobs:\n b:\n steps:\n - run: git fetch --depth=1 origin site-data\n - run: echo "marola.dev" > site/dist/CNAME\n - uses: actions/deploy-pages@v5\n', + "marola-site/marola-image": "ghcr.io/marola-dev/marola-app:jvm-8a29976@sha256:00\n", + "marola-site/scripts/board-schema.sh": '# marola-image is read here\nref="$(cat "$root/marola-image")"\n', + "marola-corpus/.github/workflows/release.yml": 'on: {push: {tags: [\'v*\']}}\njobs:\n t:\n steps:\n - run: |\n file=".tmp/marola-corpus-$TAG.tar.gz"\n gh release upload "$TAG" "$file"\n', + "marola-app/.github/workflows/docker.yml": "on: {push: {branches: [main], paths: [corpus.version]}}\nenv: {IMAGE: 'ghcr.io/${{ github.repository_owner }}/marola-app'}\njobs:\n jvm:\n steps:\n - uses: docker/build-push-action@v7\n", + "marola-app/.github/workflows/api-docs.yml": "on: {push: {branches: [main]}}\njobs:\n a:\n uses: marola-dev/marola-devkit/.github/workflows/api-docs.yml@v0.4.1\n with: {devkit-ref: v0.4.1}\n", + "marola-app/.github/workflows/notify-umbrella.yml": "on: {push: {branches: [main]}}\njobs:\n n:\n uses: marola-dev/marola-devkit/.github/workflows/notify-umbrella.yml@v0.6.0\n with: {event-type: submodule-updated}\n", + "marola-app/.github/workflows/ci.yml": "on: {push: {branches: [main]}}\njobs:\n c:\n steps:\n" + + CO % ("marola-dev/marola-site", "site-data", ", ref: site-data") + + " - run: |\n git -C site-data commit -m x\n scripts/site-data-push.sh site-data\n - run: gh api repos/marola-dev/marola-site/dispatches -f event_type=site-data-updated\n", + "marola-app/.github/workflows/ingest.yml": "on: {schedule: [{cron: '0 4 * * *'}]}\njobs:\n i:\n steps:\n" + + CO % ("marola-dev/marola-oods", "oods", "") + + " - run: |\n git -C oods commit -qm ingest\n git -C oods push\n", + "marola-app/docker-compose.yml": "services:\n ollama-local:\n image: ghcr.io/marola-dev/marola-ml:local\n", + "marola-app/flake.lock": json.dumps( + {"nodes": {"marola-devkit": {"original": {"repo": "marola-devkit", "ref": "v0.4.1"}}}} + ), + "marola-ml/.github/workflows/docker-local.yml": "on: {push: {branches: [main]}}\nenv: {IMAGE: ghcr.io/marola-dev/marola-ml}\njobs:\n b:\n steps:\n - uses: docker/build-push-action@v7\n", + "marola-ml/.github/workflows/compile-prompt.yml": "on: {workflow_dispatch: {}}\nenv: {APP_REPO: marola-dev/marola-app}\njobs:\n c:\n steps:\n" + + CO % ("'${{ env.APP_REPO }}'", "app", "") + + ' - run: |\n cd app\n git add core/src/main/resources/recommendation_prompt.json core/src/main/resources/review_prompt.json\n git push -q origin "$BRANCH"\n gh pr create --repo "$APP_REPO" --base main\n', + "marola-ml/corpus.version": "v0.2.0\n", + "marola-ml/scripts/corpus-fetch.sh": 'pin="$(<"$root/corpus.version")"\nbase="https://github.com/marola-dev/marola-corpus/releases/download"\n', +} + + +# Each case is rows the fixture's block must hold; a "!" line is text it must not hold. +CASES = { + "dispatch_types_listened": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |\n| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |", + "dispatch_send_step": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |", + "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", + "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |", + "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | marola-site `marola-image` | marola-site `scripts/board-schema.sh` |", + "deploy_pages_site": "| marola `docs.yml` | docs.marola.dev |\n| marola-site `site.yml` | marola.dev |", + "pin_file_and_reader": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |\n| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", + "branch_artifact_api_docs": "| `api-docs` branch | marola-app `api-docs.yml` (`api-docs.yml@v0.4.1`, on a push to `main`) | — | marola `scripts/fetch-api-docs.sh` |", + "branch_artifact_site_data": "| `site-data` branch of marola-site | marola-app `ci.yml` (on a push to `main`) | — | marola-site `site.yml` |\n!pushes to marola-site", + "compose_image_reader": "| `ghcr.io/marola-dev/marola-ml` image | marola-ml `docker-local.yml` (on a push to `main`) | — | marola-app `docker-compose.yml` |", + "cross_repo_pr": "| PRs into marola-app: `recommendation_prompt.json`, `review_prompt.json` | marola-ml `compile-prompt.yml` (by hand) | — | marola-app |\n!pushes to marola-app", + "cross_repo_push": "| pushes to marola-oods | marola-app `ingest.yml` (on a schedule) | — | marola-oods |", +} + + +def self_test() -> int: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for rel, text in FIXTURE.items(): + (root / rel).parent.mkdir(parents=True, exist_ok=True) + (root / rel).write_text(text, encoding="utf-8") + w = scan(root) + block = render(w) + (root / "REPOS.md").write_text( + f"# Repos\n\n{START}\nstale\n{END}\n\nAfter.\n", encoding="utf-8" + ) + write_block(root / "REPOS.md", block) + repos_md = (root / "REPOS.md").read_text(encoding="utf-8") + rows = block.splitlines() + cases = [ + (name, all(x[1:] not in block if x[0] == "!" else x in rows for x in want.split("\n"))) + for name, want in CASES.items() + ] + cases.append(("block_between_markers", f"{START}\n\n{block}\n{END}\n\nAfter." in repos_md)) + fails = [n for n, ok in cases if not ok] + for name, ok in cases: + print(f" {'ok ' if ok else 'FAIL'} {name}") + if fails: + print(block, file=sys.stderr) + print(f"wiring self-test: {len(fails)} failure(s)", file=sys.stderr) + return 1 + print("wiring self-test: ok") + return 0 + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "file", nargs="?", type=Path, help="rewrite the block between this file's markers" + ) + ap.add_argument("--root", type=Path, default=Path.cwd()) + ap.add_argument("--self-test", action="store_true") + args = ap.parse_args(argv) + if args.self_test: + return self_test() + block = render(scan(args.root)) + if args.file: + write_block(args.file, block) + else: + sys.stdout.write(block) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/self-tests.sh b/tests/self-tests.sh index 9154b9a..1039457 100755 --- a/tests/self-tests.sh +++ b/tests/self-tests.sh @@ -13,7 +13,7 @@ sh_tests=( plugins/marola-devkit/hooks/session-start.sh ) py_tests=( - scripts/cost-split.py scripts/gemini_review.py scripts/pr_label_nlp.py scripts/workflow_runners.py scripts/docs_lint.py + scripts/cost-split.py scripts/gemini_review.py scripts/pr_label_nlp.py scripts/workflow_runners.py scripts/docs_lint.py scripts/wiring.py scripts/lib/req_merge.py scripts/lib/uses_merge.py scripts/lib/mip_index_merge.py scripts/lib/tasks_issues.py plugins/marola-devkit/skills/voice-note-ingest/scripts/transcribe.py ) From f87f449254d816605bd7484011e9c97f06aaba64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Mon, 5 Oct 2026 09:03:46 +0200 Subject: [PATCH 03/13] feat(wiring): on PATH as `wiring`, a `just wiring` recipe, its tools row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PyYAML joins the flake's python, which every tool wrapper and the self-tests check run with. Refs #25 Tested: just quality line by line without nix (ruff check/format, shellcheck, actionlint, tests/self-tests.sh, agents-check, docs_lint); nix flake check not run, no nix here Cost: ~$2.40 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- devkit.just | 4 ++++ docs/4-reference_tools.md | 1 + flake.nix | 5 +++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/devkit.just b/devkit.just index 17e34ff..7c17997 100644 --- a/devkit.just +++ b/devkit.just @@ -115,6 +115,10 @@ branches-clean *args: branches-open *args: branches open {{ args }} +# The cross-repo wiring tables (MIP-0076), printed or rewritten between a file's wiring markers. +wiring *args: + wiring {{ args }} + # The invariants block in AGENTS.md against the pinned devkit's copy. agents-check *args: agents-check {{ args }} diff --git a/docs/4-reference_tools.md b/docs/4-reference_tools.md index 5d6ad2a..44325e1 100644 --- a/docs/4-reference_tools.md +++ b/docs/4-reference_tools.md @@ -57,6 +57,7 @@ extension rather than a devkit tool. | `ruleset-sync` | `ruleset-sync.sh` | `just rulesets-check`, `just rulesets-apply` | `check [owner/repo…]` diffs a repo's live branch ruleset against `.github/rulesets/main-rule.json`; `apply ` creates or updates it; `--all-org ORG` covers an org | | `agents-check` | `agents-check.sh` | `just agents-check` | Compare AGENTS.md's invariants block byte for byte with the pinned devkit's `agents/invariants.md` (`--block` or `MAROLA_INVARIANTS_BLOCK` to use another) | | `docs-lint` | `docs_lint.py` | (none) | MIP-0074 §7's stale-content check over `README.md` and `docs/**/*.md`: undefined or unmarked foreign recipes, another repo's paths, split-era wording, `docs/index.md`, links leaving the repo | +| `wiring` | `wiring.py` | `just wiring` | MIP-0076 §5.1's four tables (artifact, dispatch, pin bump, deploy), parsed from the umbrella's and every submodule's workflows, pin files, scripts, justfiles, Dockerfiles, compose files and `build.sbt`, with each devkit workflow call's ref. `wiring [FILE]` prints the block or rewrites it between FILE's `wiring:start`/`wiring:end` markers; run it in an umbrella checkout with submodules | | `workflow-runners` | `workflow_runners.py` | (none) | Fail when any workflow but the GPU publish one can reach a self-hosted runner ([runners](4-reference_runners.md)) | | `mip-resolve` | `mip-resolve.sh` | (none) | Print a MIP's doc or `.tasks.md` (`MIP-NNNN [tasks]`), or with `--path` where it was found, by the lookup in [design](1-design.md#how-tools-find-the-umbrella) | | `api-docs-push` | `api-docs-push.sh` | (none) | `api-docs-push [--branch api-docs]`: force-push a directory as one orphan commit, the `api-docs` workflow's push step | diff --git a/flake.nix b/flake.nix index b52168f..3bdba2f 100644 --- a/flake.nix +++ b/flake.nix @@ -21,8 +21,8 @@ pkgs = import nixpkgs { inherit system; }; lib = pkgs.lib; - # scripts/pr_label_nlp.py imports sklearn, so it has to be inside this python3. - python = pkgs.python3.withPackages (ps: [ ps.scikit-learn ]); + # scripts/pr_label_nlp.py imports sklearn and scripts/wiring.py yaml, so both live in this python3. + python = pkgs.python3.withPackages (ps: [ ps.scikit-learn ps.pyyaml ]); runtimeDeps = [ pkgs.bash pkgs.coreutils pkgs.findutils pkgs.gnugrep pkgs.gnused pkgs.gawk pkgs.perl pkgs.git pkgs.gh pkgs.jq pkgs.curl python @@ -55,6 +55,7 @@ temps = "scripts/temps.sh"; workflow-runners = "scripts/workflow_runners.py"; docs-lint = "scripts/docs_lint.py"; + wiring = "scripts/wiring.py"; }; # The whole tree is installed with its layout intact under share/marola-devkit: the scripts From d5b3d89b306c0e160de84b58b675d748ffa0df5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Mon, 5 Oct 2026 09:05:19 +0200 Subject: [PATCH 04/13] fix(notify-umbrella): decide the docs event via the compare API, not a checkout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reusable workflow's effective permissions can only keep or lower what the caller granted, never raise it, and all five current callers set permissions: {} with no override on the notify: job — so the checkout added in the previous commit would have failed every one of them once rows 6-10 bump to this tag. Drop the checkout and git diff; call GET /repos/$REPO/compare/$BEFORE...$SHA with github.token instead (a no-permission token can still read a public repo over the API) and read .files[].filename from the response with jq. changed=true on a README.md or docs/ filename, same as before, plus now also on a non-200 response (before unknown to the API, e.g. a force-push) and on a truncated compare (the API caps its file list at 300). Restores permissions: {}, timeout-minutes: 2 and the header's "no checkout needed on either side". Closes #27 Tested: actionlint ok (repo-wide); shellcheck --severity=error ok; ruff check/format ok (uvx ruff 0.16.9, no nix); bash tests/self-tests.sh all ok; scripts/agents-check.sh ok; python3 scripts/docs_lint.py clean; claude plugin validate . passed; jq filter checked by hand against four fixture compare responses (match on docs/, match on README.md, no match, empty files) Cost: ~$3.40 est. · ~1.7M tokens est. (diff-size model, 48 lines, no session log, IQR 0.4M-8.9M tokens for this diff from 75 calibrated commits) · scripts/cost-split.py --estimate 2026-10-05 Co-Authored-By: Claude --- .github/workflows/notify-umbrella.yml | 36 ++++++++++++++++----------- docs/4-reference_workflows.md | 12 +++++---- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/.github/workflows/notify-umbrella.yml b/.github/workflows/notify-umbrella.yml index 205c7f0..6458b14 100644 --- a/.github/workflows/notify-umbrella.yml +++ b/.github/workflows/notify-umbrella.yml @@ -2,7 +2,8 @@ # pointer-sync.yml can move pointers on event instead of its daily cron, and that a repo's docs # changed too (`submodule-docs-updated`) when the push touches README.md or docs/**, so # docs.yml rebuilds within minutes (MIP-0070 §5.5, MIP-0076 §5.5 step 1). Modelled on -# h0ffmann/nix-config's profile-ping.yml reusable workflow. +# h0ffmann/nix-config's profile-ping.yml reusable workflow — a plain repository_dispatch, no +# checkout needed on either side: the docs check reads the compare API instead of git. # # A consumer adds about three lines, no `paths:` filter — this workflow decides for itself which # event types a given push earns: @@ -41,33 +42,40 @@ on: description: "Fine-grained PAT with Contents: read & write on the umbrella." required: false -permissions: - contents: read +permissions: {} jobs: dispatch: runs-on: ${{ inputs.runner }} - timeout-minutes: 5 + timeout-minutes: 2 steps: - # Full history: github.event.before can be arbitrarily far back, and the diff below needs it. - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - + # A reusable workflow can only keep or lower the caller's permissions, and every caller sets + # permissions: {} — so this reads the compare API instead of checking the repo out. Even a + # token with no declared scope can read a public repo's API. - name: Did this push touch README.md or docs/**? id: docs env: + TOKEN: ${{ github.token }} BEFORE: ${{ github.event.before }} SHA: ${{ github.sha }} + REPO: ${{ github.repository }} run: | set -euo pipefail changed=false if [ -z "${BEFORE:-}" ] || [ "$BEFORE" = "0000000000000000000000000000000000000000" ]; then - changed=true # first push on the branch: no prior ref to diff against - elif ! git cat-file -e "$BEFORE^{commit}" 2>/dev/null; then - changed=true # before rewritten out of history (e.g. a force-push) - elif [ -n "$(git diff --name-only "$BEFORE" "$SHA" -- README.md docs)" ]; then - changed=true + changed=true # first push on the branch: no prior ref to compare against + else + resp="$RUNNER_TEMP/notify-umbrella-compare.json" + code="$(curl -sS -o "$resp" -w '%{http_code}' \ + -H "Authorization: Bearer $TOKEN" -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/$REPO/compare/$BEFORE...$SHA")" + if [ "$code" != 200 ]; then + changed=true # before unknown to the API (e.g. a force-push): take the conservative path + elif [ "$(jq '(.files // []) | length' "$resp")" -ge 300 ]; then + changed=true # the compare API caps its file list at 300: the list may be truncated + elif jq -e '(.files // []) | any(.[]; .filename == "README.md" or (.filename | startswith("docs/")))' "$resp" >/dev/null; then + changed=true + fi fi echo "changed=$changed" >>"$GITHUB_OUTPUT" diff --git a/docs/4-reference_workflows.md b/docs/4-reference_workflows.md index 32054da..37a6a40 100644 --- a/docs/4-reference_workflows.md +++ b/docs/4-reference_workflows.md @@ -118,9 +118,10 @@ Tells the umbrella every push to `main` via `repository_dispatch` (`submodule-up pointer-sync.yml can move pointers on the event instead of waiting for its daily cron, and that a repo's docs changed too (`submodule-docs-updated`) when the push touched `README.md` or `docs/**`, so docs.yml rebuilds within minutes (MIP-0070 §5.5, MIP-0076 §5.5 step 1). Modelled on -h0ffmann/nix-config's `profile-ping.yml`. The caller needs no `paths:` filter: this workflow -checks out the caller's own repo and diffs `github.event.before..github.sha` itself to decide -which event types a given push earns. +h0ffmann/nix-config's `profile-ping.yml` — no checkout on either side. The caller needs no +`paths:` filter: this workflow reads the compare API itself to decide which event types a given +push earns (every caller sets `permissions: {}`, which a reusable workflow can only keep or +lower, so it can't check its own repo out). ```yaml name: notify umbrella @@ -145,8 +146,9 @@ fine-grained PAT with Contents: read & write on the umbrella (`repository_dispat `GITHUB_TOKEN` cannot reach another repo). Unset is a notice, not a failure — the umbrella's daily cron still catches the change. -A zero or unreachable `github.event.before` (a branch's first push, or history rewritten out from -under it) counts as a docs change, so both events go rather than silently dropping one. +A zero `github.event.before` (a branch's first push), a compare API call that doesn't answer 200 +(e.g. `before` unknown to it after a force-push), or a truncated compare (the API caps its file +list at 300) all count as a docs change too, so both events go rather than silently dropping one. ## labels-sync From 60463c7224e3d52ce0edd6f7fab34b3dc083d29e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Mon, 5 Oct 2026 09:15:48 +0200 Subject: [PATCH 05/13] fix(wiring): read each reusable-workflow call at its own ref; fail on missing submodules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 1 on #25. A call is now resolved with `git show :.github/workflows/` in the devkit checkout (`--devkit DIR`, default `.devkit`, else this repo), so a caller still on an old tag shows that tag's dispatch type. An unknown ref falls back to the tree with a warning and "(resolved at )" in the row; a callee missing at its ref is a warning, not a silent drop. An uninitialised submodule is an error instead of a near-empty block. The umbrella is named by `--name` (default `marola`), not the enclosing git config. Smaller: `event_type=` is matched after substitution; `-push.sh` on a `--self-test` line is ignored; a branch reader needs a `git … fetch` line; the self-exclusion is only the devkit's `scripts/wiring.py`; more than three push paths render as "touching N paths"; `git add` flags are not PR files; malformed YAML names its file; the README lists `wiring`. Closes #25 Tested: wiring --self-test (16 cases; the new ref and submodule cases fail with their fix stubbed out); ruff check/format, actionlint, shellcheck, tests/self-tests.sh, agents-check, docs_lint; a run over the fresh umbrella clone with --devkit at a tagged devkit checkout Cost: ~$8.59 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- README.md | 1 + scripts/wiring.py | 166 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 129 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 865852a..4247f59 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ claude --plugin-dir plugins/marola-devkit # the plugin, from this checkout | `issues` | Labels sync, Definition of Ready, the `agent-ready` queue, claims, `tasks-to-issues`, the board | | `agents-check` | Compare AGENTS.md's invariants block with the pinned devkit's `agents/invariants.md` | | `docs-lint` | MIP-0074's stale-content check over `README.md` and `docs/**/*.md` | +| `wiring` | MIP-0076's cross-repo wiring tables (artifacts, dispatches, pin bumps, deploys), parsed from an umbrella checkout and its submodules | | `ruleset-sync` | Check or apply `.github/rulesets/main-rule.json`'s branch ruleset to a repo, or every repo of an org | | `api-docs-push` | Force-push a directory as one orphan commit to a branch; the `api-docs` workflow's push step | | `mip-resolve` | Find a MIP or `.tasks.md` in the repo, the umbrella checkout, or via `gh api` | diff --git a/scripts/wiring.py b/scripts/wiring.py index 05c58f6..e113dfe 100755 --- a/scripts/wiring.py +++ b/scripts/wiring.py @@ -3,13 +3,17 @@ Run in an umbrella checkout with its submodules; the devkit's tree is `.devkit`, else this script's. -wiring [--root DIR] [FILE] print the block, or rewrite it between FILE's wiring markers +wiring [--root DIR] [--devkit DIR] [--name NAME] [FILE] + print the block, or rewrite it between FILE's wiring markers; a devkit git checkout with tags + lets each reusable-workflow call be read at its own @ref wiring --self-test """ from __future__ import annotations import argparse +import contextlib +import io import json import re import subprocess @@ -20,7 +24,6 @@ import yaml ORG = "marola-dev" -SELF = Path(__file__) # its fixture names every idiom, so it is never a reader PINS = ("marola-image", "corpus.version", "resources.version") START, END = "", "" READERS = ("scripts/**/*", "justfile", "Dockerfile*", "docker-compose*.yml", "build.sbt") @@ -75,21 +78,41 @@ def read(p: Path) -> str: return "" -def repos(root: Path) -> list[tuple[str, Path]]: - url = subprocess.run( - ["git", "-C", str(root), "config", "--get", "remote.origin.url"], - capture_output=True, - text=True, - check=False, - ).stdout.strip() - out = [(re.sub(r"\.git$", "", url.rsplit("/", 1)[-1]) or "marola", root)] +def warn(msg: str) -> None: + print(f"wiring: warning: {msg}", file=sys.stderr) + + +def git(d: Path, *args: str) -> str | None: + if not (d / ".git").exists(): # never the enclosing repo's answer + return None + r = subprocess.run(["git", "-C", str(d), *args], capture_output=True, text=True, check=False) + return r.stdout if r.returncode == 0 else None + + +def load(text: str, where: object) -> dict: + try: + return yaml.safe_load(text) or {} + except yaml.YAMLError as e: + raise SystemExit(f"wiring: {where}: {e}") from None + + +def version(d: Path) -> str: + if v := git(d, "describe", "--tags", "--always"): + return v.strip() + m = re.search(r"marola-devkit-([\d.]+)", str(d.resolve())) + return f"v{m[1]}" if m else "its working tree" + + +def repos(root: Path, name: str, devkit: Path | None) -> list[tuple[str, Path]]: + out = [(name, root)] for path in re.findall(r"^\s*path\s*=\s*(\S+)", read(root / ".gitmodules"), re.M): + if not (root / path / ".git").exists(): + raise SystemExit(f"wiring: {path} is not checked out (git submodule update --init)") out.append((Path(path).name, root / path)) - devkit = root / ".devkit" - out.append( - ("marola-devkit", devkit if devkit.is_dir() else Path(__file__).resolve().parents[1]) - ) - return [(n, p) for n, p in out if p.is_dir()] + if devkit is None: + devkit = root / ".devkit" + devkit = devkit if devkit.is_dir() else Path(__file__).resolve().parents[1] + return [(n, p) for n, p in out if p.is_dir()] + [("marola-devkit", devkit)] def triggers(doc: dict) -> dict: @@ -105,7 +128,8 @@ def when(on: dict) -> str: return f"on a `{push['tags'][0]}` tag" if "push" in on: paths = push.get("paths") or [] - touching = " touching " + ", ".join(f"`{p}`" for p in paths) if 0 < len(paths) <= 3 else "" + touching = " touching " + ", ".join(f"`{p}`" for p in paths) if paths else "" + touching = f" touching {len(paths)} paths" if len(paths) > 3 else touching return f"on a push to `{(push.get('branches') or ['main'])[0]}`{touching}" if "schedule" in on: return "on a schedule" @@ -121,20 +145,20 @@ def site_of(repo_dir: Path, text: str) -> str: return "?" -def effects(doc: dict, repo: str, inputs: dict, workflows: dict) -> list[tuple]: +def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: """(kind, value, via) for what a workflow sends, publishes, deploys or writes elsewhere.""" out: list[tuple] = [] text = yaml.safe_dump(doc) for job in (doc.get("jobs") or {}).values(): env = {**(doc.get("env") or {}), **(job.get("env") or {})} if m := USES.match(str(job.get("uses", ""))): - target = workflows.get((m[1], m[2])) + target, note = callee(m[1], m[2], m[3]) if target: defaults = triggers(target).get("workflow_call", {}).get("inputs") or {} given = {k: v.get("default", "") for k, v in defaults.items()} given |= {k: sub(v, env, inputs) for k, v in (job.get("with") or {}).items()} - via = f"`{m[2]}@{m[3]}`" - out += [(k, v, via) for k, v, _ in effects(target, repo, given, workflows)] + via = f"`{m[2]}@{m[3]}`{note}" + out += [(k, v, via) for k, v, _ in effects(target, repo, given, callee)] out.append(("ref", (m[2], m[3]), "")) continue checkouts: dict[str, tuple[str, str]] = {} @@ -154,20 +178,26 @@ def effects(doc: dict, repo: str, inputs: dict, workflows: dict) -> list[tuple]: if not run: continue assigned = dict(ASSIGN.findall(run)) - out += [("send", t, "") for t in re.findall(r"event_type=([\w-]+)", run)] + sent = re.findall(r"event_type=([^\s\"']*)", sub(run, senv, inputs)) + out += [("send", t, "") for t in sent if t and "$" not in t] if "/dispatches" in run: out += [("send", sub(v, {}, inputs), "") for k, v in senv.items() if "EVENT" in k] for asset in RELEASE.findall(run): name = sub(asset.strip("\"'"), assigned, {}).rsplit("/", 1)[-1] out.append(("asset", SHVAR.sub("", name), "")) - if branches := re.findall(r"([\w-]+)-push\.sh\b", run): + pushing = [ln for ln in run.splitlines() if "--self-test" not in ln] + if branches := re.findall(r"([\w-]+)-push\.sh\b", "\n".join(pushing)): for b in branches: owner = next((r for r, ref in checkouts.values() if ref == b), repo) out.append(("branch", (b, owner), "")) continue if m := re.search(r"gh pr create\b.*?--repo\s+(\S+)", run, re.S): files = sorted( - {Path(f).name for f in " ".join(re.findall(r"git add\s+(.+)", run)).split()} + { + Path(f).name + for f in " ".join(re.findall(r"git add\s+(.+)", run)).split() + if not f.startswith("-") and f != "." + } ) out.append( ("pr", (sub(m[1].strip("\"'"), senv, inputs).rsplit("/", 1)[-1], files), "") @@ -179,12 +209,31 @@ def effects(doc: dict, repo: str, inputs: dict, workflows: dict) -> list[tuple]: return [(k, v, via) for k, v, via in out if k not in ("pr", "push") or v[0] != repo] -def scan(root: Path) -> Wiring: - w, rs = Wiring(), repos(root) +def scan(root: Path, name: str = "marola", devkit: Path | None = None) -> Wiring: + w, rs = Wiring(), repos(root, name, devkit) + dirs = dict(rs) docs: dict[tuple[str, str], dict] = {} - for name, d in rs: + for repo, d in rs: for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): - docs[(name, f.name)] = yaml.safe_load(read(f)) or {} + docs[(repo, f.name)] = load(read(f), f) + cache: dict[tuple[str, str, str], tuple[dict | None, str]] = {} + + def callee(repo: str, file: str, ref: str) -> tuple[dict | None, str]: + key, d = (repo, file, ref), dirs.get(repo) + if key in cache or d is None: + return cache.get(key, (None, "")) + if git(d, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}"): + text = git(d, "show", f"{ref}:.github/workflows/{file}") + cache[key] = (load(text, f"{repo} {ref}:{file}"), "") if text else (None, "") + elif (repo, file) in docs: + warn(f"{repo} has no ref {ref} here; reading {file} at {version(d)}") + cache[key] = (docs[(repo, file)], f" (resolved at {version(d)})") + else: + cache[key] = (None, "") + if cache[key][0] is None: + warn(f"{repo}/{file}@{ref} not found; its effects are not shown") + return cache[key] + for (repo, file), doc in docs.items(): on = triggers(doc) if set(on) == {"workflow_call"}: @@ -195,7 +244,7 @@ def scan(root: Path) -> Wiring: for pin in set(PINS + ("flake.lock",)) & set(on.get("push", {}).get("paths") or []): add(w.bumps.setdefault(f"{repo} `{pin}`", []), label) repo_dir = dict(rs)[repo] - for kind, v, via in effects(doc, repo, {}, docs): + for kind, v, via in effects(doc, repo, {}, callee): parts = [p for p in (via, when(on)) if p] pub = label + (f" ({', '.join(parts)})" if parts else "") if kind == "send": @@ -210,19 +259,21 @@ def scan(root: Path) -> Wiring: add(w.art(key, owner, b)["pub"], pub) elif kind == "pr": a = w.art(f"PRs into {v[0]}: " + ", ".join(f"`{f}`" for f in v[1]), v[0]) - add(a["pub"], pub), add(a["read"], v[0]) + add(a["pub"], pub) + add(a["read"], v[0]) elif kind == "push": a = w.art(f"pushes to {v}", v) - add(a["pub"], pub), add(a["read"], v) + add(a["pub"], pub) + add(a["read"], v) elif kind == "deploy": w.deploys.append((label, site_of(repo_dir, v))) elif kind == "ref": add(w.calls.setdefault(v[0], {}).setdefault(v[1], []), repo) for repo, d in rs: - files = sorted( - {f for g in READERS for f in d.glob(g) if f.is_file() and f.name != SELF.name} - ) + files = sorted({f for g in READERS for f in d.glob(g) if f.is_file()}) texts = {f.relative_to(d).as_posix(): lines(read(f)) for f in files} + if repo == "marola-devkit": + texts.pop("scripts/wiring.py", None) # its fixture names every idiom for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): texts[f".github/workflows/{f.name}"] = lines(read(f)) for rel, text in texts.items(): @@ -235,7 +286,7 @@ def scan(root: Path) -> Wiring: continue names = {b} | {v for v, x in ASSIGN.findall(text) if x == b} if any( - "fetch" in ln + re.search(r"\bgit\b[^\n]*\bfetch\b", ln) and any(re.search(rf"(? None: "marola-site/.github/workflows/site.yml": 'on:\n push: {paths: [marola-image]}\n repository_dispatch: {types: [site-data-updated]}\njobs:\n b:\n steps:\n - run: git fetch --depth=1 origin site-data\n - run: echo "marola.dev" > site/dist/CNAME\n - uses: actions/deploy-pages@v5\n', "marola-site/marola-image": "ghcr.io/marola-dev/marola-app:jvm-8a29976@sha256:00\n", "marola-site/scripts/board-schema.sh": '# marola-image is read here\nref="$(cat "$root/marola-image")"\n', + "marola-corpus/.github/workflows/api-docs.yml": "on: {push: {branches: [main]}}\njobs:\n a:\n uses: marola-dev/marola-devkit/.github/workflows/api-docs.yml@v9.9.9\n", + "marola-site/.github/workflows/gemini.yml": "on: {pull_request: {}}\njobs:\n g:\n uses: marola-dev/marola-devkit/.github/workflows/gemini-review.yml@v0.5.0\n", "marola-corpus/.github/workflows/release.yml": 'on: {push: {tags: [\'v*\']}}\njobs:\n t:\n steps:\n - run: |\n file=".tmp/marola-corpus-$TAG.tar.gz"\n gh release upload "$TAG" "$file"\n', "marola-app/.github/workflows/docker.yml": "on: {push: {branches: [main], paths: [corpus.version]}}\nenv: {IMAGE: 'ghcr.io/${{ github.repository_owner }}/marola-app'}\njobs:\n jvm:\n steps:\n - uses: docker/build-push-action@v7\n", "marola-app/.github/workflows/api-docs.yml": "on: {push: {branches: [main]}}\njobs:\n a:\n uses: marola-dev/marola-devkit/.github/workflows/api-docs.yml@v0.4.1\n with: {devkit-ref: v0.4.1}\n", @@ -350,17 +403,20 @@ def write_block(path: Path, block: str) -> None: "marola-ml/scripts/corpus-fetch.sh": 'pin="$(<"$root/corpus.version")"\nbase="https://github.com/marola-dev/marola-corpus/releases/download"\n', } +API_ROW = "| `api-docs` branch | marola-app `api-docs.yml` (`api-docs.yml@v0.4.1`, on a push to `main`), marola-corpus `api-docs.yml` (`api-docs.yml@v9.9.9` (resolved at v0.6.0), on a push to `main`) | — | marola `scripts/fetch-api-docs.sh` |" # Each case is rows the fixture's block must hold; a "!" line is text it must not hold. CASES = { "dispatch_types_listened": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |\n| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |", "dispatch_send_step": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |", - "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", + "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app), v9.9.9 (marola-corpus); `gemini-review.yml` v0.5.0 (marola-site); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", + "reusable_workflow_resolved_at_caller_ref": "| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |\n| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n" + + API_ROW, "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |", "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | marola-site `marola-image` | marola-site `scripts/board-schema.sh` |", "deploy_pages_site": "| marola `docs.yml` | docs.marola.dev |\n| marola-site `site.yml` | marola.dev |", "pin_file_and_reader": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |\n| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", - "branch_artifact_api_docs": "| `api-docs` branch | marola-app `api-docs.yml` (`api-docs.yml@v0.4.1`, on a push to `main`) | — | marola `scripts/fetch-api-docs.sh` |", + "branch_artifact_api_docs": API_ROW, "branch_artifact_site_data": "| `site-data` branch of marola-site | marola-app `ci.yml` (on a push to `main`) | — | marola-site `site.yml` |\n!pushes to marola-site", "compose_image_reader": "| `ghcr.io/marola-dev/marola-ml` image | marola-ml `docker-local.yml` (on a push to `main`) | — | marola-app `docker-compose.yml` |", "cross_repo_pr": "| PRs into marola-app: `recommendation_prompt.json`, `review_prompt.json` | marola-ml `compile-prompt.yml` (by hand) | — | marola-app |\n!pushes to marola-app", @@ -368,14 +424,44 @@ def write_block(path: Path, block: str) -> None: } +def _devkit_tags(dk: Path) -> None: + def g(*args: str) -> None: + cfg = ["-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false"] + subprocess.run(["git", "-C", str(dk), *cfg, *args], check=True, capture_output=True) + + g("init", "-q") + g("add", "-A") + g("commit", "-qm", "v0.3.1", "--no-verify") + g("tag", "v0.3.1") + g("tag", "v0.4.1") + nu = dk / ".github/workflows/notify-umbrella.yml" + nu.write_text( + nu.read_text().replace("default: submodule-docs-updated", "default: submodule-updated") + ) + g("commit", "-qam", "v0.6.0", "--no-verify") + g("tag", "v0.6.0") + + def self_test() -> int: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) for rel, text in FIXTURE.items(): (root / rel).parent.mkdir(parents=True, exist_ok=True) (root / rel).write_text(text, encoding="utf-8") - w = scan(root) + for path in re.findall(r"path = (\S+)", FIXTURE[".gitmodules"]): + (root / path / ".git").write_text("gitdir: unused\n", encoding="utf-8") + _devkit_tags(root / ".devkit") + err = io.StringIO() + with contextlib.redirect_stderr(err): + w = scan(root) block = render(w) + (root / "marola-x").mkdir() + (root / ".gitmodules").write_text("[submodule]\n\tpath = marola-x\n", encoding="utf-8") + try: + scan(root) + uninit = "" + except SystemExit as e: + uninit = str(e) (root / "REPOS.md").write_text( f"# Repos\n\n{START}\nstale\n{END}\n\nAfter.\n", encoding="utf-8" ) @@ -386,6 +472,8 @@ def self_test() -> int: (name, all(x[1:] not in block if x[0] == "!" else x in rows for x in want.split("\n"))) for name, want in CASES.items() ] + cases.append(("missing_callee_warns", "gemini-review.yml@v0.5.0 not found" in err.getvalue())) + cases.append(("uninitialised_submodule_fails", "marola-x is not checked out" in uninit)) cases.append(("block_between_markers", f"{START}\n\n{block}\n{END}\n\nAfter." in repos_md)) fails = [n for n, ok in cases if not ok] for name, ok in cases: @@ -406,11 +494,13 @@ def main(argv: list[str] | None = None) -> int: "file", nargs="?", type=Path, help="rewrite the block between this file's markers" ) ap.add_argument("--root", type=Path, default=Path.cwd()) + ap.add_argument("--devkit", type=Path, help="default: /.devkit, else this script's repo") + ap.add_argument("--name", default="marola", help="the umbrella's repo name") ap.add_argument("--self-test", action="store_true") args = ap.parse_args(argv) if args.self_test: return self_test() - block = render(scan(args.root)) + block = render(scan(args.root, args.name, args.devkit)) if args.file: write_block(args.file, block) else: From 34bd04dcf9cf31be721c78c1deb0558dd85de7fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Fri, 9 Oct 2026 23:20:54 +0200 Subject: [PATCH 06/13] refactor(wiring): this PR is the workflow side only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit marola-devkit#25 is split in two (its row's own ~400-line clause): this PR keeps dispatches, reusable-workflow calls, release assets, images, pin bumps, deploys, the devkit-pin row, rendering and the markers, with callees read from the devkit's tree. Pin readers, branch artifacts, cross-repo PRs and pushes, and reading each callee at its own ref move to the stacked mip-0076/1-wiring-readers, which restores them unchanged. Tested: wiring --self-test (10 cases, pin_bump_on_push_paths new); ruff check/format; docs_lint; a run over a fresh --recurse-submodules umbrella clone Cost: ~$9.33 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- docs/4-reference_tools.md | 2 +- scripts/wiring.py | 182 +++----------------------------------- 2 files changed, 15 insertions(+), 169 deletions(-) diff --git a/docs/4-reference_tools.md b/docs/4-reference_tools.md index 44325e1..df9597a 100644 --- a/docs/4-reference_tools.md +++ b/docs/4-reference_tools.md @@ -57,7 +57,7 @@ extension rather than a devkit tool. | `ruleset-sync` | `ruleset-sync.sh` | `just rulesets-check`, `just rulesets-apply` | `check [owner/repo…]` diffs a repo's live branch ruleset against `.github/rulesets/main-rule.json`; `apply ` creates or updates it; `--all-org ORG` covers an org | | `agents-check` | `agents-check.sh` | `just agents-check` | Compare AGENTS.md's invariants block byte for byte with the pinned devkit's `agents/invariants.md` (`--block` or `MAROLA_INVARIANTS_BLOCK` to use another) | | `docs-lint` | `docs_lint.py` | (none) | MIP-0074 §7's stale-content check over `README.md` and `docs/**/*.md`: undefined or unmarked foreign recipes, another repo's paths, split-era wording, `docs/index.md`, links leaving the repo | -| `wiring` | `wiring.py` | `just wiring` | MIP-0076 §5.1's four tables (artifact, dispatch, pin bump, deploy), parsed from the umbrella's and every submodule's workflows, pin files, scripts, justfiles, Dockerfiles, compose files and `build.sbt`, with each devkit workflow call's ref. `wiring [FILE]` prints the block or rewrites it between FILE's `wiring:start`/`wiring:end` markers; run it in an umbrella checkout with submodules | +| `wiring` | `wiring.py` | `just wiring` | MIP-0076 §5.1's four tables (artifact, dispatch, pin bump, deploy), parsed from the umbrella's and every submodule's workflows (pin readers, branch artifacts and cross-repo writes are not read yet), with each devkit workflow call's ref. `wiring [FILE]` prints the block or rewrites it between FILE's `wiring:start`/`wiring:end` markers; run it in an umbrella checkout with submodules | | `workflow-runners` | `workflow_runners.py` | (none) | Fail when any workflow but the GPU publish one can reach a self-hosted runner ([runners](4-reference_runners.md)) | | `mip-resolve` | `mip-resolve.sh` | (none) | Print a MIP's doc or `.tasks.md` (`MIP-NNNN [tasks]`), or with `--path` where it was found, by the lookup in [design](1-design.md#how-tools-find-the-umbrella) | | `api-docs-push` | `api-docs-push.sh` | (none) | `api-docs-push [--branch api-docs]`: force-push a directory as one orphan commit, the `api-docs` workflow's push step | diff --git a/scripts/wiring.py b/scripts/wiring.py index e113dfe..4e70236 100755 --- a/scripts/wiring.py +++ b/scripts/wiring.py @@ -4,8 +4,7 @@ Run in an umbrella checkout with its submodules; the devkit's tree is `.devkit`, else this script's. wiring [--root DIR] [--devkit DIR] [--name NAME] [FILE] - print the block, or rewrite it between FILE's wiring markers; a devkit git checkout with tags - lets each reusable-workflow call be read at its own @ref + print the block, or rewrite it between FILE's wiring markers wiring --self-test """ @@ -16,7 +15,6 @@ import io import json import re -import subprocess import sys import tempfile from pathlib import Path @@ -26,7 +24,6 @@ ORG = "marola-dev" PINS = ("marola-image", "corpus.version", "resources.version") START, END = "", "" -READERS = ("scripts/**/*", "justfile", "Dockerfile*", "docker-compose*.yml", "build.sbt") USES = re.compile(rf"^{ORG}/([\w.-]+)/\.github/workflows/([\w.-]+)@([\w.-]+)$") IMAGE = re.compile(rf"ghcr\.io/{ORG}/[\w.-]+") RELEASE = re.compile(r"gh release (?:upload|create)\s+\S+\s+(\S+)") @@ -44,11 +41,8 @@ def __init__(self) -> None: self.devkit: dict[str, list[str]] = {} self.calls: dict[str, dict[str, list[str]]] = {} - def art(self, key: str, repo: str = "", match: str = "") -> dict: - a = self.artifacts.setdefault(key, {"pub": [], "pin": [], "read": [], "repos": set()}) - a["repos"].add(repo) - a.setdefault("match", match) - return a + def art(self, key: str) -> dict: + return self.artifacts.setdefault(key, {"pub": [], "pin": [], "read": []}) def add(xs: list[str], x: str) -> None: @@ -67,10 +61,6 @@ def expr(m: re.Match) -> str: return SHVAR.sub(lambda m: str(env.get(m[1], m[0])), s) -def lines(text: str) -> str: - return "\n".join(x for x in text.splitlines() if not x.lstrip().startswith(("#", "//"))) - - def read(p: Path) -> str: try: return p.read_text(encoding="utf-8") @@ -82,13 +72,6 @@ def warn(msg: str) -> None: print(f"wiring: warning: {msg}", file=sys.stderr) -def git(d: Path, *args: str) -> str | None: - if not (d / ".git").exists(): # never the enclosing repo's answer - return None - r = subprocess.run(["git", "-C", str(d), *args], capture_output=True, text=True, check=False) - return r.stdout if r.returncode == 0 else None - - def load(text: str, where: object) -> dict: try: return yaml.safe_load(text) or {} @@ -96,13 +79,6 @@ def load(text: str, where: object) -> dict: raise SystemExit(f"wiring: {where}: {e}") from None -def version(d: Path) -> str: - if v := git(d, "describe", "--tags", "--always"): - return v.strip() - m = re.search(r"marola-devkit-([\d.]+)", str(d.resolve())) - return f"v{m[1]}" if m else "its working tree" - - def repos(root: Path, name: str, devkit: Path | None) -> list[tuple[str, Path]]: out = [(name, root)] for path in re.findall(r"^\s*path\s*=\s*(\S+)", read(root / ".gitmodules"), re.M): @@ -146,7 +122,7 @@ def site_of(repo_dir: Path, text: str) -> str: def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: - """(kind, value, via) for what a workflow sends, publishes, deploys or writes elsewhere.""" + """(kind, value, via) for what a workflow sends, publishes or deploys.""" out: list[tuple] = [] text = yaml.safe_dump(doc) for job in (doc.get("jobs") or {}).values(): @@ -161,13 +137,9 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: out += [(k, v, via) for k, v, _ in effects(target, repo, given, callee)] out.append(("ref", (m[2], m[3]), "")) continue - checkouts: dict[str, tuple[str, str]] = {} for step in job.get("steps") or []: senv = {**env, **(step.get("env") or {})} - uses, w = str(step.get("uses", "")), step.get("with") or {} - if uses.startswith("actions/checkout") and w.get("repository"): - name = sub(w["repository"], senv, inputs).rsplit("/", 1)[-1] - checkouts[str(w.get("path", "."))] = (name, str(w.get("ref", ""))) + uses = str(step.get("uses", "")) if uses.startswith("docker/build-push-action") and ( m := IMAGE.search(sub(env.get("IMAGE", ""), senv, inputs)) ): @@ -185,54 +157,20 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: for asset in RELEASE.findall(run): name = sub(asset.strip("\"'"), assigned, {}).rsplit("/", 1)[-1] out.append(("asset", SHVAR.sub("", name), "")) - pushing = [ln for ln in run.splitlines() if "--self-test" not in ln] - if branches := re.findall(r"([\w-]+)-push\.sh\b", "\n".join(pushing)): - for b in branches: - owner = next((r for r, ref in checkouts.values() if ref == b), repo) - out.append(("branch", (b, owner), "")) - continue - if m := re.search(r"gh pr create\b.*?--repo\s+(\S+)", run, re.S): - files = sorted( - { - Path(f).name - for f in " ".join(re.findall(r"git add\s+(.+)", run)).split() - if not f.startswith("-") and f != "." - } - ) - out.append( - ("pr", (sub(m[1].strip("\"'"), senv, inputs).rsplit("/", 1)[-1], files), "") - ) - elif m := re.search(r"(?:git -C (\S+) |cd (\S+)[^\n]*\n(?:.*\n)*?\s*git )push\b", run): - target = checkouts.get((m[1] or m[2]).strip("\"'")) - if target: - out.append(("push", target[0], "")) - return [(k, v, via) for k, v, via in out if k not in ("pr", "push") or v[0] != repo] + return out def scan(root: Path, name: str = "marola", devkit: Path | None = None) -> Wiring: w, rs = Wiring(), repos(root, name, devkit) - dirs = dict(rs) docs: dict[tuple[str, str], dict] = {} for repo, d in rs: for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): docs[(repo, f.name)] = load(read(f), f) - cache: dict[tuple[str, str, str], tuple[dict | None, str]] = {} def callee(repo: str, file: str, ref: str) -> tuple[dict | None, str]: - key, d = (repo, file, ref), dirs.get(repo) - if key in cache or d is None: - return cache.get(key, (None, "")) - if git(d, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}"): - text = git(d, "show", f"{ref}:.github/workflows/{file}") - cache[key] = (load(text, f"{repo} {ref}:{file}"), "") if text else (None, "") - elif (repo, file) in docs: - warn(f"{repo} has no ref {ref} here; reading {file} at {version(d)}") - cache[key] = (docs[(repo, file)], f" (resolved at {version(d)})") - else: - cache[key] = (None, "") - if cache[key][0] is None: + if (repo, file) not in docs: warn(f"{repo}/{file}@{ref} not found; its effects are not shown") - return cache[key] + return docs.get((repo, file)), "" for (repo, file), doc in docs.items(): on = triggers(doc) @@ -250,66 +188,14 @@ def callee(repo: str, file: str, ref: str) -> tuple[dict | None, str]: if kind == "send": add(w.dispatch.setdefault(v, {"sent": [], "listen": []})["sent"], pub) elif kind == "image": - add(w.art(f"`{v}` image", repo, v)["pub"], pub) + add(w.art(f"`{v}` image")["pub"], pub) elif kind == "asset": - add(w.art(f"`{v}` release asset", repo)["pub"], pub) - elif kind == "branch": - b, owner = v - key = f"`{b}` branch" + (f" of {owner}" if owner != repo else "") - add(w.art(key, owner, b)["pub"], pub) - elif kind == "pr": - a = w.art(f"PRs into {v[0]}: " + ", ".join(f"`{f}`" for f in v[1]), v[0]) - add(a["pub"], pub) - add(a["read"], v[0]) - elif kind == "push": - a = w.art(f"pushes to {v}", v) - add(a["pub"], pub) - add(a["read"], v) + add(w.art(f"`{v}` release asset")["pub"], pub) elif kind == "deploy": w.deploys.append((label, site_of(repo_dir, v))) elif kind == "ref": add(w.calls.setdefault(v[0], {}).setdefault(v[1], []), repo) for repo, d in rs: - files = sorted({f for g in READERS for f in d.glob(g) if f.is_file()}) - texts = {f.relative_to(d).as_posix(): lines(read(f)) for f in files} - if repo == "marola-devkit": - texts.pop("scripts/wiring.py", None) # its fixture names every idiom - for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): - texts[f".github/workflows/{f.name}"] = lines(read(f)) - for rel, text in texts.items(): - for img in re.findall(r"(?:image:|FROM)\s+(" + IMAGE.pattern + ")", text): - if (key := f"`{img}` image") in w.artifacts: - add(w.artifacts[key]["read"], f"{repo} `{Path(rel).name}`") - for key, a in w.artifacts.items(): - b = a["match"] - if " branch" not in key or rel.endswith(f"{b}-push.sh"): - continue - names = {b} | {v for v, x in ASSIGN.findall(text) if x == b} - if any( - re.search(r"\bgit\b[^\n]*\bfetch\b", ln) - and any(re.search(rf"(? str: return ", ".join(xs) or "—" out = ["| Artifact | Published by | Pinned in | Read by |", "|---|---|---|---|"] - kinds = (" image", "release asset", " branch", "PRs into", "pushes to") + kinds = (" image", "release asset") for key, a in sorted( w.artifacts.items(), key=lambda kv: ([k in kv[0] for k in kinds].index(True), kv[0]) ): @@ -370,15 +256,12 @@ def write_block(path: Path, block: str) -> None: for r in ("marola-app", "marola-site", "marola-corpus", "marola-ml") ), "mkdocs/mkdocs.yml": "site_name: x\nsite_url: https://docs.marola.dev/\n", - "scripts/fetch-api-docs.sh": 'API_DOCS_BRANCH="api-docs"\ngit -C "$tmp" fetch -q "$url" "$API_DOCS_BRANCH"\n', ".github/workflows/pointer-sync.yml": "on:\n repository_dispatch:\n types: [submodule-updated, submodule-docs-updated]\njobs: {}\n", ".github/workflows/docs.yml": "on: {push: {paths: [flake.lock]}}\njobs:\n d:\n steps:\n - uses: actions/deploy-pages@v5\n", ".devkit/.github/workflows/notify-umbrella.yml": "on:\n workflow_call:\n inputs:\n event-type: {type: string, default: submodule-docs-updated}\njobs:\n dispatch:\n steps:\n - env: {EVENT_TYPE: '${{ inputs.event-type }}'}\n run: curl https://api.github.com/repos/$UMBRELLA/dispatches -d x\n", ".devkit/.github/workflows/api-docs.yml": "on: {workflow_call: {inputs: {devkit-ref: {type: string}}}}\njobs:\n publish:\n steps:\n - run: bash .devkit-checkout/scripts/api-docs-push.sh out url sha\n", "marola-site/.github/workflows/notify-umbrella.yml": "on: {push: {branches: [main], paths: [README.md, 'docs/**']}}\njobs:\n n:\n uses: marola-dev/marola-devkit/.github/workflows/notify-umbrella.yml@v0.3.1\n", "marola-site/.github/workflows/site.yml": 'on:\n push: {paths: [marola-image]}\n repository_dispatch: {types: [site-data-updated]}\njobs:\n b:\n steps:\n - run: git fetch --depth=1 origin site-data\n - run: echo "marola.dev" > site/dist/CNAME\n - uses: actions/deploy-pages@v5\n', - "marola-site/marola-image": "ghcr.io/marola-dev/marola-app:jvm-8a29976@sha256:00\n", - "marola-site/scripts/board-schema.sh": '# marola-image is read here\nref="$(cat "$root/marola-image")"\n', "marola-corpus/.github/workflows/api-docs.yml": "on: {push: {branches: [main]}}\njobs:\n a:\n uses: marola-dev/marola-devkit/.github/workflows/api-docs.yml@v9.9.9\n", "marola-site/.github/workflows/gemini.yml": "on: {pull_request: {}}\njobs:\n g:\n uses: marola-dev/marola-devkit/.github/workflows/gemini-review.yml@v0.5.0\n", "marola-corpus/.github/workflows/release.yml": 'on: {push: {tags: [\'v*\']}}\njobs:\n t:\n steps:\n - run: |\n file=".tmp/marola-corpus-$TAG.tar.gz"\n gh release upload "$TAG" "$file"\n', @@ -388,60 +271,24 @@ def write_block(path: Path, block: str) -> None: "marola-app/.github/workflows/ci.yml": "on: {push: {branches: [main]}}\njobs:\n c:\n steps:\n" + CO % ("marola-dev/marola-site", "site-data", ", ref: site-data") + " - run: |\n git -C site-data commit -m x\n scripts/site-data-push.sh site-data\n - run: gh api repos/marola-dev/marola-site/dispatches -f event_type=site-data-updated\n", - "marola-app/.github/workflows/ingest.yml": "on: {schedule: [{cron: '0 4 * * *'}]}\njobs:\n i:\n steps:\n" - + CO % ("marola-dev/marola-oods", "oods", "") - + " - run: |\n git -C oods commit -qm ingest\n git -C oods push\n", - "marola-app/docker-compose.yml": "services:\n ollama-local:\n image: ghcr.io/marola-dev/marola-ml:local\n", "marola-app/flake.lock": json.dumps( {"nodes": {"marola-devkit": {"original": {"repo": "marola-devkit", "ref": "v0.4.1"}}}} ), "marola-ml/.github/workflows/docker-local.yml": "on: {push: {branches: [main]}}\nenv: {IMAGE: ghcr.io/marola-dev/marola-ml}\njobs:\n b:\n steps:\n - uses: docker/build-push-action@v7\n", - "marola-ml/.github/workflows/compile-prompt.yml": "on: {workflow_dispatch: {}}\nenv: {APP_REPO: marola-dev/marola-app}\njobs:\n c:\n steps:\n" - + CO % ("'${{ env.APP_REPO }}'", "app", "") - + ' - run: |\n cd app\n git add core/src/main/resources/recommendation_prompt.json core/src/main/resources/review_prompt.json\n git push -q origin "$BRANCH"\n gh pr create --repo "$APP_REPO" --base main\n', - "marola-ml/corpus.version": "v0.2.0\n", - "marola-ml/scripts/corpus-fetch.sh": 'pin="$(<"$root/corpus.version")"\nbase="https://github.com/marola-dev/marola-corpus/releases/download"\n', } -API_ROW = "| `api-docs` branch | marola-app `api-docs.yml` (`api-docs.yml@v0.4.1`, on a push to `main`), marola-corpus `api-docs.yml` (`api-docs.yml@v9.9.9` (resolved at v0.6.0), on a push to `main`) | — | marola `scripts/fetch-api-docs.sh` |" - # Each case is rows the fixture's block must hold; a "!" line is text it must not hold. CASES = { "dispatch_types_listened": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |\n| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |", "dispatch_send_step": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |", "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app), v9.9.9 (marola-corpus); `gemini-review.yml` v0.5.0 (marola-site); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", - "reusable_workflow_resolved_at_caller_ref": "| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |\n| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n" - + API_ROW, - "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |", - "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | marola-site `marola-image` | marola-site `scripts/board-schema.sh` |", + "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | — | — |", + "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | — | — |", + "pin_bump_on_push_paths": "| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", "deploy_pages_site": "| marola `docs.yml` | docs.marola.dev |\n| marola-site `site.yml` | marola.dev |", - "pin_file_and_reader": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |\n| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", - "branch_artifact_api_docs": API_ROW, - "branch_artifact_site_data": "| `site-data` branch of marola-site | marola-app `ci.yml` (on a push to `main`) | — | marola-site `site.yml` |\n!pushes to marola-site", - "compose_image_reader": "| `ghcr.io/marola-dev/marola-ml` image | marola-ml `docker-local.yml` (on a push to `main`) | — | marola-app `docker-compose.yml` |", - "cross_repo_pr": "| PRs into marola-app: `recommendation_prompt.json`, `review_prompt.json` | marola-ml `compile-prompt.yml` (by hand) | — | marola-app |\n!pushes to marola-app", - "cross_repo_push": "| pushes to marola-oods | marola-app `ingest.yml` (on a schedule) | — | marola-oods |", } -def _devkit_tags(dk: Path) -> None: - def g(*args: str) -> None: - cfg = ["-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false"] - subprocess.run(["git", "-C", str(dk), *cfg, *args], check=True, capture_output=True) - - g("init", "-q") - g("add", "-A") - g("commit", "-qm", "v0.3.1", "--no-verify") - g("tag", "v0.3.1") - g("tag", "v0.4.1") - nu = dk / ".github/workflows/notify-umbrella.yml" - nu.write_text( - nu.read_text().replace("default: submodule-docs-updated", "default: submodule-updated") - ) - g("commit", "-qam", "v0.6.0", "--no-verify") - g("tag", "v0.6.0") - - def self_test() -> int: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -450,7 +297,6 @@ def self_test() -> int: (root / rel).write_text(text, encoding="utf-8") for path in re.findall(r"path = (\S+)", FIXTURE[".gitmodules"]): (root / path / ".git").write_text("gitdir: unused\n", encoding="utf-8") - _devkit_tags(root / ".devkit") err = io.StringIO() with contextlib.redirect_stderr(err): w = scan(root) From e51b936687e2e57f704f0e6c06498c05ac892bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Fri, 9 Oct 2026 23:20:54 +0200 Subject: [PATCH 07/13] ci: install PyYAML before the self-tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pr.yml's python job runs tests/self-tests.sh on a bare setup-python, so wiring.py's `import yaml` failed there (ModuleNotFoundError) while the flake's python, which has it, passed. Tested: actionlint .github/workflows/pr.yml; wiring --self-test with pyyaml 6.0.3 in a venv Cost: ~$0.74 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- .github/workflows/pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b55f65c..ad3d7e5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -14,7 +14,9 @@ jobs: if: github.event.action != 'closed' uses: ./.github/workflows/python-ci.yml with: + # wiring.py imports yaml; the flake's python has it, a bare setup-python does not. self-test-commands: | + pip install "pyyaml==6.0.3" bash tests/self-tests.sh static: From e5a8dbaafe0ff74e2a3f76500897036470fe56fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Fri, 9 Oct 2026 23:21:46 +0200 Subject: [PATCH 08/13] fix(wiring): a `gh release create` with no files publishes no asset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release regex took the token after the tag as the asset, so the umbrella's and the devkit's `gh release create "$TAG" --repo "$GITHUB_REPOSITORY" …` rendered a `--repo` release asset. The files are now the non-flag arguments after the tag, shell-split so a quoted `--title` stays whole. Tested: wiring --self-test, release_create_without_assets red before, green after; the real umbrella run loses only the `--repo` row and is byte-identical across two runs Cost: ~$1.87 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- scripts/wiring.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/wiring.py b/scripts/wiring.py index 4e70236..f90a97f 100755 --- a/scripts/wiring.py +++ b/scripts/wiring.py @@ -15,6 +15,7 @@ import io import json import re +import shlex import sys import tempfile from pathlib import Path @@ -26,7 +27,9 @@ START, END = "", "" USES = re.compile(rf"^{ORG}/([\w.-]+)/\.github/workflows/([\w.-]+)@([\w.-]+)$") IMAGE = re.compile(rf"ghcr\.io/{ORG}/[\w.-]+") -RELEASE = re.compile(r"gh release (?:upload|create)\s+\S+\s+(\S+)") +RELEASE = re.compile(r"gh release (?:upload|create)\s+\S+([^\n]*)") +# gh release flags that take a value; everything else after the tag that is not a flag is a file. +VALUED = {"--repo", "-R", "--title", "-t", "--notes", "-n", "--notes-file", "-F", "--target"} ASSIGN = re.compile(r"""^\s*(\w+)=["']?([^"'\s]+)""", re.M) SHVAR = re.compile(r"\$\{?(\w+)\}?") EXPR = re.compile(r"\$\{\{\s*(env|inputs|github)\.([\w-]+)\s*\}\}") @@ -68,6 +71,23 @@ def read(p: Path) -> str: return "" +def release_files(run: str) -> list[str]: + out: list[str] = [] + for rest in RELEASE.findall(run): + try: + toks = shlex.split(rest.rstrip(" \\"), comments=True) + except ValueError: + toks = rest.split() + skip = False + for tok in toks: + if tok in ("&&", "||", "|", ";"): + break + if not skip and not tok.startswith("-"): + out.append(tok) + skip = not skip and tok in VALUED + return out + + def warn(msg: str) -> None: print(f"wiring: warning: {msg}", file=sys.stderr) @@ -154,8 +174,8 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: out += [("send", t, "") for t in sent if t and "$" not in t] if "/dispatches" in run: out += [("send", sub(v, {}, inputs), "") for k, v in senv.items() if "EVENT" in k] - for asset in RELEASE.findall(run): - name = sub(asset.strip("\"'"), assigned, {}).rsplit("/", 1)[-1] + for asset in release_files(run): + name = sub(asset, assigned, {}).rsplit("/", 1)[-1] out.append(("asset", SHVAR.sub("", name), "")) return out @@ -257,6 +277,7 @@ def write_block(path: Path, block: str) -> None: ), "mkdocs/mkdocs.yml": "site_name: x\nsite_url: https://docs.marola.dev/\n", ".github/workflows/pointer-sync.yml": "on:\n repository_dispatch:\n types: [submodule-updated, submodule-docs-updated]\njobs: {}\n", + ".github/workflows/release.yml": 'on: {push: {tags: [v*]}}\njobs:\n r:\n steps:\n - run: gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --verify-tag --title "x $TAG"\n', ".github/workflows/docs.yml": "on: {push: {paths: [flake.lock]}}\njobs:\n d:\n steps:\n - uses: actions/deploy-pages@v5\n", ".devkit/.github/workflows/notify-umbrella.yml": "on:\n workflow_call:\n inputs:\n event-type: {type: string, default: submodule-docs-updated}\njobs:\n dispatch:\n steps:\n - env: {EVENT_TYPE: '${{ inputs.event-type }}'}\n run: curl https://api.github.com/repos/$UMBRELLA/dispatches -d x\n", ".devkit/.github/workflows/api-docs.yml": "on: {workflow_call: {inputs: {devkit-ref: {type: string}}}}\njobs:\n publish:\n steps:\n - run: bash .devkit-checkout/scripts/api-docs-push.sh out url sha\n", @@ -283,6 +304,7 @@ def write_block(path: Path, block: str) -> None: "dispatch_send_step": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |", "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app), v9.9.9 (marola-corpus); `gemini-review.yml` v0.5.0 (marola-site); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | — | — |", + "release_create_without_assets": "!`--repo` release asset\n!`` release asset", "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | — | — |", "pin_bump_on_push_paths": "| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", "deploy_pages_site": "| marola `docs.yml` | docs.marola.dev |\n| marola-site `site.yml` | marola.dev |", From ee190e5c8347f9270c8c4ad037ab5a0b291a1b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Fri, 9 Oct 2026 23:22:30 +0200 Subject: [PATCH 09/13] feat(wiring): pin readers, branch artifacts, cross-repo writes, callees at their own ref MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The second half of marola-devkit#25, restoring what the stacked base set aside: the pin, compose, Dockerfile, justfile and build.sbt readers; the `api-docs` and `site-data` branches and who fetches them (Decision 1); the compiled-prompt PRs and pushes into another repo (Decision 3); and each reusable-workflow call read at its caller's ref in a devkit checkout with tags. Tested: wiring --self-test (17 cases); ruff check/format; docs_lint; the fresh umbrella run is #33's original block minus the `--repo` row Cost: ~$9.33 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- docs/4-reference_tools.md | 2 +- scripts/wiring.py | 182 +++++++++++++++++++++++++++++++++++--- 2 files changed, 169 insertions(+), 15 deletions(-) diff --git a/docs/4-reference_tools.md b/docs/4-reference_tools.md index df9597a..44325e1 100644 --- a/docs/4-reference_tools.md +++ b/docs/4-reference_tools.md @@ -57,7 +57,7 @@ extension rather than a devkit tool. | `ruleset-sync` | `ruleset-sync.sh` | `just rulesets-check`, `just rulesets-apply` | `check [owner/repo…]` diffs a repo's live branch ruleset against `.github/rulesets/main-rule.json`; `apply ` creates or updates it; `--all-org ORG` covers an org | | `agents-check` | `agents-check.sh` | `just agents-check` | Compare AGENTS.md's invariants block byte for byte with the pinned devkit's `agents/invariants.md` (`--block` or `MAROLA_INVARIANTS_BLOCK` to use another) | | `docs-lint` | `docs_lint.py` | (none) | MIP-0074 §7's stale-content check over `README.md` and `docs/**/*.md`: undefined or unmarked foreign recipes, another repo's paths, split-era wording, `docs/index.md`, links leaving the repo | -| `wiring` | `wiring.py` | `just wiring` | MIP-0076 §5.1's four tables (artifact, dispatch, pin bump, deploy), parsed from the umbrella's and every submodule's workflows (pin readers, branch artifacts and cross-repo writes are not read yet), with each devkit workflow call's ref. `wiring [FILE]` prints the block or rewrites it between FILE's `wiring:start`/`wiring:end` markers; run it in an umbrella checkout with submodules | +| `wiring` | `wiring.py` | `just wiring` | MIP-0076 §5.1's four tables (artifact, dispatch, pin bump, deploy), parsed from the umbrella's and every submodule's workflows, pin files, scripts, justfiles, Dockerfiles, compose files and `build.sbt`, with each devkit workflow call's ref. `wiring [FILE]` prints the block or rewrites it between FILE's `wiring:start`/`wiring:end` markers; run it in an umbrella checkout with submodules | | `workflow-runners` | `workflow_runners.py` | (none) | Fail when any workflow but the GPU publish one can reach a self-hosted runner ([runners](4-reference_runners.md)) | | `mip-resolve` | `mip-resolve.sh` | (none) | Print a MIP's doc or `.tasks.md` (`MIP-NNNN [tasks]`), or with `--path` where it was found, by the lookup in [design](1-design.md#how-tools-find-the-umbrella) | | `api-docs-push` | `api-docs-push.sh` | (none) | `api-docs-push [--branch api-docs]`: force-push a directory as one orphan commit, the `api-docs` workflow's push step | diff --git a/scripts/wiring.py b/scripts/wiring.py index f90a97f..566f11d 100755 --- a/scripts/wiring.py +++ b/scripts/wiring.py @@ -4,7 +4,8 @@ Run in an umbrella checkout with its submodules; the devkit's tree is `.devkit`, else this script's. wiring [--root DIR] [--devkit DIR] [--name NAME] [FILE] - print the block, or rewrite it between FILE's wiring markers + print the block, or rewrite it between FILE's wiring markers; a devkit git checkout with tags + lets each reusable-workflow call be read at its own @ref wiring --self-test """ @@ -16,6 +17,7 @@ import json import re import shlex +import subprocess import sys import tempfile from pathlib import Path @@ -25,6 +27,7 @@ ORG = "marola-dev" PINS = ("marola-image", "corpus.version", "resources.version") START, END = "", "" +READERS = ("scripts/**/*", "justfile", "Dockerfile*", "docker-compose*.yml", "build.sbt") USES = re.compile(rf"^{ORG}/([\w.-]+)/\.github/workflows/([\w.-]+)@([\w.-]+)$") IMAGE = re.compile(rf"ghcr\.io/{ORG}/[\w.-]+") RELEASE = re.compile(r"gh release (?:upload|create)\s+\S+([^\n]*)") @@ -44,8 +47,11 @@ def __init__(self) -> None: self.devkit: dict[str, list[str]] = {} self.calls: dict[str, dict[str, list[str]]] = {} - def art(self, key: str) -> dict: - return self.artifacts.setdefault(key, {"pub": [], "pin": [], "read": []}) + def art(self, key: str, repo: str = "", match: str = "") -> dict: + a = self.artifacts.setdefault(key, {"pub": [], "pin": [], "read": [], "repos": set()}) + a["repos"].add(repo) + a.setdefault("match", match) + return a def add(xs: list[str], x: str) -> None: @@ -64,6 +70,10 @@ def expr(m: re.Match) -> str: return SHVAR.sub(lambda m: str(env.get(m[1], m[0])), s) +def lines(text: str) -> str: + return "\n".join(x for x in text.splitlines() if not x.lstrip().startswith(("#", "//"))) + + def read(p: Path) -> str: try: return p.read_text(encoding="utf-8") @@ -92,6 +102,13 @@ def warn(msg: str) -> None: print(f"wiring: warning: {msg}", file=sys.stderr) +def git(d: Path, *args: str) -> str | None: + if not (d / ".git").exists(): # never the enclosing repo's answer + return None + r = subprocess.run(["git", "-C", str(d), *args], capture_output=True, text=True, check=False) + return r.stdout if r.returncode == 0 else None + + def load(text: str, where: object) -> dict: try: return yaml.safe_load(text) or {} @@ -99,6 +116,13 @@ def load(text: str, where: object) -> dict: raise SystemExit(f"wiring: {where}: {e}") from None +def version(d: Path) -> str: + if v := git(d, "describe", "--tags", "--always"): + return v.strip() + m = re.search(r"marola-devkit-([\d.]+)", str(d.resolve())) + return f"v{m[1]}" if m else "its working tree" + + def repos(root: Path, name: str, devkit: Path | None) -> list[tuple[str, Path]]: out = [(name, root)] for path in re.findall(r"^\s*path\s*=\s*(\S+)", read(root / ".gitmodules"), re.M): @@ -142,7 +166,7 @@ def site_of(repo_dir: Path, text: str) -> str: def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: - """(kind, value, via) for what a workflow sends, publishes or deploys.""" + """(kind, value, via) for what a workflow sends, publishes, deploys or writes elsewhere.""" out: list[tuple] = [] text = yaml.safe_dump(doc) for job in (doc.get("jobs") or {}).values(): @@ -157,9 +181,13 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: out += [(k, v, via) for k, v, _ in effects(target, repo, given, callee)] out.append(("ref", (m[2], m[3]), "")) continue + checkouts: dict[str, tuple[str, str]] = {} for step in job.get("steps") or []: senv = {**env, **(step.get("env") or {})} - uses = str(step.get("uses", "")) + uses, w = str(step.get("uses", "")), step.get("with") or {} + if uses.startswith("actions/checkout") and w.get("repository"): + name = sub(w["repository"], senv, inputs).rsplit("/", 1)[-1] + checkouts[str(w.get("path", "."))] = (name, str(w.get("ref", ""))) if uses.startswith("docker/build-push-action") and ( m := IMAGE.search(sub(env.get("IMAGE", ""), senv, inputs)) ): @@ -177,20 +205,54 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: for asset in release_files(run): name = sub(asset, assigned, {}).rsplit("/", 1)[-1] out.append(("asset", SHVAR.sub("", name), "")) - return out + pushing = [ln for ln in run.splitlines() if "--self-test" not in ln] + if branches := re.findall(r"([\w-]+)-push\.sh\b", "\n".join(pushing)): + for b in branches: + owner = next((r for r, ref in checkouts.values() if ref == b), repo) + out.append(("branch", (b, owner), "")) + continue + if m := re.search(r"gh pr create\b.*?--repo\s+(\S+)", run, re.S): + files = sorted( + { + Path(f).name + for f in " ".join(re.findall(r"git add\s+(.+)", run)).split() + if not f.startswith("-") and f != "." + } + ) + out.append( + ("pr", (sub(m[1].strip("\"'"), senv, inputs).rsplit("/", 1)[-1], files), "") + ) + elif m := re.search(r"(?:git -C (\S+) |cd (\S+)[^\n]*\n(?:.*\n)*?\s*git )push\b", run): + target = checkouts.get((m[1] or m[2]).strip("\"'")) + if target: + out.append(("push", target[0], "")) + return [(k, v, via) for k, v, via in out if k not in ("pr", "push") or v[0] != repo] def scan(root: Path, name: str = "marola", devkit: Path | None = None) -> Wiring: w, rs = Wiring(), repos(root, name, devkit) + dirs = dict(rs) docs: dict[tuple[str, str], dict] = {} for repo, d in rs: for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): docs[(repo, f.name)] = load(read(f), f) + cache: dict[tuple[str, str, str], tuple[dict | None, str]] = {} def callee(repo: str, file: str, ref: str) -> tuple[dict | None, str]: - if (repo, file) not in docs: + key, d = (repo, file, ref), dirs.get(repo) + if key in cache or d is None: + return cache.get(key, (None, "")) + if git(d, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}"): + text = git(d, "show", f"{ref}:.github/workflows/{file}") + cache[key] = (load(text, f"{repo} {ref}:{file}"), "") if text else (None, "") + elif (repo, file) in docs: + warn(f"{repo} has no ref {ref} here; reading {file} at {version(d)}") + cache[key] = (docs[(repo, file)], f" (resolved at {version(d)})") + else: + cache[key] = (None, "") + if cache[key][0] is None: warn(f"{repo}/{file}@{ref} not found; its effects are not shown") - return docs.get((repo, file)), "" + return cache[key] for (repo, file), doc in docs.items(): on = triggers(doc) @@ -208,14 +270,66 @@ def callee(repo: str, file: str, ref: str) -> tuple[dict | None, str]: if kind == "send": add(w.dispatch.setdefault(v, {"sent": [], "listen": []})["sent"], pub) elif kind == "image": - add(w.art(f"`{v}` image")["pub"], pub) + add(w.art(f"`{v}` image", repo, v)["pub"], pub) elif kind == "asset": - add(w.art(f"`{v}` release asset")["pub"], pub) + add(w.art(f"`{v}` release asset", repo)["pub"], pub) + elif kind == "branch": + b, owner = v + key = f"`{b}` branch" + (f" of {owner}" if owner != repo else "") + add(w.art(key, owner, b)["pub"], pub) + elif kind == "pr": + a = w.art(f"PRs into {v[0]}: " + ", ".join(f"`{f}`" for f in v[1]), v[0]) + add(a["pub"], pub) + add(a["read"], v[0]) + elif kind == "push": + a = w.art(f"pushes to {v}", v) + add(a["pub"], pub) + add(a["read"], v) elif kind == "deploy": w.deploys.append((label, site_of(repo_dir, v))) elif kind == "ref": add(w.calls.setdefault(v[0], {}).setdefault(v[1], []), repo) for repo, d in rs: + files = sorted({f for g in READERS for f in d.glob(g) if f.is_file()}) + texts = {f.relative_to(d).as_posix(): lines(read(f)) for f in files} + if repo == "marola-devkit": + texts.pop("scripts/wiring.py", None) # its fixture names every idiom + for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): + texts[f".github/workflows/{f.name}"] = lines(read(f)) + for rel, text in texts.items(): + for img in re.findall(r"(?:image:|FROM)\s+(" + IMAGE.pattern + ")", text): + if (key := f"`{img}` image") in w.artifacts: + add(w.artifacts[key]["read"], f"{repo} `{Path(rel).name}`") + for key, a in w.artifacts.items(): + b = a["match"] + if " branch" not in key or rel.endswith(f"{b}-push.sh"): + continue + names = {b} | {v for v, x in ASSIGN.findall(text) if x == b} + if any( + re.search(r"\bgit\b[^\n]*\bfetch\b", ln) + and any(re.search(rf"(? str: return ", ".join(xs) or "—" out = ["| Artifact | Published by | Pinned in | Read by |", "|---|---|---|---|"] - kinds = (" image", "release asset") + kinds = (" image", "release asset", " branch", "PRs into", "pushes to") for key, a in sorted( w.artifacts.items(), key=lambda kv: ([k in kv[0] for k in kinds].index(True), kv[0]) ): @@ -276,6 +390,7 @@ def write_block(path: Path, block: str) -> None: for r in ("marola-app", "marola-site", "marola-corpus", "marola-ml") ), "mkdocs/mkdocs.yml": "site_name: x\nsite_url: https://docs.marola.dev/\n", + "scripts/fetch-api-docs.sh": 'API_DOCS_BRANCH="api-docs"\ngit -C "$tmp" fetch -q "$url" "$API_DOCS_BRANCH"\n', ".github/workflows/pointer-sync.yml": "on:\n repository_dispatch:\n types: [submodule-updated, submodule-docs-updated]\njobs: {}\n", ".github/workflows/release.yml": 'on: {push: {tags: [v*]}}\njobs:\n r:\n steps:\n - run: gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --verify-tag --title "x $TAG"\n', ".github/workflows/docs.yml": "on: {push: {paths: [flake.lock]}}\njobs:\n d:\n steps:\n - uses: actions/deploy-pages@v5\n", @@ -283,6 +398,8 @@ def write_block(path: Path, block: str) -> None: ".devkit/.github/workflows/api-docs.yml": "on: {workflow_call: {inputs: {devkit-ref: {type: string}}}}\njobs:\n publish:\n steps:\n - run: bash .devkit-checkout/scripts/api-docs-push.sh out url sha\n", "marola-site/.github/workflows/notify-umbrella.yml": "on: {push: {branches: [main], paths: [README.md, 'docs/**']}}\njobs:\n n:\n uses: marola-dev/marola-devkit/.github/workflows/notify-umbrella.yml@v0.3.1\n", "marola-site/.github/workflows/site.yml": 'on:\n push: {paths: [marola-image]}\n repository_dispatch: {types: [site-data-updated]}\njobs:\n b:\n steps:\n - run: git fetch --depth=1 origin site-data\n - run: echo "marola.dev" > site/dist/CNAME\n - uses: actions/deploy-pages@v5\n', + "marola-site/marola-image": "ghcr.io/marola-dev/marola-app:jvm-8a29976@sha256:00\n", + "marola-site/scripts/board-schema.sh": '# marola-image is read here\nref="$(cat "$root/marola-image")"\n', "marola-corpus/.github/workflows/api-docs.yml": "on: {push: {branches: [main]}}\njobs:\n a:\n uses: marola-dev/marola-devkit/.github/workflows/api-docs.yml@v9.9.9\n", "marola-site/.github/workflows/gemini.yml": "on: {pull_request: {}}\njobs:\n g:\n uses: marola-dev/marola-devkit/.github/workflows/gemini-review.yml@v0.5.0\n", "marola-corpus/.github/workflows/release.yml": 'on: {push: {tags: [\'v*\']}}\njobs:\n t:\n steps:\n - run: |\n file=".tmp/marola-corpus-$TAG.tar.gz"\n gh release upload "$TAG" "$file"\n', @@ -292,25 +409,61 @@ def write_block(path: Path, block: str) -> None: "marola-app/.github/workflows/ci.yml": "on: {push: {branches: [main]}}\njobs:\n c:\n steps:\n" + CO % ("marola-dev/marola-site", "site-data", ", ref: site-data") + " - run: |\n git -C site-data commit -m x\n scripts/site-data-push.sh site-data\n - run: gh api repos/marola-dev/marola-site/dispatches -f event_type=site-data-updated\n", + "marola-app/.github/workflows/ingest.yml": "on: {schedule: [{cron: '0 4 * * *'}]}\njobs:\n i:\n steps:\n" + + CO % ("marola-dev/marola-oods", "oods", "") + + " - run: |\n git -C oods commit -qm ingest\n git -C oods push\n", + "marola-app/docker-compose.yml": "services:\n ollama-local:\n image: ghcr.io/marola-dev/marola-ml:local\n", "marola-app/flake.lock": json.dumps( {"nodes": {"marola-devkit": {"original": {"repo": "marola-devkit", "ref": "v0.4.1"}}}} ), "marola-ml/.github/workflows/docker-local.yml": "on: {push: {branches: [main]}}\nenv: {IMAGE: ghcr.io/marola-dev/marola-ml}\njobs:\n b:\n steps:\n - uses: docker/build-push-action@v7\n", + "marola-ml/.github/workflows/compile-prompt.yml": "on: {workflow_dispatch: {}}\nenv: {APP_REPO: marola-dev/marola-app}\njobs:\n c:\n steps:\n" + + CO % ("'${{ env.APP_REPO }}'", "app", "") + + ' - run: |\n cd app\n git add core/src/main/resources/recommendation_prompt.json core/src/main/resources/review_prompt.json\n git push -q origin "$BRANCH"\n gh pr create --repo "$APP_REPO" --base main\n', + "marola-ml/corpus.version": "v0.2.0\n", + "marola-ml/scripts/corpus-fetch.sh": 'pin="$(<"$root/corpus.version")"\nbase="https://github.com/marola-dev/marola-corpus/releases/download"\n', } +API_ROW = "| `api-docs` branch | marola-app `api-docs.yml` (`api-docs.yml@v0.4.1`, on a push to `main`), marola-corpus `api-docs.yml` (`api-docs.yml@v9.9.9` (resolved at v0.6.0), on a push to `main`) | — | marola `scripts/fetch-api-docs.sh` |" + # Each case is rows the fixture's block must hold; a "!" line is text it must not hold. CASES = { "dispatch_types_listened": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |\n| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |", "dispatch_send_step": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |", "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app), v9.9.9 (marola-corpus); `gemini-review.yml` v0.5.0 (marola-site); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", - "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | — | — |", + "reusable_workflow_resolved_at_caller_ref": "| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |\n| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n" + + API_ROW, + "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |", "release_create_without_assets": "!`--repo` release asset\n!`` release asset", - "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | — | — |", - "pin_bump_on_push_paths": "| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", + "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | marola-site `marola-image` | marola-site `scripts/board-schema.sh` |", "deploy_pages_site": "| marola `docs.yml` | docs.marola.dev |\n| marola-site `site.yml` | marola.dev |", + "pin_file_and_reader": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |\n| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", + "branch_artifact_api_docs": API_ROW, + "branch_artifact_site_data": "| `site-data` branch of marola-site | marola-app `ci.yml` (on a push to `main`) | — | marola-site `site.yml` |\n!pushes to marola-site", + "compose_image_reader": "| `ghcr.io/marola-dev/marola-ml` image | marola-ml `docker-local.yml` (on a push to `main`) | — | marola-app `docker-compose.yml` |", + "cross_repo_pr": "| PRs into marola-app: `recommendation_prompt.json`, `review_prompt.json` | marola-ml `compile-prompt.yml` (by hand) | — | marola-app |\n!pushes to marola-app", + "cross_repo_push": "| pushes to marola-oods | marola-app `ingest.yml` (on a schedule) | — | marola-oods |", } +def _devkit_tags(dk: Path) -> None: + def g(*args: str) -> None: + cfg = ["-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false"] + subprocess.run(["git", "-C", str(dk), *cfg, *args], check=True, capture_output=True) + + g("init", "-q") + g("add", "-A") + g("commit", "-qm", "v0.3.1", "--no-verify") + g("tag", "v0.3.1") + g("tag", "v0.4.1") + nu = dk / ".github/workflows/notify-umbrella.yml" + nu.write_text( + nu.read_text().replace("default: submodule-docs-updated", "default: submodule-updated") + ) + g("commit", "-qam", "v0.6.0", "--no-verify") + g("tag", "v0.6.0") + + def self_test() -> int: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -319,6 +472,7 @@ def self_test() -> int: (root / rel).write_text(text, encoding="utf-8") for path in re.findall(r"path = (\S+)", FIXTURE[".gitmodules"]): (root / path / ".git").write_text("gitdir: unused\n", encoding="utf-8") + _devkit_tags(root / ".devkit") err = io.StringIO() with contextlib.redirect_stderr(err): w = scan(root) From 0489baa315c5c0f29aebb57fbd0ad589abe9a90e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Fri, 9 Oct 2026 23:29:04 +0200 Subject: [PATCH 10/13] fix(wiring): release files, other repos' reusable workflows, unresolved event types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From review: `gh release` arguments are now shell-tokenised across `\` continuations, the tag is the first non-flag argument (so a flag before it no longer becomes an asset), and a glued `;`, `&&` or `>` ends the file list; the devkit-tag row only counts marola-devkit's own workflows; an `EVENT*` env that stays an unresolved `${{ … }}` is not a dispatch type. Tested: wiring --self-test, release_files_shell_forms, only_devkit_calls_in_tag_row and unresolved_event_type_not_sent red before and green after, release_create_without_assets now fails if --title is dropped from VALUED; the real umbrella run is unchanged Cost: ~$1.96 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- scripts/wiring.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/scripts/wiring.py b/scripts/wiring.py index f90a97f..6bf10f2 100755 --- a/scripts/wiring.py +++ b/scripts/wiring.py @@ -27,9 +27,10 @@ START, END = "", "" USES = re.compile(rf"^{ORG}/([\w.-]+)/\.github/workflows/([\w.-]+)@([\w.-]+)$") IMAGE = re.compile(rf"ghcr\.io/{ORG}/[\w.-]+") -RELEASE = re.compile(r"gh release (?:upload|create)\s+\S+([^\n]*)") -# gh release flags that take a value; everything else after the tag that is not a flag is a file. +RELEASE = re.compile(r"gh release (?:upload|create)\b([^\n]*)") +# gh release flags that take a value; the first other argument is the tag, the rest are files. VALUED = {"--repo", "-R", "--title", "-t", "--notes", "-n", "--notes-file", "-F", "--target"} +VALUED |= {"--discussion-category", "--notes-start-tag"} ASSIGN = re.compile(r"""^\s*(\w+)=["']?([^"'\s]+)""", re.M) SHVAR = re.compile(r"\$\{?(\w+)\}?") EXPR = re.compile(r"\$\{\{\s*(env|inputs|github)\.([\w-]+)\s*\}\}") @@ -73,18 +74,22 @@ def read(p: Path) -> str: def release_files(run: str) -> list[str]: out: list[str] = [] - for rest in RELEASE.findall(run): + for rest in RELEASE.findall(run.replace("\\\n", " ")): + lex = shlex.shlex(rest, posix=True, punctuation_chars=True) + lex.whitespace_split = True try: - toks = shlex.split(rest.rstrip(" \\"), comments=True) + toks = list(lex) except ValueError: toks = rest.split() + args: list[str] = [] skip = False for tok in toks: - if tok in ("&&", "||", "|", ";"): + if set(tok) <= set(lex.punctuation_chars): break if not skip and not tok.startswith("-"): - out.append(tok) + args.append(tok) skip = not skip and tok in VALUED + out += args[1:] return out @@ -155,7 +160,8 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: given |= {k: sub(v, env, inputs) for k, v in (job.get("with") or {}).items()} via = f"`{m[2]}@{m[3]}`{note}" out += [(k, v, via) for k, v, _ in effects(target, repo, given, callee)] - out.append(("ref", (m[2], m[3]), "")) + if m[1] == "marola-devkit": + out.append(("ref", (m[2], m[3]), "")) continue for step in job.get("steps") or []: senv = {**env, **(step.get("env") or {})} @@ -173,7 +179,8 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: sent = re.findall(r"event_type=([^\s\"']*)", sub(run, senv, inputs)) out += [("send", t, "") for t in sent if t and "$" not in t] if "/dispatches" in run: - out += [("send", sub(v, {}, inputs), "") for k, v in senv.items() if "EVENT" in k] + evs = [sub(v, {}, inputs) for k, v in senv.items() if "EVENT" in k] + out += [("send", t, "") for t in evs if "$" not in t] for asset in release_files(run): name = sub(asset, assigned, {}).rsplit("/", 1)[-1] out.append(("asset", SHVAR.sub("", name), "")) @@ -284,6 +291,7 @@ def write_block(path: Path, block: str) -> None: "marola-site/.github/workflows/notify-umbrella.yml": "on: {push: {branches: [main], paths: [README.md, 'docs/**']}}\njobs:\n n:\n uses: marola-dev/marola-devkit/.github/workflows/notify-umbrella.yml@v0.3.1\n", "marola-site/.github/workflows/site.yml": 'on:\n push: {paths: [marola-image]}\n repository_dispatch: {types: [site-data-updated]}\njobs:\n b:\n steps:\n - run: git fetch --depth=1 origin site-data\n - run: echo "marola.dev" > site/dist/CNAME\n - uses: actions/deploy-pages@v5\n', "marola-corpus/.github/workflows/api-docs.yml": "on: {push: {branches: [main]}}\njobs:\n a:\n uses: marola-dev/marola-devkit/.github/workflows/api-docs.yml@v9.9.9\n", + "marola-site/.github/workflows/odd.yml": "on: {workflow_dispatch: {}}\njobs:\n r:\n uses: marola-dev/marola-app/.github/workflows/reusable.yml@main\n d:\n env: {EVENT_TYPE: '${{ inputs.ev }}'}\n steps:\n - run: curl https://api.github.com/repos/x/dispatches -d y\n", "marola-site/.github/workflows/gemini.yml": "on: {pull_request: {}}\njobs:\n g:\n uses: marola-dev/marola-devkit/.github/workflows/gemini-review.yml@v0.5.0\n", "marola-corpus/.github/workflows/release.yml": 'on: {push: {tags: [\'v*\']}}\njobs:\n t:\n steps:\n - run: |\n file=".tmp/marola-corpus-$TAG.tar.gz"\n gh release upload "$TAG" "$file"\n', "marola-app/.github/workflows/docker.yml": "on: {push: {branches: [main], paths: [corpus.version]}}\nenv: {IMAGE: 'ghcr.io/${{ github.repository_owner }}/marola-app'}\njobs:\n jvm:\n steps:\n - uses: docker/build-push-action@v7\n", @@ -304,7 +312,9 @@ def write_block(path: Path, block: str) -> None: "dispatch_send_step": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |", "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app), v9.9.9 (marola-corpus); `gemini-review.yml` v0.5.0 (marola-site); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | — | — |", - "release_create_without_assets": "!`--repo` release asset\n!`` release asset", + "release_create_without_assets": "!marola `release.yml`", + "only_devkit_calls_in_tag_row": "!reusable.yml", + "unresolved_event_type_not_sent": "!inputs.ev", "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | — | — |", "pin_bump_on_push_paths": "| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", "deploy_pages_site": "| marola `docs.yml` | docs.marola.dev |\n| marola-site `site.yml` | marola.dev |", @@ -340,6 +350,8 @@ def self_test() -> int: (name, all(x[1:] not in block if x[0] == "!" else x in rows for x in want.split("\n"))) for name, want in CASES.items() ] + shell = 'gh release upload --clobber "$TAG" "$f"; echo done\ngh release create "$TAG" --discussion-category General --notes-start-tag v1 \\\n dist/a.tgz>out.log\n' + cases.append(("release_files_shell_forms", release_files(shell) == ["$f", "dist/a.tgz"])) cases.append(("missing_callee_warns", "gemini-review.yml@v0.5.0 not found" in err.getvalue())) cases.append(("uninitialised_submodule_fails", "marola-x is not checked out" in uninit)) cases.append(("block_between_markers", f"{START}\n\n{block}\n{END}\n\nAfter." in repos_md)) From 797c7eeb4ec975b70b345236fc7f16db4e0bb22c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Fri, 9 Oct 2026 23:29:41 +0200 Subject: [PATCH 11/13] feat(wiring): --check fails on a stale block, an orphan dispatch, an unread artifact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit marola-devkit#26, MIP-0076 §5.1's gate: the umbrella's quality-other will run it so the block in REPOS.md cannot drift and a dispatch or artifact with one side missing shows up in review. Every problem is reported in one run. An unread artifact is excepted by `/wiring.allow` (Decision 4), one `: ` per line; an entry without a reason, or naming no artifact, fails too, so an exception cannot outlive what it excepts. Tested: wiring --self-test, the 6 new check cases red against a stub then green (24 cases); ruff check/format; docs_lint; tests/self-tests.sh; --check over a fresh umbrella clone fails on `submodule-updated` only Cost: ~$5.01 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- docs/4-reference_tools.md | 2 +- scripts/wiring.py | 83 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/docs/4-reference_tools.md b/docs/4-reference_tools.md index 3991924..309b3d6 100644 --- a/docs/4-reference_tools.md +++ b/docs/4-reference_tools.md @@ -57,7 +57,7 @@ extension rather than a devkit tool. | `ruleset-sync` | `ruleset-sync.sh` | `just rulesets-check`, `just rulesets-apply` | `check [owner/repo…]` diffs a repo's live branch ruleset against `.github/rulesets/main-rule.json`; `apply ` creates or updates it; `--all-org ORG` covers an org | | `agents-check` | `agents-check.sh` | `just agents-check` | Compare AGENTS.md's invariants block byte for byte with the pinned devkit's `agents/invariants.md` (`--block` or `MAROLA_INVARIANTS_BLOCK` to use another) | | `docs-lint` | `docs_lint.py` | (none) | MIP-0074 §7's stale-content check over `README.md` and `docs/**/*.md`: undefined or unmarked foreign recipes, another repo's paths, split-era wording, `docs/index.md`, links leaving the repo | -| `wiring` | `wiring.py` | `just wiring` | MIP-0076 §5.1's four tables (artifact, dispatch, pin bump, deploy), parsed from the umbrella's and every submodule's workflows, pin files, scripts, justfiles, Dockerfiles, compose files and `build.sbt`, with each devkit workflow call's ref. `wiring [FILE]` prints the block or rewrites it between FILE's `wiring:start`/`wiring:end` markers; run it in an umbrella checkout with submodules | +| `wiring` | `wiring.py` | `just wiring` | MIP-0076 §5.1's four tables (artifact, dispatch, pin bump, deploy), parsed from the umbrella's and every submodule's workflows, pin files, scripts, justfiles, Dockerfiles, compose files and `build.sbt`, with each devkit workflow call's ref. `wiring [FILE]` prints the block or rewrites it between FILE's `wiring:start`/`wiring:end` markers; `--check [FILE]` fails on a stale block, a dispatch sent or listened for on one side only, or an artifact nobody reads that the umbrella's `wiring.allow` (`: ` per line) does not list; run it in an umbrella checkout with submodules | | `workflow-runners` | `workflow_runners.py` | (none) | Fail when any workflow but the GPU publish one can reach a self-hosted runner ([runners](4-reference_runners.md)) | | `skills-vendor` | `skills_vendor.py` | `just skills-check`, `just skills-outdated`, `just skills-update` | Pin vendored skills to an upstream commit in `skills.lock` (below) | diff --git a/scripts/wiring.py b/scripts/wiring.py index 566f11d..cd4e502 100755 --- a/scripts/wiring.py +++ b/scripts/wiring.py @@ -6,6 +6,9 @@ wiring [--root DIR] [--devkit DIR] [--name NAME] [FILE] print the block, or rewrite it between FILE's wiring markers; a devkit git checkout with tags lets each reusable-workflow call be read at its own @ref +wiring --check [FILE] + fail on a stale block in FILE, an orphan dispatch, or an artifact nobody reads that + /wiring.allow does not list as `: ` wiring --self-test """ @@ -383,6 +386,38 @@ def write_block(path: Path, block: str) -> None: path.write_text(f"{head}{START}\n\n{block}\n{END}{tail}", encoding="utf-8") +def check(w: Wiring, block: str, path: Path | None, allow: str) -> list[str]: + out: list[str] = [] + if path: + _, s, rest = read(path).partition(START) + inner, e, _ = rest.partition(END) + if not (s and e) or inner != f"\n\n{block}\n": + out.append(f"{path}'s wiring block is stale: run `just wiring {path}`") + for t, d in sorted(w.dispatch.items()): + if not (d["sent"] and d["listen"]): + side = "sent but never listened for" if d["sent"] else "listened for but never sent" + out.append(f"dispatch `{t}` is {side}") + names = {k.replace("`", ""): a for k, a in w.artifacts.items()} + allowed = set() + for ln in map(str.strip, allow.splitlines()): + if not ln or ln.startswith("#"): + continue + # an artifact name can hold ": " itself (PRs into …: files), so match the known ones first + name = next((n for n in names if ln.startswith(f"{n}:")), ln.partition(":")[0]) + if not ln[len(name) + 1 :].strip(): + out.append(f"wiring.allow: `{name}` needs a reason") + elif name not in names: + out.append(f"wiring.allow: `{name}` names no artifact") + else: + allowed.add(name) + for n, a in names.items(): + if not a["read"] and n not in allowed: + out.append( + f"`{n}` has no reader; list it in wiring.allow with a reason if that is intended" + ) + return out + + CO = " - uses: actions/checkout@v7\n with: {repository: %s, path: %s%s}\n" FIXTURE = { ".gitmodules": "".join( @@ -489,6 +524,19 @@ def self_test() -> int: ) write_block(root / "REPOS.md", block) repos_md = (root / "REPOS.md").read_text(encoding="utf-8") + (root / "STALE.md").write_text(f"{START}\n\n{block}| x |\n{END}\n", encoding="utf-8") + fresh = check(w, block, root / "REPOS.md", "") + stale = check(w, block, root / "STALE.md", "") + ml = "ghcr.io/marola-dev/marola-ml image" + w.artifacts["`ghcr.io/marola-dev/marola-ml` image"]["read"].clear() + unread = check(w, block, None, "") + allowed = check(w, block, None, f"# kept\n\n{ml}: kept for local runs\n") + no_reason = check(w, block, None, f"{ml}: \n") + no_artifact = check(w, block, None, f"{ml}: kept\ngone image: retired\n") + w.dispatch["site-data-updated"]["sent"].clear() + w.dispatch["submodule-updated"]["listen"].clear() + orphans = check(w, block, None, f"{ml}: kept\n") + every = check(w, block, root / "STALE.md", "") rows = block.splitlines() cases = [ (name, all(x[1:] not in block if x[0] == "!" else x in rows for x in want.split("\n"))) @@ -497,6 +545,32 @@ def self_test() -> int: cases.append(("missing_callee_warns", "gemini-review.yml@v0.5.0 not found" in err.getvalue())) cases.append(("uninitialised_submodule_fails", "marola-x is not checked out" in uninit)) cases.append(("block_between_markers", f"{START}\n\n{block}\n{END}\n\nAfter." in repos_md)) + cases.append(("check_passes_fresh_block_and_reasoned_allow", fresh == allowed == [])) + cases.append(("check_fails_on_stale_block", len(stale) == 1 and "just wiring" in stale[0])) + cases.append( + ( + "check_fails_on_orphan_dispatch", + orphans + == [ + "dispatch `site-data-updated` is listened for but never sent", + "dispatch `submodule-updated` is sent but never listened for", + ], + ) + ) + cases.append( + ( + "check_fails_on_unread_artifact", + len(unread) == 1 and unread[0].startswith(f"`{ml}` has no reader"), + ) + ) + cases.append(("allow_entry_needs_reason", any("needs a reason" in p for p in no_reason))) + cases.append( + ( + "allow_entry_names_an_artifact", + no_artifact == ["wiring.allow: `gone image` names no artifact"], + ) + ) + cases.append(("check_reports_every_problem", len(every) == 4)) fails = [n for n, ok in cases if not ok] for name, ok in cases: print(f" {'ok ' if ok else 'FAIL'} {name}") @@ -518,11 +592,18 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--root", type=Path, default=Path.cwd()) ap.add_argument("--devkit", type=Path, help="default: /.devkit, else this script's repo") ap.add_argument("--name", default="marola", help="the umbrella's repo name") + ap.add_argument("--check", action="store_true") ap.add_argument("--self-test", action="store_true") args = ap.parse_args(argv) if args.self_test: return self_test() - block = render(scan(args.root, args.name, args.devkit)) + w = scan(args.root, args.name, args.devkit) + block = render(w) + if args.check: + problems = check(w, block, args.file, read(args.root / "wiring.allow")) + for p in problems: + print(f"wiring: {p}", file=sys.stderr) + return 1 if problems else 0 if args.file: write_block(args.file, block) else: From a2a9cb4e3a868622f74a9fe95b1bf3b73b0de5e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Fri, 9 Oct 2026 23:31:46 +0200 Subject: [PATCH 12/13] fix(notify-umbrella): drop the event-type input, as row 3 says MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No caller in the org passes it (checked every notify-umbrella caller in a fresh umbrella clone with submodules), so keeping it declared-but-ignored only kept a dead input in the contract. Tested: actionlint; docs_lint; grep of every caller for event-type (0) Cost: ~$1.53 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- .github/workflows/notify-umbrella.yml | 4 ---- docs/4-reference_workflows.md | 1 - 2 files changed, 5 deletions(-) diff --git a/.github/workflows/notify-umbrella.yml b/.github/workflows/notify-umbrella.yml index 4c1e427..c585859 100644 --- a/.github/workflows/notify-umbrella.yml +++ b/.github/workflows/notify-umbrella.yml @@ -29,10 +29,6 @@ on: description: "owner/repo to dispatch to (MAROLA_UMBRELLA, MIP-0070 §5.6)." type: string default: marola-dev/marola - event-type: - description: "Deprecated, ignored (MIP-0076): every call now sends submodule-updated, plus submodule-docs-updated on a docs change. Kept declared so a caller still passing it doesn't fail GitHub's reusable-workflow input check." - type: string - default: "" runner: description: "Runner label for the dispatch job — a reusable workflow's runs-on resolves in the caller's repository." type: string diff --git a/docs/4-reference_workflows.md b/docs/4-reference_workflows.md index 0a5aef3..c13670e 100644 --- a/docs/4-reference_workflows.md +++ b/docs/4-reference_workflows.md @@ -138,7 +138,6 @@ jobs: | Input | Default | Notes | |---|---|---| | `umbrella` | `marola-dev/marola` | MAROLA_UMBRELLA, MIP-0070 §5.6 | -| `event-type` | `""` | deprecated, ignored (MIP-0076) — kept so a caller still passing it doesn't fail GitHub's input check | | `runner` | `ubuntu-latest` | resolved in the *caller's* repo | **Secret** `token` (optional): every repo passes the org secret `MAROLA_CROSS_REPO_PAT`, a From 6dc709e19539c29cb5fa6fda7451348fed7d1050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Fri, 9 Oct 2026 23:36:34 +0200 Subject: [PATCH 13/13] fix(wiring): read each call at its ref without git; PRs into consumers.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under Nix `.devkit` is a store path with no `.git`, so every `@ref` fell back to the working tree and a v0.3.1 caller was read at the newest version: the block depended on whether the machine had a tagged clone. A ref the local checkout lacks is now fetched from raw.githubusercontent.com (a version tag cached under ~/.cache/marola-wiring, a branch never); a 404 is "not found", and only a network failure falls back to the working tree with the "(resolved at …)" note. bump-consumers.yml's PRs (`gh pr create -R "$repo"` inside scripts/bump-consumers.sh, one per line of .github/consumers.txt) were a missing cross-repo write: PR detection now takes `-R`, reads a repo script a `run:` invokes, and turns a target looped over an owner/repo list into one row. Also from review: a call into a repo that is not checked out warns, a PR into the caller's own repo via `$GITHUB_REPOSITORY`/`${{ github.repository }}` is dropped, `FROM --platform=…`, quoted and `${X:-…}` images are readers, and the self-test survives global tag signing. Tested: wiring --self-test (28 cases; cross_repo_pr_each_listed, own_repo_pr_dropped, image_reader_forms, uncloned_callee_warns, callee_fetched_at_ref_without_git, callee_404_not_resolved_at_tree, tag_fetch_cached_branch_not red before and green after; commented_pin_not_a_reader red with the comment filter removed), ruff check/format, docs_lint, tests/self-tests.sh; real umbrella run differs only by the bump-consumers row, and is identical with a .git-less devkit fetching over the network Cost: ~$8.54 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- docs/4-reference_tools.md | 2 +- scripts/wiring.py | 164 ++++++++++++++++++++++++++++++++------ 2 files changed, 141 insertions(+), 25 deletions(-) diff --git a/docs/4-reference_tools.md b/docs/4-reference_tools.md index 3991924..9731f0b 100644 --- a/docs/4-reference_tools.md +++ b/docs/4-reference_tools.md @@ -57,7 +57,7 @@ extension rather than a devkit tool. | `ruleset-sync` | `ruleset-sync.sh` | `just rulesets-check`, `just rulesets-apply` | `check [owner/repo…]` diffs a repo's live branch ruleset against `.github/rulesets/main-rule.json`; `apply ` creates or updates it; `--all-org ORG` covers an org | | `agents-check` | `agents-check.sh` | `just agents-check` | Compare AGENTS.md's invariants block byte for byte with the pinned devkit's `agents/invariants.md` (`--block` or `MAROLA_INVARIANTS_BLOCK` to use another) | | `docs-lint` | `docs_lint.py` | (none) | MIP-0074 §7's stale-content check over `README.md` and `docs/**/*.md`: undefined or unmarked foreign recipes, another repo's paths, split-era wording, `docs/index.md`, links leaving the repo | -| `wiring` | `wiring.py` | `just wiring` | MIP-0076 §5.1's four tables (artifact, dispatch, pin bump, deploy), parsed from the umbrella's and every submodule's workflows, pin files, scripts, justfiles, Dockerfiles, compose files and `build.sbt`, with each devkit workflow call's ref. `wiring [FILE]` prints the block or rewrites it between FILE's `wiring:start`/`wiring:end` markers; run it in an umbrella checkout with submodules | +| `wiring` | `wiring.py` | `just wiring` | MIP-0076 §5.1's four tables (artifact, dispatch, pin bump, deploy), parsed from the umbrella's and every submodule's workflows, pin files, scripts, justfiles, Dockerfiles, compose files and `build.sbt`, with each devkit workflow call's ref (each call is read at that ref: from a local devkit clone's tags, else fetched from GitHub, a tag cached under `~/.cache/marola-wiring`). `wiring [FILE]` prints the block or rewrites it between FILE's `wiring:start`/`wiring:end` markers; run it in an umbrella checkout with submodules | | `workflow-runners` | `workflow_runners.py` | (none) | Fail when any workflow but the GPU publish one can reach a self-hosted runner ([runners](4-reference_runners.md)) | | `skills-vendor` | `skills_vendor.py` | `just skills-check`, `just skills-outdated`, `just skills-update` | Pin vendored skills to an upstream commit in `skills.lock` (below) | diff --git a/scripts/wiring.py b/scripts/wiring.py index d73fdb3..d7846f8 100755 --- a/scripts/wiring.py +++ b/scripts/wiring.py @@ -5,7 +5,8 @@ wiring [--root DIR] [--devkit DIR] [--name NAME] [FILE] print the block, or rewrite it between FILE's wiring markers; a devkit git checkout with tags - lets each reusable-workflow call be read at its own @ref + lets each reusable-workflow call be read at its own @ref, else it is fetched from GitHub + (a tag is cached under ~/.cache/marola-wiring) wiring --self-test """ @@ -15,11 +16,15 @@ import contextlib import io import json +import os import re import shlex +import shutil import subprocess import sys import tempfile +import urllib.error +import urllib.request from pathlib import Path import yaml @@ -30,6 +35,9 @@ READERS = ("scripts/**/*", "justfile", "Dockerfile*", "docker-compose*.yml", "build.sbt") USES = re.compile(rf"^{ORG}/([\w.-]+)/\.github/workflows/([\w.-]+)@([\w.-]+)$") IMAGE = re.compile(rf"ghcr\.io/{ORG}/[\w.-]+") +IMAGE_REF = re.compile( + r"""(?:image:\s*["']?(?:\$\{\w+:-)?|FROM\s+(?:--platform=\S+\s+)?)(""" + IMAGE.pattern + ")" +) RELEASE = re.compile(r"gh release (?:upload|create)\b([^\n]*)") # gh release flags that take a value; the first other argument is the tag, the rest are files. VALUED = {"--repo", "-R", "--title", "-t", "--notes", "-n", "--notes-file", "-F", "--target"} @@ -64,7 +72,8 @@ def sub(s: str, env: dict, inputs: dict) -> str: def expr(m: re.Match) -> str: ctx, name = m.groups() if ctx == "github": - return ORG if name == "repository_owner" else m[0] + known = {"repository_owner": ORG, "repository": env.get("GITHUB_REPOSITORY")} + return known.get(name) or m[0] return str((env if ctx == "env" else inputs).get(name, m[0])) s = EXPR.sub(expr, str(s)) @@ -103,6 +112,42 @@ def release_files(run: str) -> list[str]: return out +def fetch(url: str) -> str | None: + """url's text, None on a 404; any other failure raises OSError.""" + try: + with urllib.request.urlopen(url, timeout=10) as r: + return r.read().decode("utf-8") + except urllib.error.HTTPError as e: + if e.code == 404: + return None + raise + + +def at_ref(repo: str, ref: str, file: str) -> str | None: + cached = Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") + cached = cached / "marola-wiring" / repo / ref / file + tagged = re.fullmatch(r"v\d+\.\d+\.\d+", ref) and repo not in (".", "..") + if tagged and cached.is_file(): # a branch moves, so only a tag is cached + return read(cached) + text = fetch(f"https://raw.githubusercontent.com/{ORG}/{repo}/{ref}/.github/workflows/{file}") + if tagged and text is not None: + with contextlib.suppress(OSError): + cached.parent.mkdir(parents=True, exist_ok=True) + cached.write_text(text, encoding="utf-8") + return text + + +def listed(d: Path, text: str) -> tuple[str, list[str]] | None: + """The first file text names that holds only owner/repo lines, and those repos.""" + for tok in re.findall(r"[\w.-]*(?:/[\w.-]+)+", text): + parts = [x for x in tok.split("/") if x] + for i in range(len(parts) * (".." not in parts)): + names = lines(read(d.joinpath(*parts[i:]))).split() + if names and all(re.fullmatch(r"[\w.-]+/[\w.-]+", n) for n in names): + return "/".join(parts[i:]), [n.split("/")[1] for n in names] + return None + + def warn(msg: str) -> None: print(f"wiring: warning: {msg}", file=sys.stderr) @@ -170,12 +215,16 @@ def site_of(repo_dir: Path, text: str) -> str: return "?" -def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: +def effects(doc: dict, repo: str, inputs: dict, callee, d: Path) -> list[tuple]: """(kind, value, via) for what a workflow sends, publishes, deploys or writes elsewhere.""" out: list[tuple] = [] text = yaml.safe_dump(doc) for job in (doc.get("jobs") or {}).values(): - env = {**(doc.get("env") or {}), **(job.get("env") or {})} + env = { + "GITHUB_REPOSITORY": f"{ORG}/{repo}", + **(doc.get("env") or {}), + **(job.get("env") or {}), + } if m := USES.match(str(job.get("uses", ""))): target, note = callee(m[1], m[2], m[3]) if target: @@ -183,7 +232,7 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: given = {k: v.get("default", "") for k, v in defaults.items()} given |= {k: sub(v, env, inputs) for k, v in (job.get("with") or {}).items()} via = f"`{m[2]}@{m[3]}`{note}" - out += [(k, v, via) for k, v, _ in effects(target, repo, given, callee)] + out += [(k, v, via) for k, v, _ in effects(target, repo, given, callee, d)] if m[1] == "marola-devkit": out.append(("ref", (m[2], m[3]), "")) continue @@ -212,23 +261,32 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: for asset in release_files(run): name = sub(asset, assigned, {}).rsplit("/", 1)[-1] out.append(("asset", SHVAR.sub("", name), "")) - pushing = [ln for ln in run.splitlines() if "--self-test" not in ln] - if branches := re.findall(r"([\w-]+)-push\.sh\b", "\n".join(pushing)): + live = "\n".join(ln for ln in run.splitlines() if "--self-test" not in ln) + # A trap: any `-push.sh` call is taken as publishing branch ``. + if branches := re.findall(r"([\w-]+)-push\.sh\b", live): for b in branches: owner = next((r for r, ref in checkouts.values() if ref == b), repo) out.append(("branch", (b, owner), "")) continue - if m := re.search(r"gh pr create\b.*?--repo\s+(\S+)", run, re.S): + scripts = re.findall(r"(? Wiring def callee(repo: str, file: str, ref: str) -> tuple[dict | None, str]: key, d = (repo, file, ref), dirs.get(repo) - if key in cache or d is None: - return cache.get(key, (None, "")) - if git(d, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}"): + if key in cache: + return cache[key] + cache[key] = (None, "") + if d and git(d, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}"): text = git(d, "show", f"{ref}:.github/workflows/{file}") - cache[key] = (load(text, f"{repo} {ref}:{file}"), "") if text else (None, "") - elif (repo, file) in docs: - warn(f"{repo} has no ref {ref} here; reading {file} at {version(d)}") - cache[key] = (docs[(repo, file)], f" (resolved at {version(d)})") - else: - cache[key] = (None, "") + else: # under Nix .devkit is a store path with no .git + try: + text = at_ref(repo, ref, file) + except OSError as e: + text = None + if d and (repo, file) in docs: + warn(f"cannot fetch {repo}@{ref} ({e}); reading {file} at {version(d)}") + cache[key] = (docs[(repo, file)], f" (resolved at {version(d)})") + if text: + cache[key] = (load(text, f"{repo} {ref}:{file}"), "") if cache[key][0] is None: warn(f"{repo}/{file}@{ref} not found; its effects are not shown") return cache[key] @@ -271,7 +334,7 @@ def callee(repo: str, file: str, ref: str) -> tuple[dict | None, str]: for pin in set(PINS + ("flake.lock",)) & set(on.get("push", {}).get("paths") or []): add(w.bumps.setdefault(f"{repo} `{pin}`", []), label) repo_dir = dict(rs)[repo] - for kind, v, via in effects(doc, repo, {}, callee): + for kind, v, via in effects(doc, repo, {}, callee, dirs[repo]): parts = [p for p in (via, when(on)) if p] pub = label + (f" ({', '.join(parts)})" if parts else "") if kind == "send": @@ -288,6 +351,11 @@ def callee(repo: str, file: str, ref: str) -> tuple[dict | None, str]: a = w.art(f"PRs into {v[0]}: " + ", ".join(f"`{f}`" for f in v[1]), v[0]) add(a["pub"], pub) add(a["read"], v[0]) + elif kind == "prs": + a = w.art(f"PRs into each repo in `{v[0]}`", repo) + add(a["pub"], pub) + for r in v[1]: + add(a["read"], r) elif kind == "push": a = w.art(f"pushes to {v}", v) add(a["pub"], pub) @@ -304,7 +372,7 @@ def callee(repo: str, file: str, ref: str) -> tuple[dict | None, str]: for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): texts[f".github/workflows/{f.name}"] = lines(read(f)) for rel, text in texts.items(): - for img in re.findall(r"(?:image:|FROM)\s+(" + IMAGE.pattern + ")", text): + for img in IMAGE_REF.findall(text): if (key := f"`{img}` image") in w.artifacts: add(w.artifacts[key]["read"], f"{repo} `{Path(rel).name}`") for key, a in w.artifacts.items(): @@ -408,7 +476,7 @@ def write_block(path: Path, block: str) -> None: "marola-site/marola-image": "ghcr.io/marola-dev/marola-app:jvm-8a29976@sha256:00\n", "marola-site/scripts/board-schema.sh": '# marola-image is read here\nref="$(cat "$root/marola-image")"\n', "marola-corpus/.github/workflows/api-docs.yml": "on: {push: {branches: [main]}}\njobs:\n a:\n uses: marola-dev/marola-devkit/.github/workflows/api-docs.yml@v9.9.9\n", - "marola-site/.github/workflows/odd.yml": "on: {workflow_dispatch: {}}\njobs:\n r:\n uses: marola-dev/marola-app/.github/workflows/reusable.yml@main\n d:\n env: {EVENT_TYPE: '${{ inputs.ev }}'}\n steps:\n - run: curl https://api.github.com/repos/x/dispatches -d y\n", + "marola-site/.github/workflows/odd.yml": "on: {workflow_dispatch: {}}\njobs:\n r:\n uses: marola-dev/marola-app/.github/workflows/reusable.yml@main\n x:\n uses: marola-dev/marola-infra/.github/workflows/x.yml@v1\n d:\n env: {EVENT_TYPE: '${{ inputs.ev }}'}\n steps:\n - run: curl https://api.github.com/repos/x/dispatches -d y\n", "marola-site/.github/workflows/gemini.yml": "on: {pull_request: {}}\njobs:\n g:\n uses: marola-dev/marola-devkit/.github/workflows/gemini-review.yml@v0.5.0\n", "marola-corpus/.github/workflows/release.yml": 'on: {push: {tags: [\'v*\']}}\njobs:\n t:\n steps:\n - run: |\n file=".tmp/marola-corpus-$TAG.tar.gz"\n gh release upload "$TAG" "$file"\n', "marola-app/.github/workflows/docker.yml": "on: {push: {branches: [main], paths: [corpus.version]}}\nenv: {IMAGE: 'ghcr.io/${{ github.repository_owner }}/marola-app'}\njobs:\n jvm:\n steps:\n - uses: docker/build-push-action@v7\n", @@ -429,6 +497,15 @@ def write_block(path: Path, block: str) -> None: + CO % ("'${{ env.APP_REPO }}'", "app", "") + ' - run: |\n cd app\n git add core/src/main/resources/recommendation_prompt.json core/src/main/resources/review_prompt.json\n git push -q origin "$BRANCH"\n gh pr create --repo "$APP_REPO" --base main\n', "marola-ml/corpus.version": "v0.2.0\n", + "marola-ml/justfile": "# corpus.version is bumped by its release\nfetch:\n echo x\n", + "marola-site/.github/workflows/self-pr.yml": 'on: {workflow_dispatch: {}}\njobs:\n a:\n steps:\n - run: gh pr create --repo "$GITHUB_REPOSITORY" --base main\n b:\n steps:\n - run: gh pr create -R "${{ github.repository }}" --base main\n', + "marola-corpus/.github/workflows/docker.yml": "on: {push: {branches: [main]}}\nenv: {IMAGE: ghcr.io/marola-dev/marola-corpus}\njobs:\n b:\n steps:\n - uses: docker/build-push-action@v7\n", + "marola-site/docker-compose.yml": 'services:\n c:\n image: "ghcr.io/marola-dev/marola-corpus:1"\n', + "marola-ml/Dockerfile": "FROM --platform=$BUILDPLATFORM ghcr.io/marola-dev/marola-corpus:1\n", + "marola-ml/docker-compose.yml": "services:\n c:\n image: ${CORPUS_IMAGE:-ghcr.io/marola-dev/marola-corpus:1}\n", + ".devkit/.github/workflows/bump-consumers.yml": 'on: {workflow_dispatch: {}}\njobs:\n b:\n steps:\n - run: scripts/bump-consumers.sh "$VERSION"\n', + ".devkit/scripts/bump-consumers.sh": 'repos() { grep -v "^#" "$root/.github/consumers.txt"; }\nwhile read -r repo; do\n gh pr create -R "$repo" --head x\ndone < <(repos)\n', + ".devkit/.github/consumers.txt": "# pin the devkit\nmarola-dev/marola\nmarola-dev/marola-app\n", "marola-ml/scripts/corpus-fetch.sh": 'pin="$(<"$root/corpus.version")"\nbase="https://github.com/marola-dev/marola-corpus/releases/download"\n', } @@ -453,12 +530,17 @@ def write_block(path: Path, block: str) -> None: "compose_image_reader": "| `ghcr.io/marola-dev/marola-ml` image | marola-ml `docker-local.yml` (on a push to `main`) | — | marola-app `docker-compose.yml` |", "cross_repo_pr": "| PRs into marola-app: `recommendation_prompt.json`, `review_prompt.json` | marola-ml `compile-prompt.yml` (by hand) | — | marola-app |\n!pushes to marola-app", "cross_repo_push": "| pushes to marola-oods | marola-app `ingest.yml` (on a schedule) | — | marola-oods |", + "cross_repo_pr_each_listed": "| PRs into each repo in `.github/consumers.txt` | marola-devkit `bump-consumers.yml` (by hand) | — | marola, marola-app |", + "own_repo_pr_dropped": "!self-pr.yml", + "image_reader_forms": "| `ghcr.io/marola-dev/marola-corpus` image | marola-corpus `docker.yml` (on a push to `main`) | — | marola-site `docker-compose.yml`, marola-ml `Dockerfile`, marola-ml `docker-compose.yml` |", + "commented_pin_not_a_reader": "!marola-ml `justfile`", } def _devkit_tags(dk: Path) -> None: def g(*args: str) -> None: cfg = ["-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false"] + cfg += ["-c", "tag.gpgsign=false"] subprocess.run(["git", "-C", str(dk), *cfg, *args], check=True, capture_output=True) g("init", "-q") @@ -475,8 +557,18 @@ def g(*args: str) -> None: def self_test() -> int: + global fetch + served: dict[str, str | None] = {} + + def fake(url: str) -> str | None: + if url not in served: + raise OSError("offline") + return served[url] + + fetch = fake with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) + os.environ["XDG_CACHE_HOME"] = str(root / ".cache") for rel, text in FIXTURE.items(): (root / rel).parent.mkdir(parents=True, exist_ok=True) (root / rel).write_text(text, encoding="utf-8") @@ -487,6 +579,19 @@ def self_test() -> int: with contextlib.redirect_stderr(err): w = scan(root) block = render(w) + shutil.rmtree(root / ".devkit" / ".git") # a Nix store path + raw = f"https://raw.githubusercontent.com/{ORG}/%s/%s/.github/workflows/%s" + served[raw % ("marola-devkit", "v0.3.1", "notify-umbrella.yml")] = FIXTURE[ + ".devkit/.github/workflows/notify-umbrella.yml" + ] + served[raw % ("marola-devkit", "v9.9.9", "api-docs.yml")] = None + served[raw % ("marola-app", "main", "reusable.yml")] = "on: {workflow_call: {}}\n" + err2 = io.StringIO() + with contextlib.redirect_stderr(err2): + fetched = render(scan(root)).splitlines() + cache = root / ".cache" / "marola-wiring" + tagged = (cache / "marola-devkit" / "v0.3.1" / "notify-umbrella.yml").is_file() + branch = (cache / "marola-app").exists() (root / "marola-x").mkdir() (root / ".gitmodules").write_text("[submodule]\n\tpath = marola-x\n", encoding="utf-8") try: @@ -507,6 +612,17 @@ def self_test() -> int: shell = 'gh release upload --clobber "$TAG" "$f"; echo done\ngh release create "$TAG" --discussion-category General --notes-start-tag v1 \\\n dist/a.tgz>out.log\n' cases.append(("release_files_shell_forms", release_files(shell) == ["$f", "dist/a.tgz"])) cases.append(("missing_callee_warns", "gemini-review.yml@v0.5.0 not found" in err.getvalue())) + cases.append(("uncloned_callee_warns", "marola-infra/x.yml@v1 not found" in err.getvalue())) + nu = "| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |" + cases.append(("callee_fetched_at_ref_without_git", nu in fetched)) + cases.append( + ( + "callee_404_not_resolved_at_tree", + "api-docs.yml@v9.9.9 not found" in err2.getvalue() + and not any("v9.9.9` (resolved" in r for r in fetched), + ) + ) + cases.append(("tag_fetch_cached_branch_not", tagged and not branch)) cases.append(("uninitialised_submodule_fails", "marola-x is not checked out" in uninit)) cases.append(("block_between_markers", f"{START}\n\n{block}\n{END}\n\nAfter." in repos_md)) fails = [n for n, ok in cases if not ok]