From 81646923240d885953293cc707d3ffab71950a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Mon, 5 Oct 2026 09:03:36 +0200 Subject: [PATCH 1/7] feat(wiring): extract the org's cross-repo wiring into four tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/wiring.py, MIP-0076 §5.1 without the gate: it reads .gitmodules, the devkit's tree under .devkit (else its own), every repo's workflows, pin files, and the scripts, justfiles, Dockerfiles, docker-compose*.yml and build.sbt naming a pin or image. It renders four tables (artifact, dispatch, pin bump, deploy) between a file's wiring:start/wiring:end markers. Beyond §5.1's sources, per the tasks file's decisions: branch artifacts (api-docs, site-data) from the `-push.sh` idiom, read by any file fetching the branch (Decision 1); compose `image:` lines as image readers (Decision 2); a workflow step opening a PR in or pushing to another marola-dev repo (Decision 3). A devkit reusable workflow's effects are attributed to each caller with the call's ref, and its inputs resolved from the caller's `with:`. Closes #25 Tested: wiring --self-test (13 cases, red against a stubbed scan, green after); a run over a fresh --recurse-submodules umbrella clone at 0145e12 Cost: ~$19.79 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- scripts/wiring.py | 422 ++++++++++++++++++++++++++++++++++++++++++++ tests/self-tests.sh | 2 +- 2 files changed, 423 insertions(+), 1 deletion(-) create mode 100755 scripts/wiring.py diff --git a/scripts/wiring.py b/scripts/wiring.py new file mode 100755 index 0000000..05c58f6 --- /dev/null +++ b/scripts/wiring.py @@ -0,0 +1,422 @@ +#!/usr/bin/env python3 +"""The org's cross-repo wiring, parsed from every repo's workflows, pins and scripts (MIP-0076 §5.1). + +Run in an umbrella checkout with its submodules; the devkit's tree is `.devkit`, else this script's. + +wiring [--root DIR] [FILE] print the block, or rewrite it between FILE's wiring markers +wiring --self-test +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +ORG = "marola-dev" +SELF = Path(__file__) # its fixture names every idiom, so it is never a reader +PINS = ("marola-image", "corpus.version", "resources.version") +START, END = "", "" +READERS = ("scripts/**/*", "justfile", "Dockerfile*", "docker-compose*.yml", "build.sbt") +USES = re.compile(rf"^{ORG}/([\w.-]+)/\.github/workflows/([\w.-]+)@([\w.-]+)$") +IMAGE = re.compile(rf"ghcr\.io/{ORG}/[\w.-]+") +RELEASE = re.compile(r"gh release (?:upload|create)\s+\S+\s+(\S+)") +ASSIGN = re.compile(r"""^\s*(\w+)=["']?([^"'\s]+)""", re.M) +SHVAR = re.compile(r"\$\{?(\w+)\}?") +EXPR = re.compile(r"\$\{\{\s*(env|inputs|github)\.([\w-]+)\s*\}\}") + + +class Wiring: + def __init__(self) -> None: + self.artifacts: dict[str, dict] = {} + self.dispatch: dict[str, dict[str, list[str]]] = {} + self.bumps: dict[str, list[str]] = {} + self.deploys: list[tuple[str, str]] = [] + self.devkit: dict[str, list[str]] = {} + self.calls: dict[str, dict[str, list[str]]] = {} + + def art(self, key: str, repo: str = "", match: str = "") -> dict: + a = self.artifacts.setdefault(key, {"pub": [], "pin": [], "read": [], "repos": set()}) + a["repos"].add(repo) + a.setdefault("match", match) + return a + + +def add(xs: list[str], x: str) -> None: + if x not in xs: + xs.append(x) + + +def sub(s: str, env: dict, inputs: dict) -> str: + def expr(m: re.Match) -> str: + ctx, name = m.groups() + if ctx == "github": + return ORG if name == "repository_owner" else m[0] + return str((env if ctx == "env" else inputs).get(name, m[0])) + + s = EXPR.sub(expr, str(s)) + return SHVAR.sub(lambda m: str(env.get(m[1], m[0])), s) + + +def lines(text: str) -> str: + return "\n".join(x for x in text.splitlines() if not x.lstrip().startswith(("#", "//"))) + + +def read(p: Path) -> str: + try: + return p.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return "" + + +def repos(root: Path) -> list[tuple[str, Path]]: + url = subprocess.run( + ["git", "-C", str(root), "config", "--get", "remote.origin.url"], + capture_output=True, + text=True, + check=False, + ).stdout.strip() + out = [(re.sub(r"\.git$", "", url.rsplit("/", 1)[-1]) or "marola", root)] + for path in re.findall(r"^\s*path\s*=\s*(\S+)", read(root / ".gitmodules"), re.M): + out.append((Path(path).name, root / path)) + devkit = root / ".devkit" + out.append( + ("marola-devkit", devkit if devkit.is_dir() else Path(__file__).resolve().parents[1]) + ) + return [(n, p) for n, p in out if p.is_dir()] + + +def triggers(doc: dict) -> dict: + on = doc.get("on", doc.get(True)) or {} + if isinstance(on, str | list): + on = {k: None for k in ([on] if isinstance(on, str) else on)} + return {k: v or {} for k, v in on.items()} + + +def when(on: dict) -> str: + push = on.get("push", {}) + if push.get("tags"): + return f"on a `{push['tags'][0]}` tag" + if "push" in on: + paths = push.get("paths") or [] + touching = " touching " + ", ".join(f"`{p}`" for p in paths) if 0 < len(paths) <= 3 else "" + return f"on a push to `{(push.get('branches') or ['main'])[0]}`{touching}" + if "schedule" in on: + return "on a schedule" + return "by hand" if set(on) == {"workflow_dispatch"} else "" + + +def site_of(repo_dir: Path, text: str) -> str: + if m := re.search(r"""echo\s+["']?([\w.-]+\.[a-z]+)["']?\s*>\s*\S*CNAME""", text): + return m[1] + for f in ("mkdocs/mkdocs.yml", "mkdocs.yml"): + if m := re.search(r"^site_url:\s*https?://([^/\s]+)", read(repo_dir / f), re.M): + return m[1] + return "?" + + +def effects(doc: dict, repo: str, inputs: dict, workflows: dict) -> list[tuple]: + """(kind, value, via) for what a workflow sends, publishes, deploys or writes elsewhere.""" + out: list[tuple] = [] + text = yaml.safe_dump(doc) + for job in (doc.get("jobs") or {}).values(): + env = {**(doc.get("env") or {}), **(job.get("env") or {})} + if m := USES.match(str(job.get("uses", ""))): + target = workflows.get((m[1], m[2])) + if target: + defaults = triggers(target).get("workflow_call", {}).get("inputs") or {} + given = {k: v.get("default", "") for k, v in defaults.items()} + given |= {k: sub(v, env, inputs) for k, v in (job.get("with") or {}).items()} + via = f"`{m[2]}@{m[3]}`" + out += [(k, v, via) for k, v, _ in effects(target, repo, given, workflows)] + out.append(("ref", (m[2], m[3]), "")) + continue + checkouts: dict[str, tuple[str, str]] = {} + for step in job.get("steps") or []: + senv = {**env, **(step.get("env") or {})} + uses, w = str(step.get("uses", "")), step.get("with") or {} + if uses.startswith("actions/checkout") and w.get("repository"): + name = sub(w["repository"], senv, inputs).rsplit("/", 1)[-1] + checkouts[str(w.get("path", "."))] = (name, str(w.get("ref", ""))) + if uses.startswith("docker/build-push-action") and ( + m := IMAGE.search(sub(env.get("IMAGE", ""), senv, inputs)) + ): + out.append(("image", m[0], "")) + if uses.startswith("actions/deploy-pages"): + out.append(("deploy", text, "")) + run = step.get("run") + if not run: + continue + assigned = dict(ASSIGN.findall(run)) + out += [("send", t, "") for t in re.findall(r"event_type=([\w-]+)", run)] + if "/dispatches" in run: + out += [("send", sub(v, {}, inputs), "") for k, v in senv.items() if "EVENT" in k] + for asset in RELEASE.findall(run): + name = sub(asset.strip("\"'"), assigned, {}).rsplit("/", 1)[-1] + out.append(("asset", SHVAR.sub("", name), "")) + if branches := re.findall(r"([\w-]+)-push\.sh\b", run): + for b in branches: + owner = next((r for r, ref in checkouts.values() if ref == b), repo) + out.append(("branch", (b, owner), "")) + continue + if m := re.search(r"gh pr create\b.*?--repo\s+(\S+)", run, re.S): + files = sorted( + {Path(f).name for f in " ".join(re.findall(r"git add\s+(.+)", run)).split()} + ) + out.append( + ("pr", (sub(m[1].strip("\"'"), senv, inputs).rsplit("/", 1)[-1], files), "") + ) + elif m := re.search(r"(?:git -C (\S+) |cd (\S+)[^\n]*\n(?:.*\n)*?\s*git )push\b", run): + target = checkouts.get((m[1] or m[2]).strip("\"'")) + if target: + out.append(("push", target[0], "")) + return [(k, v, via) for k, v, via in out if k not in ("pr", "push") or v[0] != repo] + + +def scan(root: Path) -> Wiring: + w, rs = Wiring(), repos(root) + docs: dict[tuple[str, str], dict] = {} + for name, d in rs: + for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): + docs[(name, f.name)] = yaml.safe_load(read(f)) or {} + for (repo, file), doc in docs.items(): + on = triggers(doc) + if set(on) == {"workflow_call"}: + continue + label = f"{repo} `{file}`" + for t in on.get("repository_dispatch", {}).get("types") or []: + add(w.dispatch.setdefault(t, {"sent": [], "listen": []})["listen"], label) + for pin in set(PINS + ("flake.lock",)) & set(on.get("push", {}).get("paths") or []): + add(w.bumps.setdefault(f"{repo} `{pin}`", []), label) + repo_dir = dict(rs)[repo] + for kind, v, via in effects(doc, repo, {}, docs): + parts = [p for p in (via, when(on)) if p] + pub = label + (f" ({', '.join(parts)})" if parts else "") + if kind == "send": + add(w.dispatch.setdefault(v, {"sent": [], "listen": []})["sent"], pub) + elif kind == "image": + add(w.art(f"`{v}` image", repo, v)["pub"], pub) + elif kind == "asset": + add(w.art(f"`{v}` release asset", repo)["pub"], pub) + elif kind == "branch": + b, owner = v + key = f"`{b}` branch" + (f" of {owner}" if owner != repo else "") + add(w.art(key, owner, b)["pub"], pub) + elif kind == "pr": + a = w.art(f"PRs into {v[0]}: " + ", ".join(f"`{f}`" for f in v[1]), v[0]) + add(a["pub"], pub), add(a["read"], v[0]) + elif kind == "push": + a = w.art(f"pushes to {v}", v) + add(a["pub"], pub), add(a["read"], v) + elif kind == "deploy": + w.deploys.append((label, site_of(repo_dir, v))) + elif kind == "ref": + add(w.calls.setdefault(v[0], {}).setdefault(v[1], []), repo) + for repo, d in rs: + files = sorted( + {f for g in READERS for f in d.glob(g) if f.is_file() and f.name != SELF.name} + ) + texts = {f.relative_to(d).as_posix(): lines(read(f)) for f in files} + for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): + texts[f".github/workflows/{f.name}"] = lines(read(f)) + for rel, text in texts.items(): + for img in re.findall(r"(?:image:|FROM)\s+(" + IMAGE.pattern + ")", text): + if (key := f"`{img}` image") in w.artifacts: + add(w.artifacts[key]["read"], f"{repo} `{Path(rel).name}`") + for key, a in w.artifacts.items(): + b = a["match"] + if " branch" not in key or rel.endswith(f"{b}-push.sh"): + continue + names = {b} | {v for v, x in ASSIGN.findall(text) if x == b} + if any( + "fetch" in ln + and any(re.search(rf"(? str: + def cell(xs) -> str: + return ", ".join(xs) or "—" + + out = ["| Artifact | Published by | Pinned in | Read by |", "|---|---|---|---|"] + kinds = (" image", "release asset", " branch", "PRs into", "pushes to") + for key, a in sorted( + w.artifacts.items(), key=lambda kv: ([k in kv[0] for k in kinds].index(True), kv[0]) + ): + out.append(f"| {key} | {cell(a['pub'])} | {cell(a['pin'])} | {cell(a['read'])} |") + pins = "; ".join(f"{r} {', '.join(v)}" for r, v in w.devkit.items()) or "—" + calls = "; ".join( + f"`{f}` " + ", ".join(f"{ref} ({' '.join(rs)})" for ref, rs in sorted(refs.items())) + for f, refs in sorted(w.calls.items()) + ) + out += [f"| a marola-devkit tag | marola-devkit | {pins} | {calls or '—'} |", ""] + out += ["| Dispatch | Sent by | Triggers |", "|---|---|---|"] + for t, d in sorted(w.dispatch.items()): + out.append(f"| `{t}` | {cell(d['sent'])} | {cell(d['listen'])} |") + out += ["", "| Pin bump | Workflow |", "|---|---|"] + out += [f"| {p} | {cell(wf)} |" for p, wf in sorted(w.bumps.items())] + out += ["", "| Deploy | Site |", "|---|---|"] + out += [f"| {wf} | {site} |" for wf, site in w.deploys] + return "\n".join(out) + "\n" + + +def write_block(path: Path, block: str) -> None: + text = path.read_text(encoding="utf-8") + head, sep, rest = text.partition(START) + _, sep2, tail = rest.partition(END) + if not (sep and sep2): + raise SystemExit(f"wiring: {path} has no {START} … {END} markers") + path.write_text(f"{head}{START}\n\n{block}\n{END}{tail}", encoding="utf-8") + + +CO = " - uses: actions/checkout@v7\n with: {repository: %s, path: %s%s}\n" +FIXTURE = { + ".gitmodules": "".join( + f'[submodule "{r}"]\n\tpath = {r}\n' + for r in ("marola-app", "marola-site", "marola-corpus", "marola-ml") + ), + "mkdocs/mkdocs.yml": "site_name: x\nsite_url: https://docs.marola.dev/\n", + "scripts/fetch-api-docs.sh": 'API_DOCS_BRANCH="api-docs"\ngit -C "$tmp" fetch -q "$url" "$API_DOCS_BRANCH"\n', + ".github/workflows/pointer-sync.yml": "on:\n repository_dispatch:\n types: [submodule-updated, submodule-docs-updated]\njobs: {}\n", + ".github/workflows/docs.yml": "on: {push: {paths: [flake.lock]}}\njobs:\n d:\n steps:\n - uses: actions/deploy-pages@v5\n", + ".devkit/.github/workflows/notify-umbrella.yml": "on:\n workflow_call:\n inputs:\n event-type: {type: string, default: submodule-docs-updated}\njobs:\n dispatch:\n steps:\n - env: {EVENT_TYPE: '${{ inputs.event-type }}'}\n run: curl https://api.github.com/repos/$UMBRELLA/dispatches -d x\n", + ".devkit/.github/workflows/api-docs.yml": "on: {workflow_call: {inputs: {devkit-ref: {type: string}}}}\njobs:\n publish:\n steps:\n - run: bash .devkit-checkout/scripts/api-docs-push.sh out url sha\n", + "marola-site/.github/workflows/notify-umbrella.yml": "on: {push: {branches: [main], paths: [README.md, 'docs/**']}}\njobs:\n n:\n uses: marola-dev/marola-devkit/.github/workflows/notify-umbrella.yml@v0.3.1\n", + "marola-site/.github/workflows/site.yml": 'on:\n push: {paths: [marola-image]}\n repository_dispatch: {types: [site-data-updated]}\njobs:\n b:\n steps:\n - run: git fetch --depth=1 origin site-data\n - run: echo "marola.dev" > site/dist/CNAME\n - uses: actions/deploy-pages@v5\n', + "marola-site/marola-image": "ghcr.io/marola-dev/marola-app:jvm-8a29976@sha256:00\n", + "marola-site/scripts/board-schema.sh": '# marola-image is read here\nref="$(cat "$root/marola-image")"\n', + "marola-corpus/.github/workflows/release.yml": 'on: {push: {tags: [\'v*\']}}\njobs:\n t:\n steps:\n - run: |\n file=".tmp/marola-corpus-$TAG.tar.gz"\n gh release upload "$TAG" "$file"\n', + "marola-app/.github/workflows/docker.yml": "on: {push: {branches: [main], paths: [corpus.version]}}\nenv: {IMAGE: 'ghcr.io/${{ github.repository_owner }}/marola-app'}\njobs:\n jvm:\n steps:\n - uses: docker/build-push-action@v7\n", + "marola-app/.github/workflows/api-docs.yml": "on: {push: {branches: [main]}}\njobs:\n a:\n uses: marola-dev/marola-devkit/.github/workflows/api-docs.yml@v0.4.1\n with: {devkit-ref: v0.4.1}\n", + "marola-app/.github/workflows/notify-umbrella.yml": "on: {push: {branches: [main]}}\njobs:\n n:\n uses: marola-dev/marola-devkit/.github/workflows/notify-umbrella.yml@v0.6.0\n with: {event-type: submodule-updated}\n", + "marola-app/.github/workflows/ci.yml": "on: {push: {branches: [main]}}\njobs:\n c:\n steps:\n" + + CO % ("marola-dev/marola-site", "site-data", ", ref: site-data") + + " - run: |\n git -C site-data commit -m x\n scripts/site-data-push.sh site-data\n - run: gh api repos/marola-dev/marola-site/dispatches -f event_type=site-data-updated\n", + "marola-app/.github/workflows/ingest.yml": "on: {schedule: [{cron: '0 4 * * *'}]}\njobs:\n i:\n steps:\n" + + CO % ("marola-dev/marola-oods", "oods", "") + + " - run: |\n git -C oods commit -qm ingest\n git -C oods push\n", + "marola-app/docker-compose.yml": "services:\n ollama-local:\n image: ghcr.io/marola-dev/marola-ml:local\n", + "marola-app/flake.lock": json.dumps( + {"nodes": {"marola-devkit": {"original": {"repo": "marola-devkit", "ref": "v0.4.1"}}}} + ), + "marola-ml/.github/workflows/docker-local.yml": "on: {push: {branches: [main]}}\nenv: {IMAGE: ghcr.io/marola-dev/marola-ml}\njobs:\n b:\n steps:\n - uses: docker/build-push-action@v7\n", + "marola-ml/.github/workflows/compile-prompt.yml": "on: {workflow_dispatch: {}}\nenv: {APP_REPO: marola-dev/marola-app}\njobs:\n c:\n steps:\n" + + CO % ("'${{ env.APP_REPO }}'", "app", "") + + ' - run: |\n cd app\n git add core/src/main/resources/recommendation_prompt.json core/src/main/resources/review_prompt.json\n git push -q origin "$BRANCH"\n gh pr create --repo "$APP_REPO" --base main\n', + "marola-ml/corpus.version": "v0.2.0\n", + "marola-ml/scripts/corpus-fetch.sh": 'pin="$(<"$root/corpus.version")"\nbase="https://github.com/marola-dev/marola-corpus/releases/download"\n', +} + + +# Each case is rows the fixture's block must hold; a "!" line is text it must not hold. +CASES = { + "dispatch_types_listened": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |\n| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |", + "dispatch_send_step": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |", + "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", + "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |", + "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | marola-site `marola-image` | marola-site `scripts/board-schema.sh` |", + "deploy_pages_site": "| marola `docs.yml` | docs.marola.dev |\n| marola-site `site.yml` | marola.dev |", + "pin_file_and_reader": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |\n| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", + "branch_artifact_api_docs": "| `api-docs` branch | marola-app `api-docs.yml` (`api-docs.yml@v0.4.1`, on a push to `main`) | — | marola `scripts/fetch-api-docs.sh` |", + "branch_artifact_site_data": "| `site-data` branch of marola-site | marola-app `ci.yml` (on a push to `main`) | — | marola-site `site.yml` |\n!pushes to marola-site", + "compose_image_reader": "| `ghcr.io/marola-dev/marola-ml` image | marola-ml `docker-local.yml` (on a push to `main`) | — | marola-app `docker-compose.yml` |", + "cross_repo_pr": "| PRs into marola-app: `recommendation_prompt.json`, `review_prompt.json` | marola-ml `compile-prompt.yml` (by hand) | — | marola-app |\n!pushes to marola-app", + "cross_repo_push": "| pushes to marola-oods | marola-app `ingest.yml` (on a schedule) | — | marola-oods |", +} + + +def self_test() -> int: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for rel, text in FIXTURE.items(): + (root / rel).parent.mkdir(parents=True, exist_ok=True) + (root / rel).write_text(text, encoding="utf-8") + w = scan(root) + block = render(w) + (root / "REPOS.md").write_text( + f"# Repos\n\n{START}\nstale\n{END}\n\nAfter.\n", encoding="utf-8" + ) + write_block(root / "REPOS.md", block) + repos_md = (root / "REPOS.md").read_text(encoding="utf-8") + rows = block.splitlines() + cases = [ + (name, all(x[1:] not in block if x[0] == "!" else x in rows for x in want.split("\n"))) + for name, want in CASES.items() + ] + cases.append(("block_between_markers", f"{START}\n\n{block}\n{END}\n\nAfter." in repos_md)) + fails = [n for n, ok in cases if not ok] + for name, ok in cases: + print(f" {'ok ' if ok else 'FAIL'} {name}") + if fails: + print(block, file=sys.stderr) + print(f"wiring self-test: {len(fails)} failure(s)", file=sys.stderr) + return 1 + print("wiring self-test: ok") + return 0 + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + ap.add_argument( + "file", nargs="?", type=Path, help="rewrite the block between this file's markers" + ) + ap.add_argument("--root", type=Path, default=Path.cwd()) + ap.add_argument("--self-test", action="store_true") + args = ap.parse_args(argv) + if args.self_test: + return self_test() + block = render(scan(args.root)) + if args.file: + write_block(args.file, block) + else: + sys.stdout.write(block) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/self-tests.sh b/tests/self-tests.sh index 9154b9a..1039457 100755 --- a/tests/self-tests.sh +++ b/tests/self-tests.sh @@ -13,7 +13,7 @@ sh_tests=( plugins/marola-devkit/hooks/session-start.sh ) py_tests=( - scripts/cost-split.py scripts/gemini_review.py scripts/pr_label_nlp.py scripts/workflow_runners.py scripts/docs_lint.py + scripts/cost-split.py scripts/gemini_review.py scripts/pr_label_nlp.py scripts/workflow_runners.py scripts/docs_lint.py scripts/wiring.py scripts/lib/req_merge.py scripts/lib/uses_merge.py scripts/lib/mip_index_merge.py scripts/lib/tasks_issues.py plugins/marola-devkit/skills/voice-note-ingest/scripts/transcribe.py ) From f87f449254d816605bd7484011e9c97f06aaba64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Mon, 5 Oct 2026 09:03:46 +0200 Subject: [PATCH 2/7] feat(wiring): on PATH as `wiring`, a `just wiring` recipe, its tools row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PyYAML joins the flake's python, which every tool wrapper and the self-tests check run with. Refs #25 Tested: just quality line by line without nix (ruff check/format, shellcheck, actionlint, tests/self-tests.sh, agents-check, docs_lint); nix flake check not run, no nix here Cost: ~$2.40 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- devkit.just | 4 ++++ docs/4-reference_tools.md | 1 + flake.nix | 5 +++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/devkit.just b/devkit.just index 17e34ff..7c17997 100644 --- a/devkit.just +++ b/devkit.just @@ -115,6 +115,10 @@ branches-clean *args: branches-open *args: branches open {{ args }} +# The cross-repo wiring tables (MIP-0076), printed or rewritten between a file's wiring markers. +wiring *args: + wiring {{ args }} + # The invariants block in AGENTS.md against the pinned devkit's copy. agents-check *args: agents-check {{ args }} diff --git a/docs/4-reference_tools.md b/docs/4-reference_tools.md index 5d6ad2a..44325e1 100644 --- a/docs/4-reference_tools.md +++ b/docs/4-reference_tools.md @@ -57,6 +57,7 @@ extension rather than a devkit tool. | `ruleset-sync` | `ruleset-sync.sh` | `just rulesets-check`, `just rulesets-apply` | `check [owner/repo…]` diffs a repo's live branch ruleset against `.github/rulesets/main-rule.json`; `apply ` creates or updates it; `--all-org ORG` covers an org | | `agents-check` | `agents-check.sh` | `just agents-check` | Compare AGENTS.md's invariants block byte for byte with the pinned devkit's `agents/invariants.md` (`--block` or `MAROLA_INVARIANTS_BLOCK` to use another) | | `docs-lint` | `docs_lint.py` | (none) | MIP-0074 §7's stale-content check over `README.md` and `docs/**/*.md`: undefined or unmarked foreign recipes, another repo's paths, split-era wording, `docs/index.md`, links leaving the repo | +| `wiring` | `wiring.py` | `just wiring` | MIP-0076 §5.1's four tables (artifact, dispatch, pin bump, deploy), parsed from the umbrella's and every submodule's workflows, pin files, scripts, justfiles, Dockerfiles, compose files and `build.sbt`, with each devkit workflow call's ref. `wiring [FILE]` prints the block or rewrites it between FILE's `wiring:start`/`wiring:end` markers; run it in an umbrella checkout with submodules | | `workflow-runners` | `workflow_runners.py` | (none) | Fail when any workflow but the GPU publish one can reach a self-hosted runner ([runners](4-reference_runners.md)) | | `mip-resolve` | `mip-resolve.sh` | (none) | Print a MIP's doc or `.tasks.md` (`MIP-NNNN [tasks]`), or with `--path` where it was found, by the lookup in [design](1-design.md#how-tools-find-the-umbrella) | | `api-docs-push` | `api-docs-push.sh` | (none) | `api-docs-push [--branch api-docs]`: force-push a directory as one orphan commit, the `api-docs` workflow's push step | diff --git a/flake.nix b/flake.nix index b52168f..3bdba2f 100644 --- a/flake.nix +++ b/flake.nix @@ -21,8 +21,8 @@ pkgs = import nixpkgs { inherit system; }; lib = pkgs.lib; - # scripts/pr_label_nlp.py imports sklearn, so it has to be inside this python3. - python = pkgs.python3.withPackages (ps: [ ps.scikit-learn ]); + # scripts/pr_label_nlp.py imports sklearn and scripts/wiring.py yaml, so both live in this python3. + python = pkgs.python3.withPackages (ps: [ ps.scikit-learn ps.pyyaml ]); runtimeDeps = [ pkgs.bash pkgs.coreutils pkgs.findutils pkgs.gnugrep pkgs.gnused pkgs.gawk pkgs.perl pkgs.git pkgs.gh pkgs.jq pkgs.curl python @@ -55,6 +55,7 @@ temps = "scripts/temps.sh"; workflow-runners = "scripts/workflow_runners.py"; docs-lint = "scripts/docs_lint.py"; + wiring = "scripts/wiring.py"; }; # The whole tree is installed with its layout intact under share/marola-devkit: the scripts From 60463c7224e3d52ce0edd6f7fab34b3dc083d29e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Mon, 5 Oct 2026 09:15:48 +0200 Subject: [PATCH 3/7] fix(wiring): read each reusable-workflow call at its own ref; fail on missing submodules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 1 on #25. A call is now resolved with `git show :.github/workflows/` in the devkit checkout (`--devkit DIR`, default `.devkit`, else this repo), so a caller still on an old tag shows that tag's dispatch type. An unknown ref falls back to the tree with a warning and "(resolved at )" in the row; a callee missing at its ref is a warning, not a silent drop. An uninitialised submodule is an error instead of a near-empty block. The umbrella is named by `--name` (default `marola`), not the enclosing git config. Smaller: `event_type=` is matched after substitution; `-push.sh` on a `--self-test` line is ignored; a branch reader needs a `git … fetch` line; the self-exclusion is only the devkit's `scripts/wiring.py`; more than three push paths render as "touching N paths"; `git add` flags are not PR files; malformed YAML names its file; the README lists `wiring`. Closes #25 Tested: wiring --self-test (16 cases; the new ref and submodule cases fail with their fix stubbed out); ruff check/format, actionlint, shellcheck, tests/self-tests.sh, agents-check, docs_lint; a run over the fresh umbrella clone with --devkit at a tagged devkit checkout Cost: ~$8.59 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- README.md | 1 + scripts/wiring.py | 166 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 129 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 865852a..4247f59 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ claude --plugin-dir plugins/marola-devkit # the plugin, from this checkout | `issues` | Labels sync, Definition of Ready, the `agent-ready` queue, claims, `tasks-to-issues`, the board | | `agents-check` | Compare AGENTS.md's invariants block with the pinned devkit's `agents/invariants.md` | | `docs-lint` | MIP-0074's stale-content check over `README.md` and `docs/**/*.md` | +| `wiring` | MIP-0076's cross-repo wiring tables (artifacts, dispatches, pin bumps, deploys), parsed from an umbrella checkout and its submodules | | `ruleset-sync` | Check or apply `.github/rulesets/main-rule.json`'s branch ruleset to a repo, or every repo of an org | | `api-docs-push` | Force-push a directory as one orphan commit to a branch; the `api-docs` workflow's push step | | `mip-resolve` | Find a MIP or `.tasks.md` in the repo, the umbrella checkout, or via `gh api` | diff --git a/scripts/wiring.py b/scripts/wiring.py index 05c58f6..e113dfe 100755 --- a/scripts/wiring.py +++ b/scripts/wiring.py @@ -3,13 +3,17 @@ Run in an umbrella checkout with its submodules; the devkit's tree is `.devkit`, else this script's. -wiring [--root DIR] [FILE] print the block, or rewrite it between FILE's wiring markers +wiring [--root DIR] [--devkit DIR] [--name NAME] [FILE] + print the block, or rewrite it between FILE's wiring markers; a devkit git checkout with tags + lets each reusable-workflow call be read at its own @ref wiring --self-test """ from __future__ import annotations import argparse +import contextlib +import io import json import re import subprocess @@ -20,7 +24,6 @@ import yaml ORG = "marola-dev" -SELF = Path(__file__) # its fixture names every idiom, so it is never a reader PINS = ("marola-image", "corpus.version", "resources.version") START, END = "", "" READERS = ("scripts/**/*", "justfile", "Dockerfile*", "docker-compose*.yml", "build.sbt") @@ -75,21 +78,41 @@ def read(p: Path) -> str: return "" -def repos(root: Path) -> list[tuple[str, Path]]: - url = subprocess.run( - ["git", "-C", str(root), "config", "--get", "remote.origin.url"], - capture_output=True, - text=True, - check=False, - ).stdout.strip() - out = [(re.sub(r"\.git$", "", url.rsplit("/", 1)[-1]) or "marola", root)] +def warn(msg: str) -> None: + print(f"wiring: warning: {msg}", file=sys.stderr) + + +def git(d: Path, *args: str) -> str | None: + if not (d / ".git").exists(): # never the enclosing repo's answer + return None + r = subprocess.run(["git", "-C", str(d), *args], capture_output=True, text=True, check=False) + return r.stdout if r.returncode == 0 else None + + +def load(text: str, where: object) -> dict: + try: + return yaml.safe_load(text) or {} + except yaml.YAMLError as e: + raise SystemExit(f"wiring: {where}: {e}") from None + + +def version(d: Path) -> str: + if v := git(d, "describe", "--tags", "--always"): + return v.strip() + m = re.search(r"marola-devkit-([\d.]+)", str(d.resolve())) + return f"v{m[1]}" if m else "its working tree" + + +def repos(root: Path, name: str, devkit: Path | None) -> list[tuple[str, Path]]: + out = [(name, root)] for path in re.findall(r"^\s*path\s*=\s*(\S+)", read(root / ".gitmodules"), re.M): + if not (root / path / ".git").exists(): + raise SystemExit(f"wiring: {path} is not checked out (git submodule update --init)") out.append((Path(path).name, root / path)) - devkit = root / ".devkit" - out.append( - ("marola-devkit", devkit if devkit.is_dir() else Path(__file__).resolve().parents[1]) - ) - return [(n, p) for n, p in out if p.is_dir()] + if devkit is None: + devkit = root / ".devkit" + devkit = devkit if devkit.is_dir() else Path(__file__).resolve().parents[1] + return [(n, p) for n, p in out if p.is_dir()] + [("marola-devkit", devkit)] def triggers(doc: dict) -> dict: @@ -105,7 +128,8 @@ def when(on: dict) -> str: return f"on a `{push['tags'][0]}` tag" if "push" in on: paths = push.get("paths") or [] - touching = " touching " + ", ".join(f"`{p}`" for p in paths) if 0 < len(paths) <= 3 else "" + touching = " touching " + ", ".join(f"`{p}`" for p in paths) if paths else "" + touching = f" touching {len(paths)} paths" if len(paths) > 3 else touching return f"on a push to `{(push.get('branches') or ['main'])[0]}`{touching}" if "schedule" in on: return "on a schedule" @@ -121,20 +145,20 @@ def site_of(repo_dir: Path, text: str) -> str: return "?" -def effects(doc: dict, repo: str, inputs: dict, workflows: dict) -> list[tuple]: +def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: """(kind, value, via) for what a workflow sends, publishes, deploys or writes elsewhere.""" out: list[tuple] = [] text = yaml.safe_dump(doc) for job in (doc.get("jobs") or {}).values(): env = {**(doc.get("env") or {}), **(job.get("env") or {})} if m := USES.match(str(job.get("uses", ""))): - target = workflows.get((m[1], m[2])) + target, note = callee(m[1], m[2], m[3]) if target: defaults = triggers(target).get("workflow_call", {}).get("inputs") or {} given = {k: v.get("default", "") for k, v in defaults.items()} given |= {k: sub(v, env, inputs) for k, v in (job.get("with") or {}).items()} - via = f"`{m[2]}@{m[3]}`" - out += [(k, v, via) for k, v, _ in effects(target, repo, given, workflows)] + via = f"`{m[2]}@{m[3]}`{note}" + out += [(k, v, via) for k, v, _ in effects(target, repo, given, callee)] out.append(("ref", (m[2], m[3]), "")) continue checkouts: dict[str, tuple[str, str]] = {} @@ -154,20 +178,26 @@ def effects(doc: dict, repo: str, inputs: dict, workflows: dict) -> list[tuple]: if not run: continue assigned = dict(ASSIGN.findall(run)) - out += [("send", t, "") for t in re.findall(r"event_type=([\w-]+)", run)] + sent = re.findall(r"event_type=([^\s\"']*)", sub(run, senv, inputs)) + out += [("send", t, "") for t in sent if t and "$" not in t] if "/dispatches" in run: out += [("send", sub(v, {}, inputs), "") for k, v in senv.items() if "EVENT" in k] for asset in RELEASE.findall(run): name = sub(asset.strip("\"'"), assigned, {}).rsplit("/", 1)[-1] out.append(("asset", SHVAR.sub("", name), "")) - if branches := re.findall(r"([\w-]+)-push\.sh\b", run): + pushing = [ln for ln in run.splitlines() if "--self-test" not in ln] + if branches := re.findall(r"([\w-]+)-push\.sh\b", "\n".join(pushing)): for b in branches: owner = next((r for r, ref in checkouts.values() if ref == b), repo) out.append(("branch", (b, owner), "")) continue if m := re.search(r"gh pr create\b.*?--repo\s+(\S+)", run, re.S): files = sorted( - {Path(f).name for f in " ".join(re.findall(r"git add\s+(.+)", run)).split()} + { + Path(f).name + for f in " ".join(re.findall(r"git add\s+(.+)", run)).split() + if not f.startswith("-") and f != "." + } ) out.append( ("pr", (sub(m[1].strip("\"'"), senv, inputs).rsplit("/", 1)[-1], files), "") @@ -179,12 +209,31 @@ def effects(doc: dict, repo: str, inputs: dict, workflows: dict) -> list[tuple]: return [(k, v, via) for k, v, via in out if k not in ("pr", "push") or v[0] != repo] -def scan(root: Path) -> Wiring: - w, rs = Wiring(), repos(root) +def scan(root: Path, name: str = "marola", devkit: Path | None = None) -> Wiring: + w, rs = Wiring(), repos(root, name, devkit) + dirs = dict(rs) docs: dict[tuple[str, str], dict] = {} - for name, d in rs: + for repo, d in rs: for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): - docs[(name, f.name)] = yaml.safe_load(read(f)) or {} + docs[(repo, f.name)] = load(read(f), f) + cache: dict[tuple[str, str, str], tuple[dict | None, str]] = {} + + def callee(repo: str, file: str, ref: str) -> tuple[dict | None, str]: + key, d = (repo, file, ref), dirs.get(repo) + if key in cache or d is None: + return cache.get(key, (None, "")) + if git(d, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}"): + text = git(d, "show", f"{ref}:.github/workflows/{file}") + cache[key] = (load(text, f"{repo} {ref}:{file}"), "") if text else (None, "") + elif (repo, file) in docs: + warn(f"{repo} has no ref {ref} here; reading {file} at {version(d)}") + cache[key] = (docs[(repo, file)], f" (resolved at {version(d)})") + else: + cache[key] = (None, "") + if cache[key][0] is None: + warn(f"{repo}/{file}@{ref} not found; its effects are not shown") + return cache[key] + for (repo, file), doc in docs.items(): on = triggers(doc) if set(on) == {"workflow_call"}: @@ -195,7 +244,7 @@ def scan(root: Path) -> Wiring: for pin in set(PINS + ("flake.lock",)) & set(on.get("push", {}).get("paths") or []): add(w.bumps.setdefault(f"{repo} `{pin}`", []), label) repo_dir = dict(rs)[repo] - for kind, v, via in effects(doc, repo, {}, docs): + for kind, v, via in effects(doc, repo, {}, callee): parts = [p for p in (via, when(on)) if p] pub = label + (f" ({', '.join(parts)})" if parts else "") if kind == "send": @@ -210,19 +259,21 @@ def scan(root: Path) -> Wiring: add(w.art(key, owner, b)["pub"], pub) elif kind == "pr": a = w.art(f"PRs into {v[0]}: " + ", ".join(f"`{f}`" for f in v[1]), v[0]) - add(a["pub"], pub), add(a["read"], v[0]) + add(a["pub"], pub) + add(a["read"], v[0]) elif kind == "push": a = w.art(f"pushes to {v}", v) - add(a["pub"], pub), add(a["read"], v) + add(a["pub"], pub) + add(a["read"], v) elif kind == "deploy": w.deploys.append((label, site_of(repo_dir, v))) elif kind == "ref": add(w.calls.setdefault(v[0], {}).setdefault(v[1], []), repo) for repo, d in rs: - files = sorted( - {f for g in READERS for f in d.glob(g) if f.is_file() and f.name != SELF.name} - ) + files = sorted({f for g in READERS for f in d.glob(g) if f.is_file()}) texts = {f.relative_to(d).as_posix(): lines(read(f)) for f in files} + if repo == "marola-devkit": + texts.pop("scripts/wiring.py", None) # its fixture names every idiom for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): texts[f".github/workflows/{f.name}"] = lines(read(f)) for rel, text in texts.items(): @@ -235,7 +286,7 @@ def scan(root: Path) -> Wiring: continue names = {b} | {v for v, x in ASSIGN.findall(text) if x == b} if any( - "fetch" in ln + re.search(r"\bgit\b[^\n]*\bfetch\b", ln) and any(re.search(rf"(? None: "marola-site/.github/workflows/site.yml": 'on:\n push: {paths: [marola-image]}\n repository_dispatch: {types: [site-data-updated]}\njobs:\n b:\n steps:\n - run: git fetch --depth=1 origin site-data\n - run: echo "marola.dev" > site/dist/CNAME\n - uses: actions/deploy-pages@v5\n', "marola-site/marola-image": "ghcr.io/marola-dev/marola-app:jvm-8a29976@sha256:00\n", "marola-site/scripts/board-schema.sh": '# marola-image is read here\nref="$(cat "$root/marola-image")"\n', + "marola-corpus/.github/workflows/api-docs.yml": "on: {push: {branches: [main]}}\njobs:\n a:\n uses: marola-dev/marola-devkit/.github/workflows/api-docs.yml@v9.9.9\n", + "marola-site/.github/workflows/gemini.yml": "on: {pull_request: {}}\njobs:\n g:\n uses: marola-dev/marola-devkit/.github/workflows/gemini-review.yml@v0.5.0\n", "marola-corpus/.github/workflows/release.yml": 'on: {push: {tags: [\'v*\']}}\njobs:\n t:\n steps:\n - run: |\n file=".tmp/marola-corpus-$TAG.tar.gz"\n gh release upload "$TAG" "$file"\n', "marola-app/.github/workflows/docker.yml": "on: {push: {branches: [main], paths: [corpus.version]}}\nenv: {IMAGE: 'ghcr.io/${{ github.repository_owner }}/marola-app'}\njobs:\n jvm:\n steps:\n - uses: docker/build-push-action@v7\n", "marola-app/.github/workflows/api-docs.yml": "on: {push: {branches: [main]}}\njobs:\n a:\n uses: marola-dev/marola-devkit/.github/workflows/api-docs.yml@v0.4.1\n with: {devkit-ref: v0.4.1}\n", @@ -350,17 +403,20 @@ def write_block(path: Path, block: str) -> None: "marola-ml/scripts/corpus-fetch.sh": 'pin="$(<"$root/corpus.version")"\nbase="https://github.com/marola-dev/marola-corpus/releases/download"\n', } +API_ROW = "| `api-docs` branch | marola-app `api-docs.yml` (`api-docs.yml@v0.4.1`, on a push to `main`), marola-corpus `api-docs.yml` (`api-docs.yml@v9.9.9` (resolved at v0.6.0), on a push to `main`) | — | marola `scripts/fetch-api-docs.sh` |" # Each case is rows the fixture's block must hold; a "!" line is text it must not hold. CASES = { "dispatch_types_listened": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |\n| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |", "dispatch_send_step": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |", - "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", + "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app), v9.9.9 (marola-corpus); `gemini-review.yml` v0.5.0 (marola-site); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", + "reusable_workflow_resolved_at_caller_ref": "| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |\n| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n" + + API_ROW, "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |", "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | marola-site `marola-image` | marola-site `scripts/board-schema.sh` |", "deploy_pages_site": "| marola `docs.yml` | docs.marola.dev |\n| marola-site `site.yml` | marola.dev |", "pin_file_and_reader": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |\n| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", - "branch_artifact_api_docs": "| `api-docs` branch | marola-app `api-docs.yml` (`api-docs.yml@v0.4.1`, on a push to `main`) | — | marola `scripts/fetch-api-docs.sh` |", + "branch_artifact_api_docs": API_ROW, "branch_artifact_site_data": "| `site-data` branch of marola-site | marola-app `ci.yml` (on a push to `main`) | — | marola-site `site.yml` |\n!pushes to marola-site", "compose_image_reader": "| `ghcr.io/marola-dev/marola-ml` image | marola-ml `docker-local.yml` (on a push to `main`) | — | marola-app `docker-compose.yml` |", "cross_repo_pr": "| PRs into marola-app: `recommendation_prompt.json`, `review_prompt.json` | marola-ml `compile-prompt.yml` (by hand) | — | marola-app |\n!pushes to marola-app", @@ -368,14 +424,44 @@ def write_block(path: Path, block: str) -> None: } +def _devkit_tags(dk: Path) -> None: + def g(*args: str) -> None: + cfg = ["-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false"] + subprocess.run(["git", "-C", str(dk), *cfg, *args], check=True, capture_output=True) + + g("init", "-q") + g("add", "-A") + g("commit", "-qm", "v0.3.1", "--no-verify") + g("tag", "v0.3.1") + g("tag", "v0.4.1") + nu = dk / ".github/workflows/notify-umbrella.yml" + nu.write_text( + nu.read_text().replace("default: submodule-docs-updated", "default: submodule-updated") + ) + g("commit", "-qam", "v0.6.0", "--no-verify") + g("tag", "v0.6.0") + + def self_test() -> int: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) for rel, text in FIXTURE.items(): (root / rel).parent.mkdir(parents=True, exist_ok=True) (root / rel).write_text(text, encoding="utf-8") - w = scan(root) + for path in re.findall(r"path = (\S+)", FIXTURE[".gitmodules"]): + (root / path / ".git").write_text("gitdir: unused\n", encoding="utf-8") + _devkit_tags(root / ".devkit") + err = io.StringIO() + with contextlib.redirect_stderr(err): + w = scan(root) block = render(w) + (root / "marola-x").mkdir() + (root / ".gitmodules").write_text("[submodule]\n\tpath = marola-x\n", encoding="utf-8") + try: + scan(root) + uninit = "" + except SystemExit as e: + uninit = str(e) (root / "REPOS.md").write_text( f"# Repos\n\n{START}\nstale\n{END}\n\nAfter.\n", encoding="utf-8" ) @@ -386,6 +472,8 @@ def self_test() -> int: (name, all(x[1:] not in block if x[0] == "!" else x in rows for x in want.split("\n"))) for name, want in CASES.items() ] + cases.append(("missing_callee_warns", "gemini-review.yml@v0.5.0 not found" in err.getvalue())) + cases.append(("uninitialised_submodule_fails", "marola-x is not checked out" in uninit)) cases.append(("block_between_markers", f"{START}\n\n{block}\n{END}\n\nAfter." in repos_md)) fails = [n for n, ok in cases if not ok] for name, ok in cases: @@ -406,11 +494,13 @@ def main(argv: list[str] | None = None) -> int: "file", nargs="?", type=Path, help="rewrite the block between this file's markers" ) ap.add_argument("--root", type=Path, default=Path.cwd()) + ap.add_argument("--devkit", type=Path, help="default: /.devkit, else this script's repo") + ap.add_argument("--name", default="marola", help="the umbrella's repo name") ap.add_argument("--self-test", action="store_true") args = ap.parse_args(argv) if args.self_test: return self_test() - block = render(scan(args.root)) + block = render(scan(args.root, args.name, args.devkit)) if args.file: write_block(args.file, block) else: From 34bd04dcf9cf31be721c78c1deb0558dd85de7fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Fri, 9 Oct 2026 23:20:54 +0200 Subject: [PATCH 4/7] refactor(wiring): this PR is the workflow side only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit marola-devkit#25 is split in two (its row's own ~400-line clause): this PR keeps dispatches, reusable-workflow calls, release assets, images, pin bumps, deploys, the devkit-pin row, rendering and the markers, with callees read from the devkit's tree. Pin readers, branch artifacts, cross-repo PRs and pushes, and reading each callee at its own ref move to the stacked mip-0076/1-wiring-readers, which restores them unchanged. Tested: wiring --self-test (10 cases, pin_bump_on_push_paths new); ruff check/format; docs_lint; a run over a fresh --recurse-submodules umbrella clone Cost: ~$9.33 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- docs/4-reference_tools.md | 2 +- scripts/wiring.py | 182 +++----------------------------------- 2 files changed, 15 insertions(+), 169 deletions(-) diff --git a/docs/4-reference_tools.md b/docs/4-reference_tools.md index 44325e1..df9597a 100644 --- a/docs/4-reference_tools.md +++ b/docs/4-reference_tools.md @@ -57,7 +57,7 @@ extension rather than a devkit tool. | `ruleset-sync` | `ruleset-sync.sh` | `just rulesets-check`, `just rulesets-apply` | `check [owner/repo…]` diffs a repo's live branch ruleset against `.github/rulesets/main-rule.json`; `apply ` creates or updates it; `--all-org ORG` covers an org | | `agents-check` | `agents-check.sh` | `just agents-check` | Compare AGENTS.md's invariants block byte for byte with the pinned devkit's `agents/invariants.md` (`--block` or `MAROLA_INVARIANTS_BLOCK` to use another) | | `docs-lint` | `docs_lint.py` | (none) | MIP-0074 §7's stale-content check over `README.md` and `docs/**/*.md`: undefined or unmarked foreign recipes, another repo's paths, split-era wording, `docs/index.md`, links leaving the repo | -| `wiring` | `wiring.py` | `just wiring` | MIP-0076 §5.1's four tables (artifact, dispatch, pin bump, deploy), parsed from the umbrella's and every submodule's workflows, pin files, scripts, justfiles, Dockerfiles, compose files and `build.sbt`, with each devkit workflow call's ref. `wiring [FILE]` prints the block or rewrites it between FILE's `wiring:start`/`wiring:end` markers; run it in an umbrella checkout with submodules | +| `wiring` | `wiring.py` | `just wiring` | MIP-0076 §5.1's four tables (artifact, dispatch, pin bump, deploy), parsed from the umbrella's and every submodule's workflows (pin readers, branch artifacts and cross-repo writes are not read yet), with each devkit workflow call's ref. `wiring [FILE]` prints the block or rewrites it between FILE's `wiring:start`/`wiring:end` markers; run it in an umbrella checkout with submodules | | `workflow-runners` | `workflow_runners.py` | (none) | Fail when any workflow but the GPU publish one can reach a self-hosted runner ([runners](4-reference_runners.md)) | | `mip-resolve` | `mip-resolve.sh` | (none) | Print a MIP's doc or `.tasks.md` (`MIP-NNNN [tasks]`), or with `--path` where it was found, by the lookup in [design](1-design.md#how-tools-find-the-umbrella) | | `api-docs-push` | `api-docs-push.sh` | (none) | `api-docs-push [--branch api-docs]`: force-push a directory as one orphan commit, the `api-docs` workflow's push step | diff --git a/scripts/wiring.py b/scripts/wiring.py index e113dfe..4e70236 100755 --- a/scripts/wiring.py +++ b/scripts/wiring.py @@ -4,8 +4,7 @@ Run in an umbrella checkout with its submodules; the devkit's tree is `.devkit`, else this script's. wiring [--root DIR] [--devkit DIR] [--name NAME] [FILE] - print the block, or rewrite it between FILE's wiring markers; a devkit git checkout with tags - lets each reusable-workflow call be read at its own @ref + print the block, or rewrite it between FILE's wiring markers wiring --self-test """ @@ -16,7 +15,6 @@ import io import json import re -import subprocess import sys import tempfile from pathlib import Path @@ -26,7 +24,6 @@ ORG = "marola-dev" PINS = ("marola-image", "corpus.version", "resources.version") START, END = "", "" -READERS = ("scripts/**/*", "justfile", "Dockerfile*", "docker-compose*.yml", "build.sbt") USES = re.compile(rf"^{ORG}/([\w.-]+)/\.github/workflows/([\w.-]+)@([\w.-]+)$") IMAGE = re.compile(rf"ghcr\.io/{ORG}/[\w.-]+") RELEASE = re.compile(r"gh release (?:upload|create)\s+\S+\s+(\S+)") @@ -44,11 +41,8 @@ def __init__(self) -> None: self.devkit: dict[str, list[str]] = {} self.calls: dict[str, dict[str, list[str]]] = {} - def art(self, key: str, repo: str = "", match: str = "") -> dict: - a = self.artifacts.setdefault(key, {"pub": [], "pin": [], "read": [], "repos": set()}) - a["repos"].add(repo) - a.setdefault("match", match) - return a + def art(self, key: str) -> dict: + return self.artifacts.setdefault(key, {"pub": [], "pin": [], "read": []}) def add(xs: list[str], x: str) -> None: @@ -67,10 +61,6 @@ def expr(m: re.Match) -> str: return SHVAR.sub(lambda m: str(env.get(m[1], m[0])), s) -def lines(text: str) -> str: - return "\n".join(x for x in text.splitlines() if not x.lstrip().startswith(("#", "//"))) - - def read(p: Path) -> str: try: return p.read_text(encoding="utf-8") @@ -82,13 +72,6 @@ def warn(msg: str) -> None: print(f"wiring: warning: {msg}", file=sys.stderr) -def git(d: Path, *args: str) -> str | None: - if not (d / ".git").exists(): # never the enclosing repo's answer - return None - r = subprocess.run(["git", "-C", str(d), *args], capture_output=True, text=True, check=False) - return r.stdout if r.returncode == 0 else None - - def load(text: str, where: object) -> dict: try: return yaml.safe_load(text) or {} @@ -96,13 +79,6 @@ def load(text: str, where: object) -> dict: raise SystemExit(f"wiring: {where}: {e}") from None -def version(d: Path) -> str: - if v := git(d, "describe", "--tags", "--always"): - return v.strip() - m = re.search(r"marola-devkit-([\d.]+)", str(d.resolve())) - return f"v{m[1]}" if m else "its working tree" - - def repos(root: Path, name: str, devkit: Path | None) -> list[tuple[str, Path]]: out = [(name, root)] for path in re.findall(r"^\s*path\s*=\s*(\S+)", read(root / ".gitmodules"), re.M): @@ -146,7 +122,7 @@ def site_of(repo_dir: Path, text: str) -> str: def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: - """(kind, value, via) for what a workflow sends, publishes, deploys or writes elsewhere.""" + """(kind, value, via) for what a workflow sends, publishes or deploys.""" out: list[tuple] = [] text = yaml.safe_dump(doc) for job in (doc.get("jobs") or {}).values(): @@ -161,13 +137,9 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: out += [(k, v, via) for k, v, _ in effects(target, repo, given, callee)] out.append(("ref", (m[2], m[3]), "")) continue - checkouts: dict[str, tuple[str, str]] = {} for step in job.get("steps") or []: senv = {**env, **(step.get("env") or {})} - uses, w = str(step.get("uses", "")), step.get("with") or {} - if uses.startswith("actions/checkout") and w.get("repository"): - name = sub(w["repository"], senv, inputs).rsplit("/", 1)[-1] - checkouts[str(w.get("path", "."))] = (name, str(w.get("ref", ""))) + uses = str(step.get("uses", "")) if uses.startswith("docker/build-push-action") and ( m := IMAGE.search(sub(env.get("IMAGE", ""), senv, inputs)) ): @@ -185,54 +157,20 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: for asset in RELEASE.findall(run): name = sub(asset.strip("\"'"), assigned, {}).rsplit("/", 1)[-1] out.append(("asset", SHVAR.sub("", name), "")) - pushing = [ln for ln in run.splitlines() if "--self-test" not in ln] - if branches := re.findall(r"([\w-]+)-push\.sh\b", "\n".join(pushing)): - for b in branches: - owner = next((r for r, ref in checkouts.values() if ref == b), repo) - out.append(("branch", (b, owner), "")) - continue - if m := re.search(r"gh pr create\b.*?--repo\s+(\S+)", run, re.S): - files = sorted( - { - Path(f).name - for f in " ".join(re.findall(r"git add\s+(.+)", run)).split() - if not f.startswith("-") and f != "." - } - ) - out.append( - ("pr", (sub(m[1].strip("\"'"), senv, inputs).rsplit("/", 1)[-1], files), "") - ) - elif m := re.search(r"(?:git -C (\S+) |cd (\S+)[^\n]*\n(?:.*\n)*?\s*git )push\b", run): - target = checkouts.get((m[1] or m[2]).strip("\"'")) - if target: - out.append(("push", target[0], "")) - return [(k, v, via) for k, v, via in out if k not in ("pr", "push") or v[0] != repo] + return out def scan(root: Path, name: str = "marola", devkit: Path | None = None) -> Wiring: w, rs = Wiring(), repos(root, name, devkit) - dirs = dict(rs) docs: dict[tuple[str, str], dict] = {} for repo, d in rs: for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): docs[(repo, f.name)] = load(read(f), f) - cache: dict[tuple[str, str, str], tuple[dict | None, str]] = {} def callee(repo: str, file: str, ref: str) -> tuple[dict | None, str]: - key, d = (repo, file, ref), dirs.get(repo) - if key in cache or d is None: - return cache.get(key, (None, "")) - if git(d, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}"): - text = git(d, "show", f"{ref}:.github/workflows/{file}") - cache[key] = (load(text, f"{repo} {ref}:{file}"), "") if text else (None, "") - elif (repo, file) in docs: - warn(f"{repo} has no ref {ref} here; reading {file} at {version(d)}") - cache[key] = (docs[(repo, file)], f" (resolved at {version(d)})") - else: - cache[key] = (None, "") - if cache[key][0] is None: + if (repo, file) not in docs: warn(f"{repo}/{file}@{ref} not found; its effects are not shown") - return cache[key] + return docs.get((repo, file)), "" for (repo, file), doc in docs.items(): on = triggers(doc) @@ -250,66 +188,14 @@ def callee(repo: str, file: str, ref: str) -> tuple[dict | None, str]: if kind == "send": add(w.dispatch.setdefault(v, {"sent": [], "listen": []})["sent"], pub) elif kind == "image": - add(w.art(f"`{v}` image", repo, v)["pub"], pub) + add(w.art(f"`{v}` image")["pub"], pub) elif kind == "asset": - add(w.art(f"`{v}` release asset", repo)["pub"], pub) - elif kind == "branch": - b, owner = v - key = f"`{b}` branch" + (f" of {owner}" if owner != repo else "") - add(w.art(key, owner, b)["pub"], pub) - elif kind == "pr": - a = w.art(f"PRs into {v[0]}: " + ", ".join(f"`{f}`" for f in v[1]), v[0]) - add(a["pub"], pub) - add(a["read"], v[0]) - elif kind == "push": - a = w.art(f"pushes to {v}", v) - add(a["pub"], pub) - add(a["read"], v) + add(w.art(f"`{v}` release asset")["pub"], pub) elif kind == "deploy": w.deploys.append((label, site_of(repo_dir, v))) elif kind == "ref": add(w.calls.setdefault(v[0], {}).setdefault(v[1], []), repo) for repo, d in rs: - files = sorted({f for g in READERS for f in d.glob(g) if f.is_file()}) - texts = {f.relative_to(d).as_posix(): lines(read(f)) for f in files} - if repo == "marola-devkit": - texts.pop("scripts/wiring.py", None) # its fixture names every idiom - for f in sorted((d / ".github" / "workflows").glob("*.y*ml")): - texts[f".github/workflows/{f.name}"] = lines(read(f)) - for rel, text in texts.items(): - for img in re.findall(r"(?:image:|FROM)\s+(" + IMAGE.pattern + ")", text): - if (key := f"`{img}` image") in w.artifacts: - add(w.artifacts[key]["read"], f"{repo} `{Path(rel).name}`") - for key, a in w.artifacts.items(): - b = a["match"] - if " branch" not in key or rel.endswith(f"{b}-push.sh"): - continue - names = {b} | {v for v, x in ASSIGN.findall(text) if x == b} - if any( - re.search(r"\bgit\b[^\n]*\bfetch\b", ln) - and any(re.search(rf"(? str: return ", ".join(xs) or "—" out = ["| Artifact | Published by | Pinned in | Read by |", "|---|---|---|---|"] - kinds = (" image", "release asset", " branch", "PRs into", "pushes to") + kinds = (" image", "release asset") for key, a in sorted( w.artifacts.items(), key=lambda kv: ([k in kv[0] for k in kinds].index(True), kv[0]) ): @@ -370,15 +256,12 @@ def write_block(path: Path, block: str) -> None: for r in ("marola-app", "marola-site", "marola-corpus", "marola-ml") ), "mkdocs/mkdocs.yml": "site_name: x\nsite_url: https://docs.marola.dev/\n", - "scripts/fetch-api-docs.sh": 'API_DOCS_BRANCH="api-docs"\ngit -C "$tmp" fetch -q "$url" "$API_DOCS_BRANCH"\n', ".github/workflows/pointer-sync.yml": "on:\n repository_dispatch:\n types: [submodule-updated, submodule-docs-updated]\njobs: {}\n", ".github/workflows/docs.yml": "on: {push: {paths: [flake.lock]}}\njobs:\n d:\n steps:\n - uses: actions/deploy-pages@v5\n", ".devkit/.github/workflows/notify-umbrella.yml": "on:\n workflow_call:\n inputs:\n event-type: {type: string, default: submodule-docs-updated}\njobs:\n dispatch:\n steps:\n - env: {EVENT_TYPE: '${{ inputs.event-type }}'}\n run: curl https://api.github.com/repos/$UMBRELLA/dispatches -d x\n", ".devkit/.github/workflows/api-docs.yml": "on: {workflow_call: {inputs: {devkit-ref: {type: string}}}}\njobs:\n publish:\n steps:\n - run: bash .devkit-checkout/scripts/api-docs-push.sh out url sha\n", "marola-site/.github/workflows/notify-umbrella.yml": "on: {push: {branches: [main], paths: [README.md, 'docs/**']}}\njobs:\n n:\n uses: marola-dev/marola-devkit/.github/workflows/notify-umbrella.yml@v0.3.1\n", "marola-site/.github/workflows/site.yml": 'on:\n push: {paths: [marola-image]}\n repository_dispatch: {types: [site-data-updated]}\njobs:\n b:\n steps:\n - run: git fetch --depth=1 origin site-data\n - run: echo "marola.dev" > site/dist/CNAME\n - uses: actions/deploy-pages@v5\n', - "marola-site/marola-image": "ghcr.io/marola-dev/marola-app:jvm-8a29976@sha256:00\n", - "marola-site/scripts/board-schema.sh": '# marola-image is read here\nref="$(cat "$root/marola-image")"\n', "marola-corpus/.github/workflows/api-docs.yml": "on: {push: {branches: [main]}}\njobs:\n a:\n uses: marola-dev/marola-devkit/.github/workflows/api-docs.yml@v9.9.9\n", "marola-site/.github/workflows/gemini.yml": "on: {pull_request: {}}\njobs:\n g:\n uses: marola-dev/marola-devkit/.github/workflows/gemini-review.yml@v0.5.0\n", "marola-corpus/.github/workflows/release.yml": 'on: {push: {tags: [\'v*\']}}\njobs:\n t:\n steps:\n - run: |\n file=".tmp/marola-corpus-$TAG.tar.gz"\n gh release upload "$TAG" "$file"\n', @@ -388,60 +271,24 @@ def write_block(path: Path, block: str) -> None: "marola-app/.github/workflows/ci.yml": "on: {push: {branches: [main]}}\njobs:\n c:\n steps:\n" + CO % ("marola-dev/marola-site", "site-data", ", ref: site-data") + " - run: |\n git -C site-data commit -m x\n scripts/site-data-push.sh site-data\n - run: gh api repos/marola-dev/marola-site/dispatches -f event_type=site-data-updated\n", - "marola-app/.github/workflows/ingest.yml": "on: {schedule: [{cron: '0 4 * * *'}]}\njobs:\n i:\n steps:\n" - + CO % ("marola-dev/marola-oods", "oods", "") - + " - run: |\n git -C oods commit -qm ingest\n git -C oods push\n", - "marola-app/docker-compose.yml": "services:\n ollama-local:\n image: ghcr.io/marola-dev/marola-ml:local\n", "marola-app/flake.lock": json.dumps( {"nodes": {"marola-devkit": {"original": {"repo": "marola-devkit", "ref": "v0.4.1"}}}} ), "marola-ml/.github/workflows/docker-local.yml": "on: {push: {branches: [main]}}\nenv: {IMAGE: ghcr.io/marola-dev/marola-ml}\njobs:\n b:\n steps:\n - uses: docker/build-push-action@v7\n", - "marola-ml/.github/workflows/compile-prompt.yml": "on: {workflow_dispatch: {}}\nenv: {APP_REPO: marola-dev/marola-app}\njobs:\n c:\n steps:\n" - + CO % ("'${{ env.APP_REPO }}'", "app", "") - + ' - run: |\n cd app\n git add core/src/main/resources/recommendation_prompt.json core/src/main/resources/review_prompt.json\n git push -q origin "$BRANCH"\n gh pr create --repo "$APP_REPO" --base main\n', - "marola-ml/corpus.version": "v0.2.0\n", - "marola-ml/scripts/corpus-fetch.sh": 'pin="$(<"$root/corpus.version")"\nbase="https://github.com/marola-dev/marola-corpus/releases/download"\n', } -API_ROW = "| `api-docs` branch | marola-app `api-docs.yml` (`api-docs.yml@v0.4.1`, on a push to `main`), marola-corpus `api-docs.yml` (`api-docs.yml@v9.9.9` (resolved at v0.6.0), on a push to `main`) | — | marola `scripts/fetch-api-docs.sh` |" - # Each case is rows the fixture's block must hold; a "!" line is text it must not hold. CASES = { "dispatch_types_listened": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |\n| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |", "dispatch_send_step": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |", "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app), v9.9.9 (marola-corpus); `gemini-review.yml` v0.5.0 (marola-site); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", - "reusable_workflow_resolved_at_caller_ref": "| `submodule-docs-updated` | marola-site `notify-umbrella.yml` (`notify-umbrella.yml@v0.3.1`, on a push to `main` touching `README.md`, `docs/**`) | marola `pointer-sync.yml` |\n| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n" - + API_ROW, - "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |", - "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | marola-site `marola-image` | marola-site `scripts/board-schema.sh` |", + "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | — | — |", + "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | — | — |", + "pin_bump_on_push_paths": "| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", "deploy_pages_site": "| marola `docs.yml` | docs.marola.dev |\n| marola-site `site.yml` | marola.dev |", - "pin_file_and_reader": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | marola-ml `corpus.version` | marola-ml `scripts/corpus-fetch.sh` |\n| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", - "branch_artifact_api_docs": API_ROW, - "branch_artifact_site_data": "| `site-data` branch of marola-site | marola-app `ci.yml` (on a push to `main`) | — | marola-site `site.yml` |\n!pushes to marola-site", - "compose_image_reader": "| `ghcr.io/marola-dev/marola-ml` image | marola-ml `docker-local.yml` (on a push to `main`) | — | marola-app `docker-compose.yml` |", - "cross_repo_pr": "| PRs into marola-app: `recommendation_prompt.json`, `review_prompt.json` | marola-ml `compile-prompt.yml` (by hand) | — | marola-app |\n!pushes to marola-app", - "cross_repo_push": "| pushes to marola-oods | marola-app `ingest.yml` (on a schedule) | — | marola-oods |", } -def _devkit_tags(dk: Path) -> None: - def g(*args: str) -> None: - cfg = ["-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false"] - subprocess.run(["git", "-C", str(dk), *cfg, *args], check=True, capture_output=True) - - g("init", "-q") - g("add", "-A") - g("commit", "-qm", "v0.3.1", "--no-verify") - g("tag", "v0.3.1") - g("tag", "v0.4.1") - nu = dk / ".github/workflows/notify-umbrella.yml" - nu.write_text( - nu.read_text().replace("default: submodule-docs-updated", "default: submodule-updated") - ) - g("commit", "-qam", "v0.6.0", "--no-verify") - g("tag", "v0.6.0") - - def self_test() -> int: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -450,7 +297,6 @@ def self_test() -> int: (root / rel).write_text(text, encoding="utf-8") for path in re.findall(r"path = (\S+)", FIXTURE[".gitmodules"]): (root / path / ".git").write_text("gitdir: unused\n", encoding="utf-8") - _devkit_tags(root / ".devkit") err = io.StringIO() with contextlib.redirect_stderr(err): w = scan(root) From e51b936687e2e57f704f0e6c06498c05ac892bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Fri, 9 Oct 2026 23:20:54 +0200 Subject: [PATCH 5/7] ci: install PyYAML before the self-tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pr.yml's python job runs tests/self-tests.sh on a bare setup-python, so wiring.py's `import yaml` failed there (ModuleNotFoundError) while the flake's python, which has it, passed. Tested: actionlint .github/workflows/pr.yml; wiring --self-test with pyyaml 6.0.3 in a venv Cost: ~$0.74 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- .github/workflows/pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b55f65c..ad3d7e5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -14,7 +14,9 @@ jobs: if: github.event.action != 'closed' uses: ./.github/workflows/python-ci.yml with: + # wiring.py imports yaml; the flake's python has it, a bare setup-python does not. self-test-commands: | + pip install "pyyaml==6.0.3" bash tests/self-tests.sh static: From e5a8dbaafe0ff74e2a3f76500897036470fe56fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Fri, 9 Oct 2026 23:21:46 +0200 Subject: [PATCH 6/7] fix(wiring): a `gh release create` with no files publishes no asset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release regex took the token after the tag as the asset, so the umbrella's and the devkit's `gh release create "$TAG" --repo "$GITHUB_REPOSITORY" …` rendered a `--repo` release asset. The files are now the non-flag arguments after the tag, shell-split so a quoted `--title` stays whole. Tested: wiring --self-test, release_create_without_assets red before, green after; the real umbrella run loses only the `--repo` row and is byte-identical across two runs Cost: ~$1.87 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- scripts/wiring.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/wiring.py b/scripts/wiring.py index 4e70236..f90a97f 100755 --- a/scripts/wiring.py +++ b/scripts/wiring.py @@ -15,6 +15,7 @@ import io import json import re +import shlex import sys import tempfile from pathlib import Path @@ -26,7 +27,9 @@ START, END = "", "" USES = re.compile(rf"^{ORG}/([\w.-]+)/\.github/workflows/([\w.-]+)@([\w.-]+)$") IMAGE = re.compile(rf"ghcr\.io/{ORG}/[\w.-]+") -RELEASE = re.compile(r"gh release (?:upload|create)\s+\S+\s+(\S+)") +RELEASE = re.compile(r"gh release (?:upload|create)\s+\S+([^\n]*)") +# gh release flags that take a value; everything else after the tag that is not a flag is a file. +VALUED = {"--repo", "-R", "--title", "-t", "--notes", "-n", "--notes-file", "-F", "--target"} ASSIGN = re.compile(r"""^\s*(\w+)=["']?([^"'\s]+)""", re.M) SHVAR = re.compile(r"\$\{?(\w+)\}?") EXPR = re.compile(r"\$\{\{\s*(env|inputs|github)\.([\w-]+)\s*\}\}") @@ -68,6 +71,23 @@ def read(p: Path) -> str: return "" +def release_files(run: str) -> list[str]: + out: list[str] = [] + for rest in RELEASE.findall(run): + try: + toks = shlex.split(rest.rstrip(" \\"), comments=True) + except ValueError: + toks = rest.split() + skip = False + for tok in toks: + if tok in ("&&", "||", "|", ";"): + break + if not skip and not tok.startswith("-"): + out.append(tok) + skip = not skip and tok in VALUED + return out + + def warn(msg: str) -> None: print(f"wiring: warning: {msg}", file=sys.stderr) @@ -154,8 +174,8 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: out += [("send", t, "") for t in sent if t and "$" not in t] if "/dispatches" in run: out += [("send", sub(v, {}, inputs), "") for k, v in senv.items() if "EVENT" in k] - for asset in RELEASE.findall(run): - name = sub(asset.strip("\"'"), assigned, {}).rsplit("/", 1)[-1] + for asset in release_files(run): + name = sub(asset, assigned, {}).rsplit("/", 1)[-1] out.append(("asset", SHVAR.sub("", name), "")) return out @@ -257,6 +277,7 @@ def write_block(path: Path, block: str) -> None: ), "mkdocs/mkdocs.yml": "site_name: x\nsite_url: https://docs.marola.dev/\n", ".github/workflows/pointer-sync.yml": "on:\n repository_dispatch:\n types: [submodule-updated, submodule-docs-updated]\njobs: {}\n", + ".github/workflows/release.yml": 'on: {push: {tags: [v*]}}\njobs:\n r:\n steps:\n - run: gh release create "$TAG" --repo "$GITHUB_REPOSITORY" --verify-tag --title "x $TAG"\n', ".github/workflows/docs.yml": "on: {push: {paths: [flake.lock]}}\njobs:\n d:\n steps:\n - uses: actions/deploy-pages@v5\n", ".devkit/.github/workflows/notify-umbrella.yml": "on:\n workflow_call:\n inputs:\n event-type: {type: string, default: submodule-docs-updated}\njobs:\n dispatch:\n steps:\n - env: {EVENT_TYPE: '${{ inputs.event-type }}'}\n run: curl https://api.github.com/repos/$UMBRELLA/dispatches -d x\n", ".devkit/.github/workflows/api-docs.yml": "on: {workflow_call: {inputs: {devkit-ref: {type: string}}}}\njobs:\n publish:\n steps:\n - run: bash .devkit-checkout/scripts/api-docs-push.sh out url sha\n", @@ -283,6 +304,7 @@ def write_block(path: Path, block: str) -> None: "dispatch_send_step": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |", "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app), v9.9.9 (marola-corpus); `gemini-review.yml` v0.5.0 (marola-site); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | — | — |", + "release_create_without_assets": "!`--repo` release asset\n!`` release asset", "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | — | — |", "pin_bump_on_push_paths": "| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", "deploy_pages_site": "| marola `docs.yml` | docs.marola.dev |\n| marola-site `site.yml` | marola.dev |", From 0489baa315c5c0f29aebb57fbd0ad589abe9a90e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Guilhermo=20de=20Barros=20Val=C3=A9rio?= Date: Fri, 9 Oct 2026 23:29:04 +0200 Subject: [PATCH 7/7] fix(wiring): release files, other repos' reusable workflows, unresolved event types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From review: `gh release` arguments are now shell-tokenised across `\` continuations, the tag is the first non-flag argument (so a flag before it no longer becomes an asset), and a glued `;`, `&&` or `>` ends the file list; the devkit-tag row only counts marola-devkit's own workflows; an `EVENT*` env that stays an unresolved `${{ … }}` is not a dispatch type. Tested: wiring --self-test, release_files_shell_forms, only_devkit_calls_in_tag_row and unresolved_event_type_not_sent red before and green after, release_create_without_assets now fails if --title is dropped from VALUED; the real umbrella run is unchanged Cost: ~$1.96 · diff-size estimate (cost-split --estimate-commit) Co-Authored-By: Claude --- scripts/wiring.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/scripts/wiring.py b/scripts/wiring.py index f90a97f..6bf10f2 100755 --- a/scripts/wiring.py +++ b/scripts/wiring.py @@ -27,9 +27,10 @@ START, END = "", "" USES = re.compile(rf"^{ORG}/([\w.-]+)/\.github/workflows/([\w.-]+)@([\w.-]+)$") IMAGE = re.compile(rf"ghcr\.io/{ORG}/[\w.-]+") -RELEASE = re.compile(r"gh release (?:upload|create)\s+\S+([^\n]*)") -# gh release flags that take a value; everything else after the tag that is not a flag is a file. +RELEASE = re.compile(r"gh release (?:upload|create)\b([^\n]*)") +# gh release flags that take a value; the first other argument is the tag, the rest are files. VALUED = {"--repo", "-R", "--title", "-t", "--notes", "-n", "--notes-file", "-F", "--target"} +VALUED |= {"--discussion-category", "--notes-start-tag"} ASSIGN = re.compile(r"""^\s*(\w+)=["']?([^"'\s]+)""", re.M) SHVAR = re.compile(r"\$\{?(\w+)\}?") EXPR = re.compile(r"\$\{\{\s*(env|inputs|github)\.([\w-]+)\s*\}\}") @@ -73,18 +74,22 @@ def read(p: Path) -> str: def release_files(run: str) -> list[str]: out: list[str] = [] - for rest in RELEASE.findall(run): + for rest in RELEASE.findall(run.replace("\\\n", " ")): + lex = shlex.shlex(rest, posix=True, punctuation_chars=True) + lex.whitespace_split = True try: - toks = shlex.split(rest.rstrip(" \\"), comments=True) + toks = list(lex) except ValueError: toks = rest.split() + args: list[str] = [] skip = False for tok in toks: - if tok in ("&&", "||", "|", ";"): + if set(tok) <= set(lex.punctuation_chars): break if not skip and not tok.startswith("-"): - out.append(tok) + args.append(tok) skip = not skip and tok in VALUED + out += args[1:] return out @@ -155,7 +160,8 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: given |= {k: sub(v, env, inputs) for k, v in (job.get("with") or {}).items()} via = f"`{m[2]}@{m[3]}`{note}" out += [(k, v, via) for k, v, _ in effects(target, repo, given, callee)] - out.append(("ref", (m[2], m[3]), "")) + if m[1] == "marola-devkit": + out.append(("ref", (m[2], m[3]), "")) continue for step in job.get("steps") or []: senv = {**env, **(step.get("env") or {})} @@ -173,7 +179,8 @@ def effects(doc: dict, repo: str, inputs: dict, callee) -> list[tuple]: sent = re.findall(r"event_type=([^\s\"']*)", sub(run, senv, inputs)) out += [("send", t, "") for t in sent if t and "$" not in t] if "/dispatches" in run: - out += [("send", sub(v, {}, inputs), "") for k, v in senv.items() if "EVENT" in k] + evs = [sub(v, {}, inputs) for k, v in senv.items() if "EVENT" in k] + out += [("send", t, "") for t in evs if "$" not in t] for asset in release_files(run): name = sub(asset, assigned, {}).rsplit("/", 1)[-1] out.append(("asset", SHVAR.sub("", name), "")) @@ -284,6 +291,7 @@ def write_block(path: Path, block: str) -> None: "marola-site/.github/workflows/notify-umbrella.yml": "on: {push: {branches: [main], paths: [README.md, 'docs/**']}}\njobs:\n n:\n uses: marola-dev/marola-devkit/.github/workflows/notify-umbrella.yml@v0.3.1\n", "marola-site/.github/workflows/site.yml": 'on:\n push: {paths: [marola-image]}\n repository_dispatch: {types: [site-data-updated]}\njobs:\n b:\n steps:\n - run: git fetch --depth=1 origin site-data\n - run: echo "marola.dev" > site/dist/CNAME\n - uses: actions/deploy-pages@v5\n', "marola-corpus/.github/workflows/api-docs.yml": "on: {push: {branches: [main]}}\njobs:\n a:\n uses: marola-dev/marola-devkit/.github/workflows/api-docs.yml@v9.9.9\n", + "marola-site/.github/workflows/odd.yml": "on: {workflow_dispatch: {}}\njobs:\n r:\n uses: marola-dev/marola-app/.github/workflows/reusable.yml@main\n d:\n env: {EVENT_TYPE: '${{ inputs.ev }}'}\n steps:\n - run: curl https://api.github.com/repos/x/dispatches -d y\n", "marola-site/.github/workflows/gemini.yml": "on: {pull_request: {}}\njobs:\n g:\n uses: marola-dev/marola-devkit/.github/workflows/gemini-review.yml@v0.5.0\n", "marola-corpus/.github/workflows/release.yml": 'on: {push: {tags: [\'v*\']}}\njobs:\n t:\n steps:\n - run: |\n file=".tmp/marola-corpus-$TAG.tar.gz"\n gh release upload "$TAG" "$file"\n', "marola-app/.github/workflows/docker.yml": "on: {push: {branches: [main], paths: [corpus.version]}}\nenv: {IMAGE: 'ghcr.io/${{ github.repository_owner }}/marola-app'}\njobs:\n jvm:\n steps:\n - uses: docker/build-push-action@v7\n", @@ -304,7 +312,9 @@ def write_block(path: Path, block: str) -> None: "dispatch_send_step": "| `site-data-updated` | marola-app `ci.yml` (on a push to `main`) | marola-site `site.yml` |", "reusable_workflow_ref_and_event_type": "| `submodule-updated` | marola-app `notify-umbrella.yml` (`notify-umbrella.yml@v0.6.0`, on a push to `main`) | marola `pointer-sync.yml` |\n| a marola-devkit tag | marola-devkit | marola-app `flake.lock` v0.4.1 | `api-docs.yml` v0.4.1 (marola-app), v9.9.9 (marola-corpus); `gemini-review.yml` v0.5.0 (marola-site); `notify-umbrella.yml` v0.3.1 (marola-site), v0.6.0 (marola-app) |", "release_upload_assets": "| `marola-corpus-.tar.gz` release asset | marola-corpus `release.yml` (on a `v*` tag) | — | — |", - "release_create_without_assets": "!`--repo` release asset\n!`` release asset", + "release_create_without_assets": "!marola `release.yml`", + "only_devkit_calls_in_tag_row": "!reusable.yml", + "unresolved_event_type_not_sent": "!inputs.ev", "image_publish": "| `ghcr.io/marola-dev/marola-app` image | marola-app `docker.yml` (on a push to `main` touching `corpus.version`) | — | — |", "pin_bump_on_push_paths": "| marola-app `corpus.version` | marola-app `docker.yml` |\n| marola-site `marola-image` | marola-site `site.yml` |", "deploy_pages_site": "| marola `docs.yml` | docs.marola.dev |\n| marola-site `site.yml` | marola.dev |", @@ -340,6 +350,8 @@ def self_test() -> int: (name, all(x[1:] not in block if x[0] == "!" else x in rows for x in want.split("\n"))) for name, want in CASES.items() ] + shell = 'gh release upload --clobber "$TAG" "$f"; echo done\ngh release create "$TAG" --discussion-category General --notes-start-tag v1 \\\n dist/a.tgz>out.log\n' + cases.append(("release_files_shell_forms", release_files(shell) == ["$f", "dist/a.tgz"])) cases.append(("missing_callee_warns", "gemini-review.yml@v0.5.0 not found" in err.getvalue())) cases.append(("uninitialised_submodule_fails", "marola-x is not checked out" in uninit)) cases.append(("block_between_markers", f"{START}\n\n{block}\n{END}\n\nAfter." in repos_md))