|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""The rows a platform package's entry lists (`platform_rows`, mcpp SPEC-013 §14). |
| 3 | +
|
| 4 | +mcpp's target table answers its platform column from the engine's own data; for a |
| 5 | +row it has no answer for, it reads the rows that the index's platform entries list. |
| 6 | +That list is derived data: it is what the platform's own manifest serves, at the |
| 7 | +entry's newest version, and is never written by hand. |
| 8 | +
|
| 9 | + platform_rows.py write pkgs/o/openkal-platform.lua [...] derive and write the list |
| 10 | + platform_rows.py check [pkgs/*/*.lua ...] every list matches its manifest, |
| 11 | + and no row has two platforms |
| 12 | +
|
| 13 | +A row is listed when the manifest serves it: `[platform] targets`, |
| 14 | +`[[platform.define]] triple`, and `[[platform.provider]] serves`, where a pattern |
| 15 | +(`*-linux-musl`) stands for the rows it matches among the platform's own and the |
| 16 | +engine's (mcpp's builtin.toml at the release the index's CI pins, $MCPP_VERSION). |
| 17 | +`[platform.hosted] targets` are the rows its hosted arrangement applies to once the |
| 18 | +platform is chosen, host rows among them; of those, only a row the engine does not |
| 19 | +hold or holds as `planned` (`i686-linux-musl`) is listed, so the index never names |
| 20 | +a platform for a host's own row. |
| 21 | +The manifest is read from the GLOBAL archive of the newest version, at the entry's |
| 22 | +`mcpp` path; archives are kept under $PLATFORM_ROWS_CACHE (default |
| 23 | +~/.cache/mcpp-index-platform-rows), keyed by sha256. |
| 24 | +""" |
| 25 | +import fnmatch, hashlib, io, os, re, sys, tarfile, tomllib, urllib.request |
| 26 | +from pathlib import Path |
| 27 | + |
| 28 | +CACHE = Path(os.environ.get("PLATFORM_ROWS_CACHE", Path.home() / ".cache" / "mcpp-index-platform-rows")) |
| 29 | +ENGINE_ROWS = "https://raw.githubusercontent.com/mcpp-community/mcpp/v{}/modules/toolchain-model/data/builtin.toml" |
| 30 | + |
| 31 | + |
| 32 | +def mcpp_version() -> str: |
| 33 | + if v := os.environ.get("MCPP_VERSION"): |
| 34 | + return v |
| 35 | + m = re.search(r'^\s*MCPP_VERSION:\s*"([^"]+)"', Path(".github/workflows/validate.yml").read_text(), re.M) |
| 36 | + if not m: |
| 37 | + raise SystemExit("MCPP_VERSION is neither set nor found in .github/workflows/validate.yml") |
| 38 | + return m.group(1) |
| 39 | + |
| 40 | + |
| 41 | +def engine_rows() -> dict[str, str]: |
| 42 | + """The rows of mcpp's built-in table and their tiers, at the release the index pins.""" |
| 43 | + version = mcpp_version() |
| 44 | + path = CACHE / f"builtin-{version}.toml" |
| 45 | + if not path.is_file(): |
| 46 | + CACHE.mkdir(parents=True, exist_ok=True) |
| 47 | + path.write_bytes(urllib.request.urlopen(ENGINE_ROWS.format(version), timeout=60).read()) |
| 48 | + doc = tomllib.loads(path.read_text()) |
| 49 | + return {d["triple"]: d.get("tier", "") for d in doc.get("platform", {}).get("define", []) if "triple" in d} |
| 50 | +ROWS_LINE = re.compile(r'^(\s*)platform_rows\s*=\s*\{([^}]*)\},?\s*$', re.M) |
| 51 | + |
| 52 | + |
| 53 | +def lua_field(text: str, key: str) -> str | None: |
| 54 | + m = re.search(rf'^\s*{key}\s*=\s*"([^"]*)"', text, re.M) |
| 55 | + return m.group(1) if m else None |
| 56 | + |
| 57 | + |
| 58 | +def newest(text: str) -> tuple[str, str, str] | None: |
| 59 | + """(version, GLOBAL url, sha256) of the newest version under `linux`.""" |
| 60 | + block = re.search(r'\blinux\s*=\s*\{(.*?)\n\s{8}\},', text, re.S) |
| 61 | + if not block: |
| 62 | + return None |
| 63 | + best = None |
| 64 | + for m in re.finditer(r'\["(\d+(?:\.\d+)*)"\]\s*=\s*\{(.*?)sha256\s*=\s*"([0-9a-f]{64})"', block.group(1), re.S): |
| 65 | + version, body, sha = m.groups() |
| 66 | + url = re.search(r'GLOBAL\s*=\s*"([^"]+)"', body) or re.search(r'url\s*=\s*"([^"]+)"', body) |
| 67 | + if not url: |
| 68 | + continue |
| 69 | + key = tuple(int(x) for x in version.split(".")) |
| 70 | + if best is None or key > best[0]: |
| 71 | + best = (key, version, url.group(1), sha) |
| 72 | + return best[1:] if best else None |
| 73 | + |
| 74 | + |
| 75 | +def archive(url: str, sha: str) -> bytes: |
| 76 | + path = CACHE / sha |
| 77 | + if path.is_file(): |
| 78 | + return path.read_bytes() |
| 79 | + data = urllib.request.urlopen(url, timeout=120).read() |
| 80 | + if hashlib.sha256(data).hexdigest() != sha: |
| 81 | + raise SystemExit(f"{url}: sha256 differs from the entry's") |
| 82 | + CACHE.mkdir(parents=True, exist_ok=True) |
| 83 | + path.write_bytes(data) |
| 84 | + return data |
| 85 | + |
| 86 | + |
| 87 | +def toml_1_0(text: str) -> str: |
| 88 | + """An inline table written over several lines (TOML 1.1) on one line, for tomllib.""" |
| 89 | + out, depth, quote, comment = [], 0, None, False |
| 90 | + for ch in text: |
| 91 | + if comment: |
| 92 | + if ch == "\n": |
| 93 | + comment = False |
| 94 | + out.append(" " if depth else ch) |
| 95 | + continue |
| 96 | + if quote: |
| 97 | + out.append(ch) |
| 98 | + if ch == quote: |
| 99 | + quote = None |
| 100 | + continue |
| 101 | + if ch in "\"'": |
| 102 | + quote = ch |
| 103 | + elif ch == "#": |
| 104 | + comment = True |
| 105 | + continue |
| 106 | + elif ch == "{": |
| 107 | + depth += 1 |
| 108 | + elif ch == "}": |
| 109 | + depth -= 1 |
| 110 | + elif ch == "\n" and depth: |
| 111 | + ch = " " |
| 112 | + out.append(ch) |
| 113 | + return re.sub(r",\s*}", " }", "".join(out)) |
| 114 | + |
| 115 | + |
| 116 | +def manifest_rows(data: bytes, pattern: str) -> list[str]: |
| 117 | + with tarfile.open(fileobj=io.BytesIO(data)) as tar: |
| 118 | + names = [n for n in tar.getnames() if fnmatch.fnmatch(n, pattern)] |
| 119 | + if not names: |
| 120 | + raise SystemExit(f"no manifest {pattern} in the archive") |
| 121 | + text = tar.extractfile(min(names, key=len)).read().decode() |
| 122 | + doc = tomllib.loads(toml_1_0(text)) |
| 123 | + platform = doc.get("platform", {}) |
| 124 | + rows = set(platform.get("targets", [])) |
| 125 | + rows |= {d["triple"] for d in platform.get("define", []) if "triple" in d} |
| 126 | + engine = engine_rows() |
| 127 | + rows |= {r for r in platform.get("hosted", {}).get("targets", []) if engine.get(r, "planned") == "planned"} |
| 128 | + known = rows | set(engine) |
| 129 | + for p in platform.get("provider", []): |
| 130 | + for s in p.get("serves", []): |
| 131 | + rows |= {r for r in known if fnmatch.fnmatchcase(r, s)} if any(c in s for c in "*?[") else {s} |
| 132 | + return sorted(rows) |
| 133 | + |
| 134 | + |
| 135 | +def derive(path: Path) -> list[str]: |
| 136 | + text = path.read_text() |
| 137 | + found = newest(text) |
| 138 | + pattern = lua_field(text, "mcpp") |
| 139 | + if not found or not pattern: |
| 140 | + raise SystemExit(f"{path}: no version with an archive, or no `mcpp` manifest path") |
| 141 | + version, url, sha = found |
| 142 | + return manifest_rows(archive(url, sha), pattern) |
| 143 | + |
| 144 | + |
| 145 | +def written(text: str) -> list[str] | None: |
| 146 | + m = ROWS_LINE.search(text) |
| 147 | + return sorted(re.findall(r'"([^"]+)"', m.group(2))) if m else None |
| 148 | + |
| 149 | + |
| 150 | +def write(path: Path) -> None: |
| 151 | + rows = derive(path) |
| 152 | + text = path.read_text() |
| 153 | + line = " platform_rows = { " + ", ".join(f'"{r}"' for r in rows) + " }," |
| 154 | + if ROWS_LINE.search(text): |
| 155 | + text = ROWS_LINE.sub(line, text, count=1) |
| 156 | + else: |
| 157 | + text = re.sub(r'^(\s*type\s*=\s*"package",\s*)$', lambda m: m.group(1) + "\n" + line, text, count=1, flags=re.M) |
| 158 | + path.write_text(text) |
| 159 | + print(f"{path}: {len(rows)} rows") |
| 160 | + |
| 161 | + |
| 162 | +def check(paths: list[Path]) -> int: |
| 163 | + claims: dict[str, list[str]] = {} |
| 164 | + fail = 0 |
| 165 | + for path in paths: |
| 166 | + text = path.read_text() |
| 167 | + listed = written(text) |
| 168 | + if listed is None: |
| 169 | + continue |
| 170 | + name = f"{lua_field(text, 'namespace') or 'mcpp'}.{lua_field(text, 'name')}" |
| 171 | + for row in listed: |
| 172 | + claims.setdefault(row, []).append(name) |
| 173 | + derived = derive(path) |
| 174 | + if listed != derived: |
| 175 | + print(f"::error file={path}::platform_rows is {listed}; the manifest of the newest version serves {derived} " |
| 176 | + f"(tools/platform_rows.py write {path})") |
| 177 | + fail = 1 |
| 178 | + for row, who in sorted(claims.items()): |
| 179 | + if len(who) > 1: |
| 180 | + print(f"::error::{row} is listed by {', '.join(who)}: a row has one platform") |
| 181 | + fail = 1 |
| 182 | + if not fail: |
| 183 | + print(f"platform_rows: {len(claims)} rows, each with one platform and matching its manifest") |
| 184 | + return fail |
| 185 | + |
| 186 | + |
| 187 | +def main(argv: list[str]) -> int: |
| 188 | + if len(argv) < 2 or argv[1] not in ("write", "check"): |
| 189 | + print(__doc__) |
| 190 | + return 2 |
| 191 | + paths = [Path(a) for a in argv[2:]] or sorted(Path("pkgs").glob("*/*.lua")) |
| 192 | + if argv[1] == "write": |
| 193 | + for p in paths: |
| 194 | + write(p) |
| 195 | + return 0 |
| 196 | + return check(paths) |
| 197 | + |
| 198 | + |
| 199 | +if __name__ == "__main__": |
| 200 | + sys.exit(main(sys.argv)) |
0 commit comments