feat(site): ECE floor calculator, held to the Python in CI (phase 1)

.github/workflows/site.yml

@@ -0,0 +1,52 @@
+name: Site
+
+# The site at tmhsdigital.github.io/plumbline carries a JavaScript
+# reimplementation of the calibration floor. If it disagrees with the Python,
+# the site misreports the tool, which is worse than having no site. This
+# workflow holds the two together; later jobs assemble and deploy the site, and
+# they depend on this one, so a wrong calculator never ships. A stale site is
+# the better failure.
+#
+# It is deliberately separate from ci.yml and release.yml: it shares no jobs,
+# no concurrency group, and no permissions with either.
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: site-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  parity:
+    name: the site's floor agrees with the Python
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.13"
+          enable-cache: true
+
+      - name: Sync dependencies
+        run: uv sync --locked
+
+      # Without this, a change to the Python floor would leave the JavaScript
+      # agreeing with an old answer and the check below passing.
+      - name: The committed fixture is what the Python produces today
+        run: uv run python scripts/floor_golden.py --check
+
+      # The runner's own Node, with no setup action and no package install.
+      # The check uses the standard library only.
+      - name: floor.js reproduces the fixture
+        run: |
+          node --version
+          node scripts/check_floor_parity.mjs

CHANGELOG.md

@@ -11,6 +11,14 @@ different event from one that moved because it was wrong.
 
 ## Unreleased
 
+### Added
+
+- A browser calculator for the ECE floor (`site/`), a JavaScript port of
+  `synthetic_floor` that reproduces numpy's seeded random stream draw for draw.
+  `scripts/floor_golden.py` exports golden values from the Python and
+  `scripts/check_floor_parity.mjs` holds the port to them within 1e-9 in CI.
+  The page is not deployed yet.
+
 ### Fixed
 
 - The inconclusive verdict read as a pass. "Not distinguishable from a

scripts/check_floor_parity.mjs

@@ -0,0 +1,59 @@
+// Holds site/floor.js to the golden values the Python exported.
+//
+// Runs on Node's standard library alone, with no package install:
+//
+//     node scripts/check_floor_parity.mjs
+//
+// Exits non-zero on any disagreement larger than the fixture's tolerance, on any
+// verdict sentence that differs by a character, and on a PCG64 starting state
+// that differs from numpy's. The site deploy depends on this passing.
+
+import { readFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "..");
+const floor = createRequire(import.meta.url)(join(root, "site", "floor.js"));
+const fixture = JSON.parse(readFileSync(join(root, "site", "floor-golden.json"), "utf8"));
+const tolerance = fixture.tolerance;
+
+const failures = [];
+
+for (const [seed, expected] of Object.entries(fixture.pcg64_initial_state)) {
+  const actual = floor._internal.seedStates[seed];
+  if (!actual || actual.state !== expected.state || actual.inc !== expected.inc) {
+    failures.push(`PCG64 state for seed ${seed} differs from numpy's`);
+  }
+}
+
+let worst = 0;
+for (const c of fixture.cases) {
+  const label = `n=${c.n} bins=${c.n_bins} accuracy=${c.accuracy}`;
+  const started = Date.now();
+  const band = floor.syntheticFloor(c.n, c.n_bins, c.accuracy);
+  const elapsed = Date.now() - started;
+
+  for (const key of ["mean", "p95"]) {
+    const gap = Math.abs(band[key] - c[key]);
+    worst = Math.max(worst, gap);
+    if (!(gap <= tolerance)) {
+      failures.push(`${label}: ${key} ${band[key]} against Python ${c[key]} (off by ${gap})`);
+    }
+  }
+  for (const [measured, expected] of Object.entries(c.statements)) {
+    const actual = floor.statement(Number(measured), band);
+    if (actual !== expected) {
+      failures.push(`${label}: verdict for ECE ${measured} differs\n  js:     ${actual}\n  python: ${expected}`);
+    }
+  }
+  console.log(`${label.padEnd(36)} mean ${band.mean.toFixed(6)}  p95 ${band.p95.toFixed(6)}  ${elapsed}ms`);
+}
+
+console.log(`\n${fixture.cases.length} cases, largest difference ${worst.toExponential(2)}, tolerance ${tolerance}`);
+if (failures.length) {
+  console.error(`\n${failures.length} disagreement(s) with the Python:\n`);
+  for (const failure of failures) console.error(failure);
+  process.exit(1);
+}
+console.log("floor.js agrees with the Python.");

scripts/floor_golden.py

@@ -0,0 +1,179 @@
+"""Export, or check, the golden values the site's floor calculator is held to.
+
+The site reimplements :func:`plumbline.metrics.calibration.synthetic_floor` in
+JavaScript (``site/floor.js``). If the two disagree, the site misreports what the
+tool says, which is worse than having no site. This script writes the Python's
+answers for a spread of configurations to ``site/floor-golden.json``, and
+``scripts/check_floor_parity.mjs`` fails CI when the JavaScript disagrees.
+
+Run with no arguments to rewrite the fixture. Run with ``--check`` to confirm the
+committed fixture still matches what the Python produces today, so a change to
+the floor cannot leave the JavaScript agreeing with a stale answer.
+
+    uv run python scripts/floor_golden.py
+    uv run python scripts/floor_golden.py --check
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from plumbline.metrics.calibration import CalibrationFigure, synthetic_floor
+
+FIXTURE = Path(__file__).resolve().parent.parent / "site" / "floor-golden.json"
+
+#: Absolute tolerance on every floor value, in both directions of the check.
+#: The JavaScript reproduces numpy's random stream draw for draw, so the only
+#: differences left are summation order (numpy sums pairwise) and last-bit
+#: differences between C's and JavaScript's pow, log, and exp: around 1e-16.
+#: One desynchronised draw moves the result by around 1e-3. 1e-9 sits far from
+#: both, and five orders below the fourth decimal the page prints.
+TOLERANCE = 1e-9
+
+#: (rows, bins, accuracy). Covers the ordinary range, the 105-row example, very
+#: small n, more bins than rows, accuracies whose gamma shape falls below 1
+#: (numpy takes a different sampling path there), and one large n.
+CASES: list[tuple[int, int, float]] = [
+    (1, 10, 0.8),
+    (5, 10, 0.8),
+    (5, 20, 0.5),
+    (10, 50, 0.8),
+    (20, 10, 0.9),
+    (30, 5, 0.7),
+    (50, 15, 0.8),
+    (100, 10, 0.5),
+    (105, 10, 0.7714),
+    (105, 10, 0.8),
+    (200, 20, 0.6),
+    (250, 100, 0.8),
+    (300, 10, 0.95),
+    (500, 10, 0.8),
+    (500, 15, 0.9),
+    (500, 5, 0.3),
+    (750, 10, 0.99),
+    (1000, 10, 0.75),
+    (1000, 20, 0.85),
+    (1000, 50, 0.95),
+    (2000, 10, 0.8),
+    (2000, 15, 0.1),
+    (4000, 10, 0.8),
+    (5000, 20, 0.97),
+    (10000, 10, 0.8),
+    (10000, 30, 0.9),
+    (20000, 10, 0.65),
+]
+
+#: Measured ECE values each case's verdict sentence is exported for, so the
+#: wording, the formatting, and the side of the line a value falls on are all
+#: checked, not only the floor.
+MEASURED = [0.005, 0.02, 0.05, 0.074, 0.1, 0.2]
+
+
+def _pcg64_state(seed: int) -> dict[str, str]:
+    state = np.random.default_rng(seed).bit_generator.state["state"]
+    return {"state": str(state["state"]), "inc": str(state["inc"])}
+
+
+def build() -> dict[str, Any]:
+    cases = []
+    for n, n_bins, accuracy in CASES:
+        band = synthetic_floor(n, n_bins=n_bins, accuracy=accuracy)["ece"]
+        statements = {
+            repr(measured): CalibrationFigure(
+                metric="ece",
+                value=measured,
+                n=band.n,
+                floor=band,
+                n_bins=n_bins,
+                binning="equal_width",
+            ).statement()
+            for measured in MEASURED
+        }
+        cases.append(
+            {
+                "n": n,
+                "n_bins": n_bins,
+                "accuracy": accuracy,
+                "mean": band.mean,
+                "p95": band.p95,
+                "statements": statements,
+            }
+        )
+    return {
+        "about": (
+            "Golden values from plumbline.metrics.calibration.synthetic_floor, "
+            "which site/floor.js must reproduce. Written by scripts/floor_golden.py."
+        ),
+        "function": "synthetic_floor(n, n_bins, accuracy), all other arguments default",
+        "defaults": {"binning": "equal_width", "n_boot": 2000, "concentration": 6.0, "seed": 0},
+        "numpy": np.__version__,
+        "tolerance": TOLERANCE,
+        "pcg64_initial_state": {"0": _pcg64_state(0), "1": _pcg64_state(1)},
+        "cases": cases,
+    }
+
+
+def _check(fresh: dict[str, Any]) -> list[str]:
+    if not FIXTURE.is_file():
+        return [f"{FIXTURE} does not exist; run this script without --check"]
+    committed = json.loads(FIXTURE.read_text(encoding="utf-8"))
+    problems = []
+    if committed["pcg64_initial_state"] != fresh["pcg64_initial_state"]:
+        problems.append("numpy's seeded PCG64 states differ from the fixture")
+    if len(committed["cases"]) != len(fresh["cases"]):
+        problems.append("the fixture holds a different set of cases; regenerate it")
+        return problems
+    for old, new in zip(committed["cases"], fresh["cases"], strict=True):
+        label = f"n={new['n']} bins={new['n_bins']} accuracy={new['accuracy']}"
+        if (old["n"], old["n_bins"], old["accuracy"]) != (
+            new["n"],
+            new["n_bins"],
+            new["accuracy"],
+        ):
+            problems.append(f"{label}: case list changed; regenerate the fixture")
+            continue
+        for key in ("mean", "p95"):
+            if abs(old[key] - new[key]) > TOLERANCE:
+                problems.append(f"{label}: {key} is {new[key]!r} now, fixture says {old[key]!r}")
+        if old["statements"] != new["statements"]:
+            problems.append(f"{label}: verdict wording changed")
+    return problems
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="fail if the committed fixture no longer matches the Python",
+    )
+    args = parser.parse_args()
+
+    fresh = build()
+    if args.check:
+        problems = _check(fresh)
+        for problem in problems:
+            print(problem, file=sys.stderr)
+        if problems:
+            print(
+                "site/floor-golden.json is stale. Regenerate it with "
+                "`uv run python scripts/floor_golden.py`, then make site/floor.js agree.",
+                file=sys.stderr,
+            )
+            return 1
+        print(f"fixture current: {len(fresh['cases'])} cases match the Python")
+        return 0
+
+    FIXTURE.write_text(json.dumps(fresh, indent=2) + "\n", encoding="utf-8")
+    print(f"wrote {len(fresh['cases'])} cases to {FIXTURE}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())