diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml new file mode 100644 index 0000000..4103273 --- /dev/null +++ b/.github/workflows/site.yml @@ -0,0 +1,52 @@ +name: Site + +# The site at tmhsdigital.github.io/plumbline carries a JavaScript +# reimplementation of the calibration floor. If it disagrees with the Python, +# the site misreports the tool, which is worse than having no site. This +# workflow holds the two together; later jobs assemble and deploy the site, and +# they depend on this one, so a wrong calculator never ships. A stale site is +# the better failure. +# +# It is deliberately separate from ci.yml and release.yml: it shares no jobs, +# no concurrency group, and no permissions with either. +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: site-${{ github.ref }} + cancel-in-progress: true + +jobs: + parity: + name: the site's floor agrees with the Python + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.13" + enable-cache: true + + - name: Sync dependencies + run: uv sync --locked + + # Without this, a change to the Python floor would leave the JavaScript + # agreeing with an old answer and the check below passing. + - name: The committed fixture is what the Python produces today + run: uv run python scripts/floor_golden.py --check + + # The runner's own Node, with no setup action and no package install. + # The check uses the standard library only. + - name: floor.js reproduces the fixture + run: | + node --version + node scripts/check_floor_parity.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 87ced80..5ec4498 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,14 @@ different event from one that moved because it was wrong. ## Unreleased +### Added + +- A browser calculator for the ECE floor (`site/`), a JavaScript port of + `synthetic_floor` that reproduces numpy's seeded random stream draw for draw. + `scripts/floor_golden.py` exports golden values from the Python and + `scripts/check_floor_parity.mjs` holds the port to them within 1e-9 in CI. + The page is not deployed yet. + ### Fixed - The inconclusive verdict read as a pass. "Not distinguishable from a diff --git a/scripts/check_floor_parity.mjs b/scripts/check_floor_parity.mjs new file mode 100644 index 0000000..3689c8c --- /dev/null +++ b/scripts/check_floor_parity.mjs @@ -0,0 +1,59 @@ +// Holds site/floor.js to the golden values the Python exported. +// +// Runs on Node's standard library alone, with no package install: +// +// node scripts/check_floor_parity.mjs +// +// Exits non-zero on any disagreement larger than the fixture's tolerance, on any +// verdict sentence that differs by a character, and on a PCG64 starting state +// that differs from numpy's. The site deploy depends on this passing. + +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const floor = createRequire(import.meta.url)(join(root, "site", "floor.js")); +const fixture = JSON.parse(readFileSync(join(root, "site", "floor-golden.json"), "utf8")); +const tolerance = fixture.tolerance; + +const failures = []; + +for (const [seed, expected] of Object.entries(fixture.pcg64_initial_state)) { + const actual = floor._internal.seedStates[seed]; + if (!actual || actual.state !== expected.state || actual.inc !== expected.inc) { + failures.push(`PCG64 state for seed ${seed} differs from numpy's`); + } +} + +let worst = 0; +for (const c of fixture.cases) { + const label = `n=${c.n} bins=${c.n_bins} accuracy=${c.accuracy}`; + const started = Date.now(); + const band = floor.syntheticFloor(c.n, c.n_bins, c.accuracy); + const elapsed = Date.now() - started; + + for (const key of ["mean", "p95"]) { + const gap = Math.abs(band[key] - c[key]); + worst = Math.max(worst, gap); + if (!(gap <= tolerance)) { + failures.push(`${label}: ${key} ${band[key]} against Python ${c[key]} (off by ${gap})`); + } + } + for (const [measured, expected] of Object.entries(c.statements)) { + const actual = floor.statement(Number(measured), band); + if (actual !== expected) { + failures.push(`${label}: verdict for ECE ${measured} differs\n js: ${actual}\n python: ${expected}`); + } + } + console.log(`${label.padEnd(36)} mean ${band.mean.toFixed(6)} p95 ${band.p95.toFixed(6)} ${elapsed}ms`); +} + +console.log(`\n${fixture.cases.length} cases, largest difference ${worst.toExponential(2)}, tolerance ${tolerance}`); +if (failures.length) { + console.error(`\n${failures.length} disagreement(s) with the Python:\n`); + for (const failure of failures) console.error(failure); + process.exit(1); +} +console.log("floor.js agrees with the Python."); diff --git a/scripts/floor_golden.py b/scripts/floor_golden.py new file mode 100644 index 0000000..f921285 --- /dev/null +++ b/scripts/floor_golden.py @@ -0,0 +1,179 @@ +"""Export, or check, the golden values the site's floor calculator is held to. + +The site reimplements :func:`plumbline.metrics.calibration.synthetic_floor` in +JavaScript (``site/floor.js``). If the two disagree, the site misreports what the +tool says, which is worse than having no site. This script writes the Python's +answers for a spread of configurations to ``site/floor-golden.json``, and +``scripts/check_floor_parity.mjs`` fails CI when the JavaScript disagrees. + +Run with no arguments to rewrite the fixture. Run with ``--check`` to confirm the +committed fixture still matches what the Python produces today, so a change to +the floor cannot leave the JavaScript agreeing with a stale answer. + + uv run python scripts/floor_golden.py + uv run python scripts/floor_golden.py --check +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +import numpy as np + +from plumbline.metrics.calibration import CalibrationFigure, synthetic_floor + +FIXTURE = Path(__file__).resolve().parent.parent / "site" / "floor-golden.json" + +#: Absolute tolerance on every floor value, in both directions of the check. +#: The JavaScript reproduces numpy's random stream draw for draw, so the only +#: differences left are summation order (numpy sums pairwise) and last-bit +#: differences between C's and JavaScript's pow, log, and exp: around 1e-16. +#: One desynchronised draw moves the result by around 1e-3. 1e-9 sits far from +#: both, and five orders below the fourth decimal the page prints. +TOLERANCE = 1e-9 + +#: (rows, bins, accuracy). Covers the ordinary range, the 105-row example, very +#: small n, more bins than rows, accuracies whose gamma shape falls below 1 +#: (numpy takes a different sampling path there), and one large n. +CASES: list[tuple[int, int, float]] = [ + (1, 10, 0.8), + (5, 10, 0.8), + (5, 20, 0.5), + (10, 50, 0.8), + (20, 10, 0.9), + (30, 5, 0.7), + (50, 15, 0.8), + (100, 10, 0.5), + (105, 10, 0.7714), + (105, 10, 0.8), + (200, 20, 0.6), + (250, 100, 0.8), + (300, 10, 0.95), + (500, 10, 0.8), + (500, 15, 0.9), + (500, 5, 0.3), + (750, 10, 0.99), + (1000, 10, 0.75), + (1000, 20, 0.85), + (1000, 50, 0.95), + (2000, 10, 0.8), + (2000, 15, 0.1), + (4000, 10, 0.8), + (5000, 20, 0.97), + (10000, 10, 0.8), + (10000, 30, 0.9), + (20000, 10, 0.65), +] + +#: Measured ECE values each case's verdict sentence is exported for, so the +#: wording, the formatting, and the side of the line a value falls on are all +#: checked, not only the floor. +MEASURED = [0.005, 0.02, 0.05, 0.074, 0.1, 0.2] + + +def _pcg64_state(seed: int) -> dict[str, str]: + state = np.random.default_rng(seed).bit_generator.state["state"] + return {"state": str(state["state"]), "inc": str(state["inc"])} + + +def build() -> dict[str, Any]: + cases = [] + for n, n_bins, accuracy in CASES: + band = synthetic_floor(n, n_bins=n_bins, accuracy=accuracy)["ece"] + statements = { + repr(measured): CalibrationFigure( + metric="ece", + value=measured, + n=band.n, + floor=band, + n_bins=n_bins, + binning="equal_width", + ).statement() + for measured in MEASURED + } + cases.append( + { + "n": n, + "n_bins": n_bins, + "accuracy": accuracy, + "mean": band.mean, + "p95": band.p95, + "statements": statements, + } + ) + return { + "about": ( + "Golden values from plumbline.metrics.calibration.synthetic_floor, " + "which site/floor.js must reproduce. Written by scripts/floor_golden.py." + ), + "function": "synthetic_floor(n, n_bins, accuracy), all other arguments default", + "defaults": {"binning": "equal_width", "n_boot": 2000, "concentration": 6.0, "seed": 0}, + "numpy": np.__version__, + "tolerance": TOLERANCE, + "pcg64_initial_state": {"0": _pcg64_state(0), "1": _pcg64_state(1)}, + "cases": cases, + } + + +def _check(fresh: dict[str, Any]) -> list[str]: + if not FIXTURE.is_file(): + return [f"{FIXTURE} does not exist; run this script without --check"] + committed = json.loads(FIXTURE.read_text(encoding="utf-8")) + problems = [] + if committed["pcg64_initial_state"] != fresh["pcg64_initial_state"]: + problems.append("numpy's seeded PCG64 states differ from the fixture") + if len(committed["cases"]) != len(fresh["cases"]): + problems.append("the fixture holds a different set of cases; regenerate it") + return problems + for old, new in zip(committed["cases"], fresh["cases"], strict=True): + label = f"n={new['n']} bins={new['n_bins']} accuracy={new['accuracy']}" + if (old["n"], old["n_bins"], old["accuracy"]) != ( + new["n"], + new["n_bins"], + new["accuracy"], + ): + problems.append(f"{label}: case list changed; regenerate the fixture") + continue + for key in ("mean", "p95"): + if abs(old[key] - new[key]) > TOLERANCE: + problems.append(f"{label}: {key} is {new[key]!r} now, fixture says {old[key]!r}") + if old["statements"] != new["statements"]: + problems.append(f"{label}: verdict wording changed") + return problems + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--check", + action="store_true", + help="fail if the committed fixture no longer matches the Python", + ) + args = parser.parse_args() + + fresh = build() + if args.check: + problems = _check(fresh) + for problem in problems: + print(problem, file=sys.stderr) + if problems: + print( + "site/floor-golden.json is stale. Regenerate it with " + "`uv run python scripts/floor_golden.py`, then make site/floor.js agree.", + file=sys.stderr, + ) + return 1 + print(f"fixture current: {len(fresh['cases'])} cases match the Python") + return 0 + + FIXTURE.write_text(json.dumps(fresh, indent=2) + "\n", encoding="utf-8") + print(f"wrote {len(fresh['cases'])} cases to {FIXTURE}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/site/floor-golden.json b/site/floor-golden.json new file mode 100644 index 0000000..07603cd --- /dev/null +++ b/site/floor-golden.json @@ -0,0 +1,429 @@ +{ + "about": "Golden values from plumbline.metrics.calibration.synthetic_floor, which site/floor.js must reproduce. Written by scripts/floor_golden.py.", + "function": "synthetic_floor(n, n_bins, accuracy), all other arguments default", + "defaults": { + "binning": "equal_width", + "n_boot": 2000, + "concentration": 6.0, + "seed": 0 + }, + "numpy": "2.5.3", + "tolerance": 1e-09, + "pcg64_initial_state": { + "0": { + "state": "35399562948360463058890781895381311971", + "inc": "87136372517582989555478159403783844777" + }, + "1": { + "state": "207833532711051698738587646355624148094", + "inc": "194290289479364712180083596243593368443" + } + }, + "cases": [ + { + "n": 1, + "n_bins": 10, + "accuracy": 0.8, + "mean": 0.3800222137274552, + "p95": 0.7463609574384902, + "statements": { + "0.005": "ECE 0.0050 over 1 rows (10 equal width bins), against a calibrated-model floor of 0.3800 (95th percentile 0.7464): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 1 rows (10 equal width bins), against a calibrated-model floor of 0.3800 (95th percentile 0.7464): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 1 rows (10 equal width bins), against a calibrated-model floor of 0.3800 (95th percentile 0.7464): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 1 rows (10 equal width bins), against a calibrated-model floor of 0.3800 (95th percentile 0.7464): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.1": "ECE 0.1000 over 1 rows (10 equal width bins), against a calibrated-model floor of 0.3800 (95th percentile 0.7464): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.2": "ECE 0.2000 over 1 rows (10 equal width bins), against a calibrated-model floor of 0.3800 (95th percentile 0.7464): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable." + } + }, + { + "n": 5, + "n_bins": 10, + "accuracy": 0.8, + "mean": 0.2847577610479967, + "p95": 0.42849582212750476, + "statements": { + "0.005": "ECE 0.0050 over 5 rows (10 equal width bins), against a calibrated-model floor of 0.2848 (95th percentile 0.4285): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 5 rows (10 equal width bins), against a calibrated-model floor of 0.2848 (95th percentile 0.4285): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 5 rows (10 equal width bins), against a calibrated-model floor of 0.2848 (95th percentile 0.4285): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 5 rows (10 equal width bins), against a calibrated-model floor of 0.2848 (95th percentile 0.4285): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.1": "ECE 0.1000 over 5 rows (10 equal width bins), against a calibrated-model floor of 0.2848 (95th percentile 0.4285): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.2": "ECE 0.2000 over 5 rows (10 equal width bins), against a calibrated-model floor of 0.2848 (95th percentile 0.4285): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable." + } + }, + { + "n": 5, + "n_bins": 20, + "accuracy": 0.5, + "mean": 0.4039633559253163, + "p95": 0.555938114043049, + "statements": { + "0.005": "ECE 0.0050 over 5 rows (20 equal width bins), against a calibrated-model floor of 0.4040 (95th percentile 0.5559): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 5 rows (20 equal width bins), against a calibrated-model floor of 0.4040 (95th percentile 0.5559): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 5 rows (20 equal width bins), against a calibrated-model floor of 0.4040 (95th percentile 0.5559): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 5 rows (20 equal width bins), against a calibrated-model floor of 0.4040 (95th percentile 0.5559): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.1": "ECE 0.1000 over 5 rows (20 equal width bins), against a calibrated-model floor of 0.4040 (95th percentile 0.5559): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.2": "ECE 0.2000 over 5 rows (20 equal width bins), against a calibrated-model floor of 0.4040 (95th percentile 0.5559): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable." + } + }, + { + "n": 10, + "n_bins": 50, + "accuracy": 0.8, + "mean": 0.27711625440709153, + "p95": 0.39266384333888094, + "statements": { + "0.005": "ECE 0.0050 over 10 rows (50 equal width bins), against a calibrated-model floor of 0.2771 (95th percentile 0.3927): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 10 rows (50 equal width bins), against a calibrated-model floor of 0.2771 (95th percentile 0.3927): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 10 rows (50 equal width bins), against a calibrated-model floor of 0.2771 (95th percentile 0.3927): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 10 rows (50 equal width bins), against a calibrated-model floor of 0.2771 (95th percentile 0.3927): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.1": "ECE 0.1000 over 10 rows (50 equal width bins), against a calibrated-model floor of 0.2771 (95th percentile 0.3927): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.2": "ECE 0.2000 over 10 rows (50 equal width bins), against a calibrated-model floor of 0.2771 (95th percentile 0.3927): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable." + } + }, + { + "n": 20, + "n_bins": 10, + "accuracy": 0.9, + "mean": 0.11220229222944622, + "p95": 0.16500137870960746, + "statements": { + "0.005": "ECE 0.0050 over 20 rows (10 equal width bins), against a calibrated-model floor of 0.1122 (95th percentile 0.1650): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 20 rows (10 equal width bins), against a calibrated-model floor of 0.1122 (95th percentile 0.1650): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 20 rows (10 equal width bins), against a calibrated-model floor of 0.1122 (95th percentile 0.1650): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 20 rows (10 equal width bins), against a calibrated-model floor of 0.1122 (95th percentile 0.1650): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.1": "ECE 0.1000 over 20 rows (10 equal width bins), against a calibrated-model floor of 0.1122 (95th percentile 0.1650): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.2": "ECE 0.2000 over 20 rows (10 equal width bins), against a calibrated-model floor of 0.1122 (95th percentile 0.1650): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 30, + "n_bins": 5, + "accuracy": 0.7, + "mean": 0.12139637539943718, + "p95": 0.20461675476596455, + "statements": { + "0.005": "ECE 0.0050 over 30 rows (5 equal width bins), against a calibrated-model floor of 0.1214 (95th percentile 0.2046): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 30 rows (5 equal width bins), against a calibrated-model floor of 0.1214 (95th percentile 0.2046): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 30 rows (5 equal width bins), against a calibrated-model floor of 0.1214 (95th percentile 0.2046): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 30 rows (5 equal width bins), against a calibrated-model floor of 0.1214 (95th percentile 0.2046): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.1": "ECE 0.1000 over 30 rows (5 equal width bins), against a calibrated-model floor of 0.1214 (95th percentile 0.2046): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.2": "ECE 0.2000 over 30 rows (5 equal width bins), against a calibrated-model floor of 0.1214 (95th percentile 0.2046): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable." + } + }, + { + "n": 50, + "n_bins": 15, + "accuracy": 0.8, + "mean": 0.13313113466264428, + "p95": 0.18497730704057386, + "statements": { + "0.005": "ECE 0.0050 over 50 rows (15 equal width bins), against a calibrated-model floor of 0.1331 (95th percentile 0.1850): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 50 rows (15 equal width bins), against a calibrated-model floor of 0.1331 (95th percentile 0.1850): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 50 rows (15 equal width bins), against a calibrated-model floor of 0.1331 (95th percentile 0.1850): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 50 rows (15 equal width bins), against a calibrated-model floor of 0.1331 (95th percentile 0.1850): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.1": "ECE 0.1000 over 50 rows (15 equal width bins), against a calibrated-model floor of 0.1331 (95th percentile 0.1850): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.2": "ECE 0.2000 over 50 rows (15 equal width bins), against a calibrated-model floor of 0.1331 (95th percentile 0.1850): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 100, + "n_bins": 10, + "accuracy": 0.5, + "mean": 0.10154939772542025, + "p95": 0.1506344546031062, + "statements": { + "0.005": "ECE 0.0050 over 100 rows (10 equal width bins), against a calibrated-model floor of 0.1015 (95th percentile 0.1506): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 100 rows (10 equal width bins), against a calibrated-model floor of 0.1015 (95th percentile 0.1506): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 100 rows (10 equal width bins), against a calibrated-model floor of 0.1015 (95th percentile 0.1506): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 100 rows (10 equal width bins), against a calibrated-model floor of 0.1015 (95th percentile 0.1506): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.1": "ECE 0.1000 over 100 rows (10 equal width bins), against a calibrated-model floor of 0.1015 (95th percentile 0.1506): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.2": "ECE 0.2000 over 100 rows (10 equal width bins), against a calibrated-model floor of 0.1015 (95th percentile 0.1506): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 105, + "n_bins": 10, + "accuracy": 0.7714, + "mean": 0.07507178247177257, + "p95": 0.11790988562838273, + "statements": { + "0.005": "ECE 0.0050 over 105 rows (10 equal width bins), against a calibrated-model floor of 0.0751 (95th percentile 0.1179): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 105 rows (10 equal width bins), against a calibrated-model floor of 0.0751 (95th percentile 0.1179): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 105 rows (10 equal width bins), against a calibrated-model floor of 0.0751 (95th percentile 0.1179): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 105 rows (10 equal width bins), against a calibrated-model floor of 0.0751 (95th percentile 0.1179): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.1": "ECE 0.1000 over 105 rows (10 equal width bins), against a calibrated-model floor of 0.0751 (95th percentile 0.1179): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.2": "ECE 0.2000 over 105 rows (10 equal width bins), against a calibrated-model floor of 0.0751 (95th percentile 0.1179): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 105, + "n_bins": 10, + "accuracy": 0.8, + "mean": 0.07199355208097018, + "p95": 0.11196474547538811, + "statements": { + "0.005": "ECE 0.0050 over 105 rows (10 equal width bins), against a calibrated-model floor of 0.0720 (95th percentile 0.1120): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 105 rows (10 equal width bins), against a calibrated-model floor of 0.0720 (95th percentile 0.1120): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 105 rows (10 equal width bins), against a calibrated-model floor of 0.0720 (95th percentile 0.1120): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 105 rows (10 equal width bins), against a calibrated-model floor of 0.0720 (95th percentile 0.1120): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.1": "ECE 0.1000 over 105 rows (10 equal width bins), against a calibrated-model floor of 0.0720 (95th percentile 0.1120): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.2": "ECE 0.2000 over 105 rows (10 equal width bins), against a calibrated-model floor of 0.0720 (95th percentile 0.1120): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 200, + "n_bins": 20, + "accuracy": 0.6, + "mean": 0.09645611455860527, + "p95": 0.1281392861832022, + "statements": { + "0.005": "ECE 0.0050 over 200 rows (20 equal width bins), against a calibrated-model floor of 0.0965 (95th percentile 0.1281): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 200 rows (20 equal width bins), against a calibrated-model floor of 0.0965 (95th percentile 0.1281): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 200 rows (20 equal width bins), against a calibrated-model floor of 0.0965 (95th percentile 0.1281): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 200 rows (20 equal width bins), against a calibrated-model floor of 0.0965 (95th percentile 0.1281): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.1": "ECE 0.1000 over 200 rows (20 equal width bins), against a calibrated-model floor of 0.0965 (95th percentile 0.1281): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.2": "ECE 0.2000 over 200 rows (20 equal width bins), against a calibrated-model floor of 0.0965 (95th percentile 0.1281): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 250, + "n_bins": 100, + "accuracy": 0.8, + "mean": 0.14023235095799594, + "p95": 0.1626327053033523, + "statements": { + "0.005": "ECE 0.0050 over 250 rows (100 equal width bins), against a calibrated-model floor of 0.1402 (95th percentile 0.1626): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 250 rows (100 equal width bins), against a calibrated-model floor of 0.1402 (95th percentile 0.1626): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 250 rows (100 equal width bins), against a calibrated-model floor of 0.1402 (95th percentile 0.1626): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 250 rows (100 equal width bins), against a calibrated-model floor of 0.1402 (95th percentile 0.1626): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.1": "ECE 0.1000 over 250 rows (100 equal width bins), against a calibrated-model floor of 0.1402 (95th percentile 0.1626): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.2": "ECE 0.2000 over 250 rows (100 equal width bins), against a calibrated-model floor of 0.1402 (95th percentile 0.1626): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 300, + "n_bins": 10, + "accuracy": 0.95, + "mean": 0.023666559618355967, + "p95": 0.03662071112951838, + "statements": { + "0.005": "ECE 0.0050 over 300 rows (10 equal width bins), against a calibrated-model floor of 0.0237 (95th percentile 0.0366): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 300 rows (10 equal width bins), against a calibrated-model floor of 0.0237 (95th percentile 0.0366): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 300 rows (10 equal width bins), against a calibrated-model floor of 0.0237 (95th percentile 0.0366): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 300 rows (10 equal width bins), against a calibrated-model floor of 0.0237 (95th percentile 0.0366): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 300 rows (10 equal width bins), against a calibrated-model floor of 0.0237 (95th percentile 0.0366): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 300 rows (10 equal width bins), against a calibrated-model floor of 0.0237 (95th percentile 0.0366): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 500, + "n_bins": 10, + "accuracy": 0.8, + "mean": 0.0355612696325439, + "p95": 0.053085240536494605, + "statements": { + "0.005": "ECE 0.0050 over 500 rows (10 equal width bins), against a calibrated-model floor of 0.0356 (95th percentile 0.0531): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 500 rows (10 equal width bins), against a calibrated-model floor of 0.0356 (95th percentile 0.0531): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 500 rows (10 equal width bins), against a calibrated-model floor of 0.0356 (95th percentile 0.0531): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 500 rows (10 equal width bins), against a calibrated-model floor of 0.0356 (95th percentile 0.0531): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 500 rows (10 equal width bins), against a calibrated-model floor of 0.0356 (95th percentile 0.0531): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 500 rows (10 equal width bins), against a calibrated-model floor of 0.0356 (95th percentile 0.0531): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 500, + "n_bins": 15, + "accuracy": 0.9, + "mean": 0.02784884432264174, + "p95": 0.040260151802465075, + "statements": { + "0.005": "ECE 0.0050 over 500 rows (15 equal width bins), against a calibrated-model floor of 0.0278 (95th percentile 0.0403): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 500 rows (15 equal width bins), against a calibrated-model floor of 0.0278 (95th percentile 0.0403): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 500 rows (15 equal width bins), against a calibrated-model floor of 0.0278 (95th percentile 0.0403): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 500 rows (15 equal width bins), against a calibrated-model floor of 0.0278 (95th percentile 0.0403): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 500 rows (15 equal width bins), against a calibrated-model floor of 0.0278 (95th percentile 0.0403): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 500 rows (15 equal width bins), against a calibrated-model floor of 0.0278 (95th percentile 0.0403): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 500, + "n_bins": 5, + "accuracy": 0.3, + "mean": 0.029970541953061883, + "p95": 0.05014769842329827, + "statements": { + "0.005": "ECE 0.0050 over 500 rows (5 equal width bins), against a calibrated-model floor of 0.0300 (95th percentile 0.0501): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 500 rows (5 equal width bins), against a calibrated-model floor of 0.0300 (95th percentile 0.0501): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 500 rows (5 equal width bins), against a calibrated-model floor of 0.0300 (95th percentile 0.0501): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.074": "ECE 0.0740 over 500 rows (5 equal width bins), against a calibrated-model floor of 0.0300 (95th percentile 0.0501): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 500 rows (5 equal width bins), against a calibrated-model floor of 0.0300 (95th percentile 0.0501): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 500 rows (5 equal width bins), against a calibrated-model floor of 0.0300 (95th percentile 0.0501): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 750, + "n_bins": 10, + "accuracy": 0.99, + "mean": 0.005165671794089274, + "p95": 0.00870125465515504, + "statements": { + "0.005": "ECE 0.0050 over 750 rows (10 equal width bins), against a calibrated-model floor of 0.0052 (95th percentile 0.0087): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 750 rows (10 equal width bins), against a calibrated-model floor of 0.0052 (95th percentile 0.0087): miscalibration is distinguishable from sampling noise.", + "0.05": "ECE 0.0500 over 750 rows (10 equal width bins), against a calibrated-model floor of 0.0052 (95th percentile 0.0087): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 750 rows (10 equal width bins), against a calibrated-model floor of 0.0052 (95th percentile 0.0087): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 750 rows (10 equal width bins), against a calibrated-model floor of 0.0052 (95th percentile 0.0087): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 750 rows (10 equal width bins), against a calibrated-model floor of 0.0052 (95th percentile 0.0087): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 1000, + "n_bins": 10, + "accuracy": 0.75, + "mean": 0.02675736936838183, + "p95": 0.04020104887001681, + "statements": { + "0.005": "ECE 0.0050 over 1000 rows (10 equal width bins), against a calibrated-model floor of 0.0268 (95th percentile 0.0402): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 1000 rows (10 equal width bins), against a calibrated-model floor of 0.0268 (95th percentile 0.0402): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 1000 rows (10 equal width bins), against a calibrated-model floor of 0.0268 (95th percentile 0.0402): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 1000 rows (10 equal width bins), against a calibrated-model floor of 0.0268 (95th percentile 0.0402): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 1000 rows (10 equal width bins), against a calibrated-model floor of 0.0268 (95th percentile 0.0402): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 1000 rows (10 equal width bins), against a calibrated-model floor of 0.0268 (95th percentile 0.0402): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 1000, + "n_bins": 20, + "accuracy": 0.85, + "mean": 0.029019264436455913, + "p95": 0.03978104831245656, + "statements": { + "0.005": "ECE 0.0050 over 1000 rows (20 equal width bins), against a calibrated-model floor of 0.0290 (95th percentile 0.0398): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 1000 rows (20 equal width bins), against a calibrated-model floor of 0.0290 (95th percentile 0.0398): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 1000 rows (20 equal width bins), against a calibrated-model floor of 0.0290 (95th percentile 0.0398): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 1000 rows (20 equal width bins), against a calibrated-model floor of 0.0290 (95th percentile 0.0398): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 1000 rows (20 equal width bins), against a calibrated-model floor of 0.0290 (95th percentile 0.0398): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 1000 rows (20 equal width bins), against a calibrated-model floor of 0.0290 (95th percentile 0.0398): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 1000, + "n_bins": 50, + "accuracy": 0.95, + "mean": 0.02521845499263482, + "p95": 0.031638820373931995, + "statements": { + "0.005": "ECE 0.0050 over 1000 rows (50 equal width bins), against a calibrated-model floor of 0.0252 (95th percentile 0.0316): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 1000 rows (50 equal width bins), against a calibrated-model floor of 0.0252 (95th percentile 0.0316): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 1000 rows (50 equal width bins), against a calibrated-model floor of 0.0252 (95th percentile 0.0316): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 1000 rows (50 equal width bins), against a calibrated-model floor of 0.0252 (95th percentile 0.0316): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 1000 rows (50 equal width bins), against a calibrated-model floor of 0.0252 (95th percentile 0.0316): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 1000 rows (50 equal width bins), against a calibrated-model floor of 0.0252 (95th percentile 0.0316): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 2000, + "n_bins": 10, + "accuracy": 0.8, + "mean": 0.017489921485761555, + "p95": 0.02596877487096666, + "statements": { + "0.005": "ECE 0.0050 over 2000 rows (10 equal width bins), against a calibrated-model floor of 0.0175 (95th percentile 0.0260): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 2000 rows (10 equal width bins), against a calibrated-model floor of 0.0175 (95th percentile 0.0260): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 2000 rows (10 equal width bins), against a calibrated-model floor of 0.0175 (95th percentile 0.0260): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 2000 rows (10 equal width bins), against a calibrated-model floor of 0.0175 (95th percentile 0.0260): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 2000 rows (10 equal width bins), against a calibrated-model floor of 0.0175 (95th percentile 0.0260): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 2000 rows (10 equal width bins), against a calibrated-model floor of 0.0175 (95th percentile 0.0260): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 2000, + "n_bins": 15, + "accuracy": 0.1, + "mean": 0.013880785016449071, + "p95": 0.02054651436977377, + "statements": { + "0.005": "ECE 0.0050 over 2000 rows (15 equal width bins), against a calibrated-model floor of 0.0139 (95th percentile 0.0205): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 2000 rows (15 equal width bins), against a calibrated-model floor of 0.0139 (95th percentile 0.0205): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.05": "ECE 0.0500 over 2000 rows (15 equal width bins), against a calibrated-model floor of 0.0139 (95th percentile 0.0205): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 2000 rows (15 equal width bins), against a calibrated-model floor of 0.0139 (95th percentile 0.0205): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 2000 rows (15 equal width bins), against a calibrated-model floor of 0.0139 (95th percentile 0.0205): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 2000 rows (15 equal width bins), against a calibrated-model floor of 0.0139 (95th percentile 0.0205): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 4000, + "n_bins": 10, + "accuracy": 0.8, + "mean": 0.012206409389429185, + "p95": 0.018615275077065226, + "statements": { + "0.005": "ECE 0.0050 over 4000 rows (10 equal width bins), against a calibrated-model floor of 0.0122 (95th percentile 0.0186): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 4000 rows (10 equal width bins), against a calibrated-model floor of 0.0122 (95th percentile 0.0186): miscalibration is distinguishable from sampling noise.", + "0.05": "ECE 0.0500 over 4000 rows (10 equal width bins), against a calibrated-model floor of 0.0122 (95th percentile 0.0186): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 4000 rows (10 equal width bins), against a calibrated-model floor of 0.0122 (95th percentile 0.0186): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 4000 rows (10 equal width bins), against a calibrated-model floor of 0.0122 (95th percentile 0.0186): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 4000 rows (10 equal width bins), against a calibrated-model floor of 0.0122 (95th percentile 0.0186): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 5000, + "n_bins": 20, + "accuracy": 0.97, + "mean": 0.005268464475778579, + "p95": 0.00761369266481136, + "statements": { + "0.005": "ECE 0.0050 over 5000 rows (20 equal width bins), against a calibrated-model floor of 0.0053 (95th percentile 0.0076): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 5000 rows (20 equal width bins), against a calibrated-model floor of 0.0053 (95th percentile 0.0076): miscalibration is distinguishable from sampling noise.", + "0.05": "ECE 0.0500 over 5000 rows (20 equal width bins), against a calibrated-model floor of 0.0053 (95th percentile 0.0076): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 5000 rows (20 equal width bins), against a calibrated-model floor of 0.0053 (95th percentile 0.0076): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 5000 rows (20 equal width bins), against a calibrated-model floor of 0.0053 (95th percentile 0.0076): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 5000 rows (20 equal width bins), against a calibrated-model floor of 0.0053 (95th percentile 0.0076): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 10000, + "n_bins": 10, + "accuracy": 0.8, + "mean": 0.007729592376046467, + "p95": 0.011776573914467762, + "statements": { + "0.005": "ECE 0.0050 over 10000 rows (10 equal width bins), against a calibrated-model floor of 0.0077 (95th percentile 0.0118): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 10000 rows (10 equal width bins), against a calibrated-model floor of 0.0077 (95th percentile 0.0118): miscalibration is distinguishable from sampling noise.", + "0.05": "ECE 0.0500 over 10000 rows (10 equal width bins), against a calibrated-model floor of 0.0077 (95th percentile 0.0118): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 10000 rows (10 equal width bins), against a calibrated-model floor of 0.0077 (95th percentile 0.0118): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 10000 rows (10 equal width bins), against a calibrated-model floor of 0.0077 (95th percentile 0.0118): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 10000 rows (10 equal width bins), against a calibrated-model floor of 0.0077 (95th percentile 0.0118): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 10000, + "n_bins": 30, + "accuracy": 0.9, + "mean": 0.008988447318912384, + "p95": 0.011866513346387128, + "statements": { + "0.005": "ECE 0.0050 over 10000 rows (30 equal width bins), against a calibrated-model floor of 0.0090 (95th percentile 0.0119): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 10000 rows (30 equal width bins), against a calibrated-model floor of 0.0090 (95th percentile 0.0119): miscalibration is distinguishable from sampling noise.", + "0.05": "ECE 0.0500 over 10000 rows (30 equal width bins), against a calibrated-model floor of 0.0090 (95th percentile 0.0119): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 10000 rows (30 equal width bins), against a calibrated-model floor of 0.0090 (95th percentile 0.0119): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 10000 rows (30 equal width bins), against a calibrated-model floor of 0.0090 (95th percentile 0.0119): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 10000 rows (30 equal width bins), against a calibrated-model floor of 0.0090 (95th percentile 0.0119): miscalibration is distinguishable from sampling noise." + } + }, + { + "n": 20000, + "n_bins": 10, + "accuracy": 0.65, + "mean": 0.006857970527670982, + "p95": 0.010249792847569345, + "statements": { + "0.005": "ECE 0.0050 over 20000 rows (10 equal width bins), against a calibrated-model floor of 0.0069 (95th percentile 0.0102): INCONCLUSIVE at this sample size. A perfectly calibrated model would often score this badly on this many rows, so this dataset cannot tell the two apart. This is not a clean bill of health: nothing was established either way. Collect more rows to make the question answerable.", + "0.02": "ECE 0.0200 over 20000 rows (10 equal width bins), against a calibrated-model floor of 0.0069 (95th percentile 0.0102): miscalibration is distinguishable from sampling noise.", + "0.05": "ECE 0.0500 over 20000 rows (10 equal width bins), against a calibrated-model floor of 0.0069 (95th percentile 0.0102): miscalibration is distinguishable from sampling noise.", + "0.074": "ECE 0.0740 over 20000 rows (10 equal width bins), against a calibrated-model floor of 0.0069 (95th percentile 0.0102): miscalibration is distinguishable from sampling noise.", + "0.1": "ECE 0.1000 over 20000 rows (10 equal width bins), against a calibrated-model floor of 0.0069 (95th percentile 0.0102): miscalibration is distinguishable from sampling noise.", + "0.2": "ECE 0.2000 over 20000 rows (10 equal width bins), against a calibrated-model floor of 0.0069 (95th percentile 0.0102): miscalibration is distinguishable from sampling noise." + } + } + ] +} diff --git a/site/floor-worker.js b/site/floor-worker.js new file mode 100644 index 0000000..1d187d5 --- /dev/null +++ b/site/floor-worker.js @@ -0,0 +1,17 @@ +// Runs the floor off the main thread, so a 20,000-row bootstrap does not freeze +// the page. The arithmetic is floor.js, unchanged. +importScripts("floor.js"); + +self.onmessage = function (event) { + var job = event.data; + try { + var band = self.PlumblineFloor.syntheticFloor(job.n, job.nBins, job.accuracy, { + onProgress: function (fraction) { + self.postMessage({ id: job.id, progress: fraction }); + }, + }); + self.postMessage({ id: job.id, band: band }); + } catch (error) { + self.postMessage({ id: job.id, error: String(error.message || error) }); + } +}; diff --git a/site/floor.js b/site/floor.js new file mode 100644 index 0000000..d280992 --- /dev/null +++ b/site/floor.js @@ -0,0 +1,881 @@ +/* + * The calibrated-model floor for ECE, reimplemented from plumbline's Python. + * + * This is a port of plumbline.metrics.calibration.synthetic_floor, and it is + * meant to produce the same numbers, not numbers like them. It does that by + * reproducing numpy's random stream draw for draw: the PCG64 bit generator, + * numpy's ziggurat normal and exponential samplers, its Marsaglia-Tsang gamma, + * and its beta. With the same seeds the Python uses (0 for the invented + * probabilities, 1 for the bootstrap), every uniform, every Bernoulli outcome, + * and every bin count is identical to what numpy draws. What remains is the + * order floating point sums are taken in and last-bit differences between C's + * and JavaScript's pow, log, and exp, which move the result around 1e-16. + * + * floor-golden.json holds values exported from the Python implementation, and + * scripts/check_floor_parity.mjs fails CI when this file disagrees with them by + * more than 1e-9. A single desynchronised draw shifts every draw after it and + * moves the result by around 1e-3, so there is nothing in between: either the + * streams agree and the numbers match to 1e-16, or they do not and the check + * fails loudly. + * + * The ziggurat tables at the bottom are numpy's, copied from + * numpy/random/src/distributions/ziggurat_constants.h at numpy v2.5.3. + * numpy is BSD-3-Clause licensed, Copyright (c) 2005-2025 NumPy Developers. + * + * Loads as a plain script (browser, worker) or as a CommonJS module (Node). + * No dependencies. + */ +(function (root) { + "use strict"; + + /* ---------------------------------------------------------------- PCG64 */ + + // numpy's PCG64: 128-bit LCG state, XSL-RR output, advanced before output. + // 128-bit arithmetic is done in eight 16-bit limbs held in plain numbers so + // that every partial product is exact in a double. + + // 2549297995355413924 * 2^64 + 4865540595714422341, low limb first. + var MULT = limbs("2360ed051fc65da44385df649fccf645"); + + function limbs(hex) { + hex = hex.padStart(32, "0"); + var out = new Array(8); + for (var k = 0; k < 8; k++) { + out[k] = parseInt(hex.slice(32 - 4 * (k + 1), 32 - 4 * k), 16); + } + return out; + } + + // The initial states numpy.random.default_rng(seed) arrives at, as + // rng.bit_generator.state reports them. synthetic_floor only ever uses seeds + // 0 and 1, so SeedSequence is not reimplemented; the parity check confirms + // these against the Python on every run. + var SEED_STATES = { + 0: { + state: "35399562948360463058890781895381311971", + inc: "87136372517582989555478159403783844777", + }, + 1: { + state: "207833532711051698738587646355624148094", + inc: "194290289479364712180083596243593368443", + }, + }; + + function decimalToHex(decimal) { + // Decimal string to 32 hex digits, without BigInt, so it runs anywhere. + var digits = decimal.split("").map(Number); + var hex = ""; + while (digits.length && !(digits.length === 1 && digits[0] === 0)) { + var remainder = 0; + var next = []; + for (var i = 0; i < digits.length; i++) { + var value = remainder * 10 + digits[i]; + var q = Math.floor(value / 16); + remainder = value - q * 16; + if (next.length || q) next.push(q); + } + hex = remainder.toString(16) + hex; + digits = next.length ? next : [0]; + } + return hex || "0"; + } + + function Pcg64(stateDecimal, incDecimal) { + this.s = limbs(decimalToHex(stateDecimal)); + this.inc = limbs(decimalToHex(incDecimal)); + this.hi = 0; // upper 32 bits of the last 64-bit output + this.lo = 0; // lower 32 bits + } + + Pcg64.prototype.next = function () { + var s = this.s; + var inc = this.inc; + var s0 = s[0], s1 = s[1], s2 = s[2], s3 = s[3]; + var s4 = s[4], s5 = s[5], s6 = s[6], s7 = s[7]; + var m0 = MULT[0], m1 = MULT[1], m2 = MULT[2], m3 = MULT[3]; + var m4 = MULT[4], m5 = MULT[5], m6 = MULT[6], m7 = MULT[7]; + // state = state * MULT + inc, mod 2^128. Each column sums at most eight + // products below 2^32, plus a carry, so it stays exact. + var t, carry; + t = s0 * m0 + inc[0]; + s[0] = t % 65536; carry = (t - s[0]) / 65536; + t = s0 * m1 + s1 * m0 + inc[1] + carry; + s[1] = t % 65536; carry = (t - s[1]) / 65536; + t = s0 * m2 + s1 * m1 + s2 * m0 + inc[2] + carry; + s[2] = t % 65536; carry = (t - s[2]) / 65536; + t = s0 * m3 + s1 * m2 + s2 * m1 + s3 * m0 + inc[3] + carry; + s[3] = t % 65536; carry = (t - s[3]) / 65536; + t = s0 * m4 + s1 * m3 + s2 * m2 + s3 * m1 + s4 * m0 + inc[4] + carry; + s[4] = t % 65536; carry = (t - s[4]) / 65536; + t = s0 * m5 + s1 * m4 + s2 * m3 + s3 * m2 + s4 * m1 + s5 * m0 + inc[5] + carry; + s[5] = t % 65536; carry = (t - s[5]) / 65536; + t = s0 * m6 + s1 * m5 + s2 * m4 + s3 * m3 + s4 * m2 + s5 * m1 + s6 * m0 + inc[6] + carry; + s[6] = t % 65536; carry = (t - s[6]) / 65536; + t = s0 * m7 + s1 * m6 + s2 * m5 + s3 * m4 + s4 * m3 + s5 * m2 + s6 * m1 + s7 * m0 + + inc[7] + carry; + s[7] = t % 65536; + + // XSL-RR: rotate (high ^ low) right by the top six bits of the state. + var xh = ((s[7] << 16) | s[6]) ^ ((s[3] << 16) | s[2]); + var xl = ((s[5] << 16) | s[4]) ^ ((s[1] << 16) | s[0]); + var rot = s[7] >>> 10; + if (rot >= 32) { + var swap = xh; xh = xl; xl = swap; + rot -= 32; + } + if (rot === 0) { + this.hi = xh >>> 0; + this.lo = xl >>> 0; + } else { + this.hi = ((xh >>> rot) | (xl << (32 - rot))) >>> 0; + this.lo = ((xl >>> rot) | (xh << (32 - rot))) >>> 0; + } + }; + + // numpy's next_double: the top 53 bits of a 64-bit draw, times 2^-53. + Pcg64.prototype.nextDouble = function () { + this.next(); + return (this.hi * 2097152 + (this.lo >>> 11)) * 1.1102230246251565e-16; + }; + + function generator(seed) { + var initial = SEED_STATES[seed]; + if (!initial || !initial.state) throw new Error("no recorded PCG64 state for seed " + seed); + return new Pcg64(initial.state, initial.inc); + } + + /* -------------------------------------------------------- distributions */ + + // Line for line from numpy/random/src/distributions/distributions.c. + + function standardExponential(g) { + for (;;) { + g.next(); + var idx = (g.lo >>> 3) & 0xff; + var ri = g.hi * 2097152 + (g.lo >>> 11); // the 64-bit draw >> 11 + var x = ri * WE[idx]; + if (ri < KE[idx]) return x; + if (idx === 0) return EXP_R - Math.log1p(-g.nextDouble()); + if ((FE[idx - 1] - FE[idx]) * g.nextDouble() + FE[idx] < Math.exp(-x)) return x; + // numpy recurses here; a loop draws the same sequence. + } + } + + function standardNormal(g) { + for (;;) { + g.next(); + var lo = g.lo; + var idx = lo & 0xff; + var sign = (lo >>> 8) & 1; + // (draw >> 9) masked to 52 bits: bits 9..60 of the draw. + var rabs = (g.hi & 0x1fffffff) * 8388608 + (lo >>> 9); + var x = rabs * WI[idx]; + if (sign) x = -x; + if (rabs < KI[idx]) return x; + if (idx === 0) { + for (;;) { + var xx = -NOR_INV_R * Math.log1p(-g.nextDouble()); + var yy = -Math.log1p(-g.nextDouble()); + if (yy + yy > xx * xx) return (lo >>> 17) & 1 ? -(NOR_R + xx) : NOR_R + xx; + } + } else if ((FI[idx - 1] - FI[idx]) * g.nextDouble() + FI[idx] < Math.exp(-0.5 * x * x)) { + return x; + } + } + } + + function standardGamma(g, shape) { + var U, V, X, Y; + if (shape === 1.0) return standardExponential(g); + if (shape === 0.0) return 0.0; + if (shape < 1.0) { + for (;;) { + U = g.nextDouble(); + V = standardExponential(g); + if (U <= 1.0 - shape) { + X = Math.pow(U, 1.0 / shape); + if (X <= V) return X; + } else { + Y = -Math.log((1 - U) / shape); + X = Math.pow(1.0 - shape + shape * Y, 1.0 / shape); + if (X <= V + Y) return X; + } + } + } + var b = shape - 1.0 / 3.0; + var c = 1.0 / Math.sqrt(9 * b); + for (;;) { + do { + X = standardNormal(g); + V = 1.0 + c * X; + } while (V <= 0.0); + V = V * V * V; + U = g.nextDouble(); + if (U < 1.0 - 0.0331 * (X * X) * (X * X)) return b * V; + if (Math.log(U) < 0.5 * X * X + b * (1.0 - V + Math.log(V))) return b * V; + } + } + + function beta(g, a, b) { + // numpy switches to Johnk's algorithm when both shapes are at most 1. + // synthetic_floor's shapes always sum to its concentration of 6, so that + // branch is unreachable here and is refused rather than ported untested. + if (a <= 1.0 && b <= 1.0) throw new Error("beta shapes both <= 1 are not supported"); + var ga = standardGamma(g, a); + var gb = standardGamma(g, b); + return ga / (ga + gb); + } + + /* ---------------------------------------------------------------- floor */ + + var DEFAULT_N_BINS = 10; + var DEFAULT_N_BOOT = 2000; + var DEFAULT_CONCENTRATION = 6.0; + + // np.percentile(values, q) with the default linear method. + function percentile(values, q) { + var sorted = Float64Array.from(values).sort(); + var n = sorted.length; + var quantile = q / 100; + var virtual = n * quantile + (1 + quantile * (1 - 1 - 1)) - 1; + var below = Math.floor(virtual); + var gamma = virtual - below; + if (below < 0) return sorted[0]; + if (below >= n - 1) return sorted[n - 1]; + var a = sorted[below]; + var b = sorted[below + 1]; + var diff = b - a; + return gamma >= 0.5 ? b - diff * (1 - gamma) : a + diff * gamma; + } + + function mean(values) { + var total = 0; + for (var i = 0; i < values.length; i++) total += values[i]; + return total / values.length; + } + + /** + * The ECE a perfectly calibrated model scores on n rows, from summary numbers. + * + * Mirrors synthetic_floor(n, n_bins, accuracy) with equal-width bins: invent n + * predicted probabilities from Beta(accuracy * 6, (1 - accuracy) * 6), then + * redraw every outcome from Bernoulli(p_i) n_boot times and read the ECE of + * each resample. Returns { mean, p95, n, nBins, nBoot }. + * + * options.onProgress(fraction) is called during the bootstrap, for the page. + */ + function syntheticFloor(n, nBins, accuracy, options) { + options = options || {}; + var nBoot = options.nBoot || DEFAULT_N_BOOT; + var concentration = options.concentration || DEFAULT_CONCENTRATION; + var onProgress = options.onProgress; + nBins = nBins || DEFAULT_N_BINS; + + if (!(Number.isInteger(n) && n >= 1)) throw new Error("n must be a whole number of rows, at least 1"); + if (!(Number.isInteger(nBins) && nBins >= 1)) throw new Error("the bin count must be a whole number, at least 1"); + if (!(accuracy > 0 && accuracy < 1)) throw new Error("accuracy must lie strictly inside (0, 1)"); + + var seed = 0; + var g = generator(seed); + var a = accuracy * concentration; + var b = (1.0 - accuracy) * concentration; + var probabilities = new Float64Array(n); + for (var i = 0; i < n; i++) probabilities[i] = beta(g, a, b); + + // _bootstrap_floor, equal-width binning. + var index = new Int32Array(n); + var counts = new Float64Array(nBins); + var summed = new Float64Array(nBins); + for (i = 0; i < n; i++) { + var bin = Math.floor(probabilities[i] * nBins); + if (bin < 0) bin = 0; + if (bin > nBins - 1) bin = nBins - 1; + index[i] = bin; + counts[bin] += 1; + summed[bin] += probabilities[i]; + } + var meanProbability = new Float64Array(nBins); + for (var k = 0; k < nBins; k++) { + if (counts[k] > 0) meanProbability[k] = summed[k] / counts[k]; + } + + var boot = generator(seed + 1); + var draws = new Float64Array(nBoot); + var correct = new Float64Array(nBins); + var every = Math.max(1, Math.floor(nBoot / 50)); + for (var r = 0; r < nBoot; r++) { + correct.fill(0); + for (i = 0; i < n; i++) { + if (boot.nextDouble() < probabilities[i]) correct[index[i]] += 1; + } + var weighted = 0; + for (k = 0; k < nBins; k++) { + if (counts[k] > 0) { + weighted += Math.abs(correct[k] / counts[k] - meanProbability[k]) * counts[k]; + } + } + draws[r] = weighted / n; + if (onProgress && r % every === 0) onProgress(r / nBoot); + } + if (onProgress) onProgress(1); + + return { metric: "ece", mean: mean(draws), p95: percentile(draws, 95), n: n, nBins: nBins, nBoot: nBoot }; + } + + /* -------------------------------------------------------------- wording */ + + // Python's "%.4f": round half to even on the exact binary value. toFixed + // rounds exact ties up, which differs only when the value is a tie exactly. + function fixed4(x) { + var scaled = x * 10000; + if (Math.abs(scaled) < 1e15 && scaled - Math.floor(scaled) === 0.5 && x * 10000 === scaled) { + var down = Math.floor(scaled); + var even = down % 2 === 0 ? down : down + 1; + return (even / 10000).toFixed(4); + } + return x.toFixed(4); + } + + function isDistinguishable(measured, floor) { + return measured > floor.p95; + } + + // plumbline.metrics.calibration._judgment, word for word. + function judgment(measured, floor) { + if (isDistinguishable(measured, floor)) { + return "miscalibration is distinguishable from sampling noise."; + } + return ( + "INCONCLUSIVE at this sample size. A perfectly calibrated model would " + + "often score this badly on this many rows, so this dataset cannot tell " + + "the two apart. This is not a clean bill of health: nothing was " + + "established either way. Collect more rows to make the question " + + "answerable." + ); + } + + // CalibrationFigure.statement() for an ECE figure with equal-width bins. + function statement(measured, floor) { + return ( + "ECE " + fixed4(measured) + " over " + floor.n + " rows (" + floor.nBins + + " equal width bins), against a calibrated-model floor of " + fixed4(floor.mean) + + " (95th percentile " + fixed4(floor.p95) + "): " + judgment(measured, floor) + ); + } + + /* --------------------------------------------------------------- tables */ + + var NOR_R = 3.6541528853610087963519472518; + var NOR_INV_R = 0.27366123732975827203338247596; + var EXP_R = 7.6971174701310497140446280481; + + var KI = [ + 0x000ef33d8025ef6a, 0x0000000000000000, 0x000c08be98fbc6a8, 0x000da354fabd8142, + 0x000e51f67ec1eeea, 0x000eb255e9d3f77e, 0x000eef4b817ecab9, 0x000f19470afa44aa, + 0x000f37ed61ffcb18, 0x000f4f469561255c, 0x000f61a5e41ba396, 0x000f707a755396a4, + 0x000f7cb2ec28449a, 0x000f86f10c6357d3, 0x000f8fa6578325de, 0x000f9724c74dd0da, + 0x000f9da907dbf509, 0x000fa360f581fa74, 0x000fa86fde5b4bf8, 0x000facf160d354dc, + 0x000fb0fb6718b90f, 0x000fb49f8d5374c6, 0x000fb7ec2366fe77, 0x000fbaece9a1e50e, + 0x000fbdab9d040bed, 0x000fc03060ff6c57, 0x000fc2821037a248, 0x000fc4a67ae25bd1, + 0x000fc6a2977aee31, 0x000fc87aa92896a4, 0x000fca325e4bde85, 0x000fcbcce902231a, + 0x000fcd4d12f839c4, 0x000fceb54d8fec99, 0x000fd007bf1dc930, 0x000fd1464dd6c4e6, + 0x000fd272a8e2f450, 0x000fd38e4ff0c91e, 0x000fd49a9990b478, 0x000fd598b8920f53, + 0x000fd689c08e99ec, 0x000fd76ea9c8e832, 0x000fd848547b08e8, 0x000fd9178bad2c8c, + 0x000fd9dd07a7add2, 0x000fda9970105e8c, 0x000fdb4d5dc02e20, 0x000fdbf95c5bfcd0, + 0x000fdc9debb99a7d, 0x000fdd3b8118729d, 0x000fddd288342f90, 0x000fde6364369f64, + 0x000fdeee708d514e, 0x000fdf7401a6b42e, 0x000fdff46599ed40, 0x000fe06fe4bc24f2, + 0x000fe0e6c225a258, 0x000fe1593c28b84c, 0x000fe1c78cbc3f99, 0x000fe231e9db1caa, + 0x000fe29885da1b91, 0x000fe2fb8fb54186, 0x000fe35b33558d4a, 0x000fe3b799d0002a, + 0x000fe410e99ead7f, 0x000fe46746d47734, 0x000fe4bad34c095c, 0x000fe50baed29524, + 0x000fe559f74ebc78, 0x000fe5a5c8e41212, 0x000fe5ef3e138689, 0x000fe6366fd91078, + 0x000fe67b75c6d578, 0x000fe6be661e11aa, 0x000fe6ff55e5f4f2, 0x000fe73e5900a702, + 0x000fe77b823e9e39, 0x000fe7b6e37070a2, 0x000fe7f08d774243, 0x000fe8289053f08c, + 0x000fe85efb35173a, 0x000fe893dc840864, 0x000fe8c741f0cebc, 0x000fe8f9387d4ef6, + 0x000fe929cc879b1d, 0x000fe95909d388ea, 0x000fe986fb939aa2, 0x000fe9b3ac714866, + 0x000fe9df2694b6d5, 0x000fea0973abe67c, 0x000fea329cf166a4, 0x000fea5aab32952c, + 0x000fea81a6d5741a, 0x000feaa797de1cf0, 0x000feacc85f3d920, 0x000feaf07865e63c, + 0x000feb13762fec13, 0x000feb3585fe2a4a, 0x000feb56ae3162b4, 0x000feb76f4e284fa, + 0x000feb965fe62014, 0x000febb4f4cf9d7c, 0x000febd2b8f449d0, 0x000febefb16e2e3e, + 0x000fec0be31ebde8, 0x000fec2752b15a15, 0x000fec42049dafd3, 0x000fec5bfd29f196, + 0x000fec75406ceef4, 0x000fec8dd2500cb4, 0x000feca5b6911f12, 0x000fecbcf0c427fe, + 0x000fecd38454fb15, 0x000fece97488c8b3, 0x000fecfec47f91b7, 0x000fed1377358528, + 0x000fed278f844903, 0x000fed3b10242f4c, 0x000fed4dfbad586e, 0x000fed605498c3dd, + 0x000fed721d414fe8, 0x000fed8357e4a982, 0x000fed9406a42cc8, 0x000feda42b85b704, + 0x000fedb3c8746ab4, 0x000fedc2df416652, 0x000fedd171a46e52, 0x000feddf813c8ad3, + 0x000feded0f909980, 0x000fedfa1e0fd414, 0x000fee06ae124bc4, 0x000fee12c0d95a06, + 0x000fee1e579006e0, 0x000fee29734b6524, 0x000fee34150ae4bc, 0x000fee3e3db89b3c, + 0x000fee47ee2982f4, 0x000fee51271db086, 0x000fee59e9407f41, 0x000fee623528b42e, + 0x000fee6a0b5897f1, 0x000fee716c3e077a, 0x000fee7858327b82, 0x000fee7ecf7b06ba, + 0x000fee84d2484ab2, 0x000fee8a60b66343, 0x000fee8f7accc851, 0x000fee94207e25da, + 0x000fee9851a829ea, 0x000fee9c0e13485c, 0x000fee9f557273f4, 0x000feea22762ccae, + 0x000feea4836b42ac, 0x000feea668fc2d71, 0x000feea7d76ed6fa, 0x000feea8ce04fa0a, + 0x000feea94be8333b, 0x000feea950296410, 0x000feea8d9c0075e, 0x000feea7e7897654, + 0x000feea678481d24, 0x000feea48aa29e83, 0x000feea21d22e4da, 0x000fee9f2e352024, + 0x000fee9bbc26af2e, 0x000fee97c524f2e4, 0x000fee93473c0a3a, 0x000fee8e40557516, + 0x000fee88ae369c7a, 0x000fee828e7f3dfd, 0x000fee7bdea7b888, 0x000fee749bff37ff, + 0x000fee6cc3a9bd5e, 0x000fee64529e007e, 0x000fee5b45a32888, 0x000fee51994e57b6, + 0x000fee474a0006cf, 0x000fee3c53e12c50, 0x000fee30b2e02ad8, 0x000fee2462ad8205, + 0x000fee175eb83c5a, 0x000fee09a22a1447, 0x000fedfb27e349cc, 0x000fedebea76216c, + 0x000feddbe422047e, 0x000fedcb0ece39d3, 0x000fedb964042cf4, 0x000feda6dce938c9, + 0x000fed937237e98d, 0x000fed7f1c38a836, 0x000fed69d2b9c02b, 0x000fed538d06ae00, + 0x000fed3c41dea422, 0x000fed23e76a2fd8, 0x000fed0a732fe644, 0x000fecefda07fe34, + 0x000fecd4100eb7b8, 0x000fecb708956eb4, 0x000fec98b61230c1, 0x000fec790a0da978, + 0x000fec57f50f31fe, 0x000fec356686c962, 0x000fec114cb4b335, 0x000febeb948e6fd0, + 0x000febc429a0b692, 0x000feb9af5ee0cdc, 0x000feb6fe1c98542, 0x000feb42d3ad1f9e, + 0x000feb13b00b2d4b, 0x000feae2591a02e9, 0x000feaaeae992257, 0x000fea788d8ee326, + 0x000fea3fcffd73e5, 0x000fea044c8dd9f6, 0x000fe9c5d62f563b, 0x000fe9843ba947a4, + 0x000fe93f471d4728, 0x000fe8f6bd76c5d6, 0x000fe8aa5dc4e8e6, 0x000fe859e07ab1ea, + 0x000fe804f690a940, 0x000fe7ab488233c0, 0x000fe74c751f6aa5, 0x000fe6e8102aa202, + 0x000fe67da0b6abd8, 0x000fe60c9f38307e, 0x000fe5947338f742, 0x000fe51470977280, + 0x000fe48bd436f458, 0x000fe3f9bffd1e37, 0x000fe35d35eeb19c, 0x000fe2b5122fe4fe, + 0x000fe20003995557, 0x000fe13c82788314, 0x000fe068c4ee67b0, 0x000fdf82b02b71aa, + 0x000fde87c57efeaa, 0x000fdd7509c63bfd, 0x000fdc46e529bf13, 0x000fdaf8f82e0282, + 0x000fd985e1b2ba75, 0x000fd7e6ef48cf04, 0x000fd613adbd650b, 0x000fd40149e2f012, + 0x000fd1a1a7b4c7ac, 0x000fcee204761f9e, 0x000fcba8d85e11b2, 0x000fc7d26ecd2d22, + 0x000fc32b2f1e22ed, 0x000fbd6581c0b83a, 0x000fb606c4005434, 0x000fac40582a2874, + 0x000f9e971e014598, 0x000f89fa48a41dfc, 0x000f66c5f7f0302c, 0x000f1a5a4b331c4a, + ]; + var WI = [ + 8.68362706080130616677e-16, 4.77933017572773682428e-17, 6.35435241740526230246e-17, + 7.45487048124769627714e-17, 8.32936681579309972857e-17, 9.06806040505948228243e-17, + 9.71486007656776183958e-17, 1.02947503142410192108e-16, 1.08234302884476839838e-16, + 1.13114701961090307945e-16, 1.17663594570229211411e-16, 1.21936172787143633280e-16, + 1.25974399146370927864e-16, 1.29810998862640315416e-16, 1.33472037368241227547e-16, + 1.36978648425712032797e-16, 1.40348230012423820659e-16, 1.43595294520569430270e-16, + 1.46732087423644219083e-16, 1.49769046683910367425e-16, 1.52715150035961979750e-16, + 1.55578181694607639484e-16, 1.58364940092908853989e-16, 1.61081401752749279325e-16, + 1.63732852039698532012e-16, 1.66323990584208352778e-16, 1.68859017086765964015e-16, + 1.71341701765596607184e-16, 1.73775443658648593310e-16, 1.76163319230009959832e-16, + 1.78508123169767272927e-16, 1.80812402857991522674e-16, 1.83078487648267501776e-16, + 1.85308513886180189386e-16, 1.87504446393738816849e-16, 1.89668097007747596212e-16, + 1.91801140648386198029e-16, 1.93905129306251037069e-16, 1.95981504266288244037e-16, + 1.98031606831281739736e-16, 2.00056687762733300198e-16, 2.02057915620716538808e-16, + 2.04036384154802118313e-16, 2.05993118874037063144e-16, 2.07929082904140197311e-16, + 2.09845182223703516690e-16, 2.11742270357603418769e-16, 2.13621152594498681022e-16, + 2.15482589785814580926e-16, 2.17327301775643674990e-16, 2.19155970504272708519e-16, + 2.20969242822353175995e-16, 2.22767733047895534948e-16, 2.24552025294143552381e-16, + 2.26322675592856786566e-16, 2.28080213834501706782e-16, 2.29825145544246839061e-16, + 2.31557953510408037008e-16, 2.33279099280043561128e-16, 2.34989024534709550938e-16, + 2.36688152357916037468e-16, 2.38376888404542434981e-16, 2.40055621981350627349e-16, + 2.41724727046750252175e-16, 2.43384563137110286400e-16, 2.45035476226149539878e-16, + 2.46677799523270498158e-16, 2.48311854216108767769e-16, 2.49937950162045242375e-16, + 2.51556386532965786439e-16, 2.53167452417135826983e-16, 2.54771427381694417303e-16, + 2.56368581998939683749e-16, 2.57959178339286723500e-16, 2.59543470433517070146e-16, + 2.61121704706701939097e-16, 2.62694120385972564623e-16, 2.64260949884118951286e-16, + 2.65822419160830680292e-16, 2.67378748063236329361e-16, 2.68930150647261591777e-16, + 2.70476835481199518794e-16, 2.72019005932773206655e-16, 2.73556860440867908686e-16, + 2.75090592773016664571e-16, 2.76620392269639032183e-16, 2.78146444075954410103e-16, + 2.79668929362423005309e-16, 2.81188025534502074329e-16, 2.82703906432447923059e-16, + 2.84216742521840606520e-16, 2.85726701075460149289e-16, 2.87233946347097994381e-16, + 2.88738639737848191815e-16, 2.90240939955384233230e-16, 2.91741003166694553259e-16, + 2.93238983144718163965e-16, 2.94735031409293489611e-16, 2.96229297362806647792e-16, + 2.97721928420902891115e-16, 2.99213070138601307081e-16, 3.00702866332133102993e-16, + 3.02191459196806151971e-16, 3.03678989421180184427e-16, 3.05165596297821922381e-16, + 3.06651417830895451744e-16, 3.08136590840829717032e-16, 3.09621251066292253306e-16, + 3.11105533263689296831e-16, 3.12589571304399892784e-16, 3.14073498269944617203e-16, + 3.15557446545280064031e-16, 3.17041547910402852545e-16, 3.18525933630440648871e-16, + 3.20010734544401137886e-16, 3.21496081152744704901e-16, 3.22982103703941557538e-16, + 3.24468932280169778077e-16, 3.25956696882307838340e-16, 3.27445527514370671802e-16, + 3.28935554267536967851e-16, 3.30426907403912838589e-16, 3.31919717440175233652e-16, + 3.33414115231237245918e-16, 3.34910232054077845412e-16, 3.36408199691876507948e-16, + 3.37908150518594979994e-16, 3.39410217584148914282e-16, 3.40914534700312603713e-16, + 3.42421236527501816058e-16, 3.43930458662583133920e-16, 3.45442337727858401604e-16, + 3.46957011461378353333e-16, 3.48474618808741370700e-16, 3.49995300016538099813e-16, + 3.51519196727607440975e-16, 3.53046452078274009054e-16, 3.54577210797743572160e-16, + 3.56111619309838843415e-16, 3.57649825837265051035e-16, 3.59191980508602994994e-16, + 3.60738235468235137839e-16, 3.62288744989419151904e-16, 3.63843665590734438546e-16, + 3.65403156156136995766e-16, 3.66967378058870090021e-16, 3.68536495289491401456e-16, + 3.70110674588289834952e-16, 3.71690085582382297792e-16, 3.73274900927794352614e-16, + 3.74865296456848868882e-16, 3.76461451331202869131e-16, 3.78063548200896037651e-16, + 3.79671773369794425924e-16, 3.81286316967837738238e-16, 3.82907373130524317507e-16, + 3.84535140186095955858e-16, 3.86169820850914927119e-16, 3.87811622433558721164e-16, + 3.89460757048192620674e-16, 3.91117441837820542060e-16, 3.92781899208054153270e-16, + 3.94454357072087711446e-16, 3.96135049107613542983e-16, 3.97824215026468259474e-16, + 3.99522100857856502444e-16, 4.01228959246062907451e-16, 4.02945049763632792393e-16, + 4.04670639241074995115e-16, 4.06406002114225038723e-16, 4.08151420790493873480e-16, + 4.09907186035326643447e-16, 4.11673597380302570170e-16, 4.13450963554423599878e-16, + 4.15239602940268833891e-16, 4.17039844056831587498e-16, 4.18852026071011229572e-16, + 4.20676499339901510978e-16, 4.22513625986204937320e-16, 4.24363780509307796137e-16, + 4.26227350434779809917e-16, 4.28104737005311666397e-16, 4.29996355916383230161e-16, + 4.31902638100262944617e-16, 4.33824030562279080411e-16, 4.35760997273684900553e-16, + 4.37714020125858747008e-16, 4.39683599951052137423e-16, 4.41670257615420348435e-16, + 4.43674535190656726604e-16, 4.45696997211204306674e-16, 4.47738232024753387312e-16, + 4.49798853244554968009e-16, 4.51879501313005876278e-16, 4.53980845187003400947e-16, + 4.56103584156742206384e-16, 4.58248449810956667052e-16, 4.60416208163115281428e-16, + 4.62607661954784567754e-16, 4.64823653154320737780e-16, 4.67065065671263059081e-16, + 4.69332828309332890697e-16, 4.71627917983835129766e-16, 4.73951363232586715165e-16, + 4.76304248053313737663e-16, 4.78687716104872284247e-16, 4.81102975314741720538e-16, + 4.83551302941152515162e-16, 4.86034051145081195402e-16, 4.88552653135360343280e-16, + 4.91108629959526955862e-16, 4.93703598024033454728e-16, 4.96339277440398725619e-16, + 4.99017501309182245754e-16, 5.01740226071808946011e-16, 5.04509543081872748637e-16, + 5.07327691573354207058e-16, 5.10197073234156184149e-16, 5.13120268630678373200e-16, + 5.16100055774322824569e-16, 5.19139431175769859873e-16, 5.22241633800023428760e-16, + 5.25410172417759732697e-16, 5.28648856950494511482e-16, 5.31961834533840037535e-16, + 5.35353631181649688145e-16, 5.38829200133405320160e-16, 5.42393978220171234073e-16, + 5.46053951907478041166e-16, 5.49815735089281410703e-16, 5.53686661246787600374e-16, + 5.57674893292657647836e-16, 5.61789555355541665830e-16, 5.66040892008242216739e-16, + 5.70440462129138908417e-16, 5.75001376891989523684e-16, 5.79738594572459365014e-16, + 5.84669289345547900201e-16, 5.89813317647789942685e-16, 5.95193814964144415532e-16, + 6.00837969627190832234e-16, 6.06778040933344851394e-16, 6.13052720872528159123e-16, + 6.19708989458162555387e-16, 6.26804696330128439415e-16, 6.34412240712750598627e-16, + 6.42623965954805540945e-16, 6.51560331734499356881e-16, 6.61382788509766415145e-16, + 6.72315046250558662913e-16, 6.84680341756425875856e-16, 6.98971833638761995415e-16, + 7.15999493483066421560e-16, 7.37242430179879890722e-16, 7.65893637080557275482e-16, + 8.11384933765648418565e-16, + ]; + var FI = [ + 1.00000000000000000000e+00, 9.77101701267671596263e-01, 9.59879091800106665211e-01, + 9.45198953442299649730e-01, 9.32060075959230460718e-01, 9.19991505039347012840e-01, + 9.08726440052130879366e-01, 8.98095921898343418910e-01, 8.87984660755833377088e-01, + 8.78309655808917399966e-01, 8.69008688036857046555e-01, 8.60033621196331532488e-01, + 8.51346258458677951353e-01, 8.42915653112204177333e-01, 8.34716292986883434679e-01, + 8.26726833946221373317e-01, 8.18929191603702366642e-01, 8.11307874312656274185e-01, + 8.03849483170964274059e-01, 7.96542330422958966274e-01, 7.89376143566024590648e-01, + 7.82341832654802504798e-01, 7.75431304981187174974e-01, 7.68637315798486264740e-01, + 7.61953346836795386565e-01, 7.55373506507096115214e-01, 7.48892447219156820459e-01, + 7.42505296340151055290e-01, 7.36207598126862650112e-01, 7.29995264561476231435e-01, + 7.23864533468630222401e-01, 7.17811932630721960535e-01, 7.11834248878248421200e-01, + 7.05928501332754310127e-01, 7.00091918136511615067e-01, 6.94321916126116711609e-01, + 6.88616083004671808432e-01, 6.82972161644994857355e-01, 6.77388036218773526009e-01, + 6.71861719897082099173e-01, 6.66391343908750100056e-01, 6.60975147776663107813e-01, + 6.55611470579697264149e-01, 6.50298743110816701574e-01, 6.45035480820822293424e-01, + 6.39820277453056585060e-01, 6.34651799287623608059e-01, 6.29528779924836690007e-01, + 6.24450015547026504592e-01, 6.19414360605834324325e-01, 6.14420723888913888899e-01, + 6.09468064925773433949e-01, 6.04555390697467776029e-01, 5.99681752619125263415e-01, + 5.94846243767987448159e-01, 5.90047996332826008015e-01, 5.85286179263371453274e-01, + 5.80559996100790898232e-01, 5.75868682972353718164e-01, 5.71211506735253227163e-01, + 5.66587763256164445025e-01, 5.61996775814524340831e-01, 5.57437893618765945014e-01, + 5.52910490425832290562e-01, 5.48413963255265812791e-01, 5.43947731190026262382e-01, + 5.39511234256952132426e-01, 5.35103932380457614215e-01, 5.30725304403662057062e-01, + 5.26374847171684479008e-01, 5.22052074672321841931e-01, 5.17756517229756352272e-01, + 5.13487720747326958914e-01, 5.09245245995747941592e-01, 5.05028667943468123624e-01, + 5.00837575126148681903e-01, 4.96671569052489714213e-01, 4.92530263643868537748e-01, + 4.88413284705458028423e-01, 4.84320269426683325253e-01, 4.80250865909046753544e-01, + 4.76204732719505863248e-01, 4.72181538467730199660e-01, 4.68180961405693596422e-01, + 4.64202689048174355069e-01, 4.60246417812842867345e-01, 4.56311852678716434184e-01, + 4.52398706861848520777e-01, 4.48506701507203064949e-01, 4.44635565395739396077e-01, + 4.40785034665803987508e-01, 4.36954852547985550526e-01, 4.33144769112652261445e-01, + 4.29354541029441427735e-01, 4.25583931338021970170e-01, 4.21832709229495894654e-01, + 4.18100649837848226120e-01, 4.14387534040891125642e-01, 4.10693148270188157500e-01, + 4.07017284329473372217e-01, 4.03359739221114510510e-01, 3.99720314980197222177e-01, + 3.96098818515832451492e-01, 3.92495061459315619512e-01, 3.88908860018788715696e-01, + 3.85340034840077283462e-01, 3.81788410873393657674e-01, 3.78253817245619183840e-01, + 3.74736087137891138443e-01, 3.71235057668239498696e-01, 3.67750569779032587814e-01, + 3.64282468129004055601e-01, 3.60830600989648031529e-01, 3.57394820145780500731e-01, + 3.53974980800076777232e-01, 3.50570941481406106455e-01, 3.47182563956793643900e-01, + 3.43809713146850715049e-01, 3.40452257044521866547e-01, 3.37110066637006045021e-01, + 3.33783015830718454708e-01, 3.30470981379163586400e-01, 3.27173842813601400970e-01, + 3.23891482376391093290e-01, 3.20623784956905355514e-01, 3.17370638029913609834e-01, + 3.14131931596337177215e-01, 3.10907558126286509559e-01, 3.07697412504292056035e-01, + 3.04501391976649993243e-01, 3.01319396100803049698e-01, 2.98151326696685481377e-01, + 2.94997087799961810184e-01, 2.91856585617095209972e-01, 2.88729728482182923521e-01, + 2.85616426815501756042e-01, 2.82516593083707578948e-01, 2.79430141761637940157e-01, + 2.76356989295668320494e-01, 2.73297054068577072172e-01, 2.70250256365875463072e-01, + 2.67216518343561471038e-01, 2.64195763997261190426e-01, 2.61187919132721213522e-01, + 2.58192911337619235290e-01, 2.55210669954661961700e-01, 2.52241126055942177508e-01, + 2.49284212418528522415e-01, 2.46339863501263828249e-01, 2.43408015422750312329e-01, + 2.40488605940500588254e-01, 2.37581574431238090606e-01, 2.34686861872330010392e-01, + 2.31804410824338724684e-01, 2.28934165414680340644e-01, 2.26076071322380278694e-01, + 2.23230075763917484855e-01, 2.20396127480151998723e-01, 2.17574176724331130872e-01, + 2.14764175251173583536e-01, 2.11966076307030182324e-01, 2.09179834621125076977e-01, + 2.06405406397880797353e-01, 2.03642749310334908452e-01, 2.00891822494656591136e-01, + 1.98152586545775138971e-01, 1.95425003514134304483e-01, 1.92709036903589175926e-01, + 1.90004651670464985713e-01, 1.87311814223800304768e-01, 1.84630492426799269756e-01, + 1.81960655599522513892e-01, 1.79302274522847582272e-01, 1.76655321443734858455e-01, + 1.74019770081838553999e-01, 1.71395595637505754327e-01, 1.68782774801211288285e-01, + 1.66181285764481906364e-01, 1.63591108232365584074e-01, 1.61012223437511009516e-01, + 1.58444614155924284882e-01, 1.55888264724479197465e-01, 1.53343161060262855866e-01, + 1.50809290681845675763e-01, 1.48286642732574552861e-01, 1.45775208005994028060e-01, + 1.43274978973513461566e-01, 1.40785949814444699690e-01, 1.38308116448550733057e-01, + 1.35841476571253755301e-01, 1.33386029691669155683e-01, 1.30941777173644358090e-01, + 1.28508722279999570981e-01, 1.26086870220185887081e-01, 1.23676228201596571932e-01, + 1.21276805484790306533e-01, 1.18888613442910059947e-01, 1.16511665625610869035e-01, + 1.14145977827838487895e-01, 1.11791568163838089811e-01, 1.09448457146811797824e-01, + 1.07116667774683801961e-01, 1.04796225622487068629e-01, 1.02487158941935246892e-01, + 1.00189498768810017482e-01, 9.79032790388624646338e-02, 9.56285367130089991594e-02, + 9.33653119126910124859e-02, 9.11136480663737591268e-02, 8.88735920682758862021e-02, + 8.66451944505580717859e-02, 8.44285095703534715916e-02, 8.22235958132029043366e-02, + 8.00305158146630696292e-02, 7.78493367020961224423e-02, 7.56801303589271778804e-02, + 7.35229737139813238622e-02, 7.13779490588904025339e-02, 6.92451443970067553879e-02, + 6.71246538277884968737e-02, 6.50165779712428976156e-02, 6.29210244377581412456e-02, + 6.08381083495398780614e-02, 5.87679529209337372930e-02, 5.67106901062029017391e-02, + 5.46664613248889208474e-02, 5.26354182767921896513e-02, 5.06177238609477817000e-02, + 4.86135532158685421122e-02, 4.66230949019303814174e-02, 4.46465522512944634759e-02, + 4.26841449164744590750e-02, 4.07361106559409394401e-02, 3.88027074045261474722e-02, + 3.68842156885673053135e-02, 3.49809414617161251737e-02, 3.30932194585785779961e-02, + 3.12214171919203004046e-02, 2.93659397581333588001e-02, 2.75272356696031131329e-02, + 2.57058040085489103443e-02, 2.39022033057958785407e-02, 2.21170627073088502113e-02, + 2.03510962300445102935e-02, 1.86051212757246224594e-02, 1.68800831525431419000e-02, + 1.51770883079353092332e-02, 1.34974506017398673818e-02, 1.18427578579078790488e-02, + 1.02149714397014590439e-02, 8.61658276939872638800e-03, 7.05087547137322242369e-03, + 5.52240329925099155545e-03, 4.03797259336302356153e-03, 2.60907274610215926189e-03, + 1.26028593049859797236e-03, + ]; + var KE = [ + 0x001c5214272497c6, 0x0000000000000000, 0x00137d5bd79c317e, 0x00186ef58e3f3c10, + 0x001a9bb7320eb0ae, 0x001bd127f719447c, 0x001c951d0f88651a, 0x001d1bfe2d5c3972, + 0x001d7e5bd56b18b2, 0x001dc934dd172c70, 0x001e0409dfac9dc8, 0x001e337b71d47836, + 0x001e5a8b177cb7a2, 0x001e7b42096f046c, 0x001e970daf08ae3e, 0x001eaef5b14ef09e, + 0x001ec3bd07b46556, 0x001ed5f6f08799ce, 0x001ee614ae6e5688, 0x001ef46eca361cd0, + 0x001f014b76ddd4a4, 0x001f0ce313a796b6, 0x001f176369f1f77a, 0x001f20f20c452570, + 0x001f29ae1951a874, 0x001f31b18fb95532, 0x001f39125157c106, 0x001f3fe2eb6e694c, + 0x001f463332d788fa, 0x001f4c10bf1d3a0e, 0x001f51874c5c3322, 0x001f56a109c3ecc0, + 0x001f5b66d9099996, 0x001f5fe08210d08c, 0x001f6414dd445772, 0x001f6809f6859678, + 0x001f6bc52a2b02e6, 0x001f6f4b3d32e4f4, 0x001f72a07190f13a, 0x001f75c8974d09d6, + 0x001f78c71b045cc0, 0x001f7b9f12413ff4, 0x001f7e5346079f8a, 0x001f80e63be21138, + 0x001f835a3dad9162, 0x001f85b16056b912, 0x001f87ed89b24262, 0x001f8a10759374fa, + 0x001f8c1bba3d39ac, 0x001f8e10cc45d04a, 0x001f8ff102013e16, 0x001f91bd968358e0, + 0x001f9377ac47afd8, 0x001f95204f8b64da, 0x001f96b878633892, 0x001f98410c968892, + 0x001f99bae146ba80, 0x001f9b26bc697f00, 0x001f9c85561b717a, 0x001f9dd759cfd802, + 0x001f9f1d6761a1ce, 0x001fa058140936c0, 0x001fa187eb3a3338, 0x001fa2ad6f6bc4fc, + 0x001fa3c91ace0682, 0x001fa4db5fee6aa2, 0x001fa5e4aa4d097c, 0x001fa6e55ee46782, + 0x001fa7dddca51ec4, 0x001fa8ce7ce6a874, 0x001fa9b793ce5fee, 0x001faa9970adb858, + 0x001fab745e588232, 0x001fac48a3740584, 0x001fad1682bf9fe8, 0x001fadde3b5782c0, + 0x001faea008f21d6c, 0x001faf5c2418b07e, 0x001fb012c25b7a12, 0x001fb0c41681dff4, + 0x001fb17050b6f1fa, 0x001fb2179eb2963a, 0x001fb2ba2bdfa84a, 0x001fb358217f4e18, + 0x001fb3f1a6c9be0c, 0x001fb486e10cacd6, 0x001fb517f3c793fc, 0x001fb5a500c5fdaa, + 0x001fb62e2837fe58, 0x001fb6b388c9010a, 0x001fb7353fb50798, 0x001fb7b368dc7da8, + 0x001fb82e1ed6ba08, 0x001fb8a57b0347f6, 0x001fb919959a0f74, 0x001fb98a85ba7204, + 0x001fb9f861796f26, 0x001fba633deee286, 0x001fbacb2f41ec16, 0x001fbb3048b49144, + 0x001fbb929caea4e2, 0x001fbbf23cc8029e, 0x001fbc4f39d22994, 0x001fbca9a3e140d4, + 0x001fbd018a548f9e, 0x001fbd56fbde729c, 0x001fbdaa068bd66a, 0x001fbdfab7cb3f40, + 0x001fbe491c7364de, 0x001fbe9540c9695e, 0x001fbedf3086b128, 0x001fbf26f6de6174, + 0x001fbf6c9e828ae2, 0x001fbfb031a904c4, 0x001fbff1ba0ffdb0, 0x001fc03141024588, + 0x001fc06ecf5b54b2, 0x001fc0aa6d8b1426, 0x001fc0e42399698a, 0x001fc11bf9298a64, + 0x001fc151f57d1942, 0x001fc1861f770f4a, 0x001fc1b87d9e74b4, 0x001fc1e91620ea42, + 0x001fc217eed505de, 0x001fc2450d3c83fe, 0x001fc27076864fc2, 0x001fc29a2f90630e, + 0x001fc2c23ce98046, 0x001fc2e8a2d2c6b4, 0x001fc30d654122ec, 0x001fc33087de9c0e, + 0x001fc3520e0b7ec6, 0x001fc371fadf66f8, 0x001fc390512a2886, 0x001fc3ad137497fa, + 0x001fc3c844013348, 0x001fc3e1e4ccab40, 0x001fc3f9f78e4da8, 0x001fc4107db85060, + 0x001fc4257877fd68, 0x001fc438e8b5bfc6, 0x001fc44acf15112a, 0x001fc45b2bf447e8, + 0x001fc469ff6c4504, 0x001fc477495001b2, 0x001fc483092bfbb8, 0x001fc48d3e457ff6, + 0x001fc495e799d21a, 0x001fc49d03dd30b0, 0x001fc4a29179b432, 0x001fc4a68e8e07fc, + 0x001fc4a8f8ebfb8c, 0x001fc4a9ce16ea9e, 0x001fc4a90b41fa34, 0x001fc4a6ad4e28a0, + 0x001fc4a2b0c82e74, 0x001fc49d11e62de2, 0x001fc495cc852df4, 0x001fc48cdc265ec0, + 0x001fc4823bec237a, 0x001fc475e696dee6, 0x001fc467d6817e82, 0x001fc458059dc036, + 0x001fc4466d702e20, 0x001fc433070bcb98, 0x001fc41dcb0d6e0e, 0x001fc406b196bbf6, + 0x001fc3edb248cb62, 0x001fc3d2c43e593c, 0x001fc3b5de0591b4, 0x001fc396f599614c, + 0x001fc376005a4592, 0x001fc352f3069370, 0x001fc32dc1b22818, 0x001fc3065fbd7888, + 0x001fc2dcbfcbf262, 0x001fc2b0d3b99f9e, 0x001fc2828c8ffcf0, 0x001fc251da79f164, + 0x001fc21eacb6d39e, 0x001fc1e8f18c6756, 0x001fc1b09637bb3c, 0x001fc17586dccd10, + 0x001fc137ae74d6b6, 0x001fc0f6f6bb2414, 0x001fc0b348184da4, 0x001fc06c898baff0, + 0x001fc022a092f364, 0x001fbfd5710f72b8, 0x001fbf84dd29488e, 0x001fbf30c52fc60a, + 0x001fbed907770cc6, 0x001fbe7d80327dda, 0x001fbe1e094ba614, 0x001fbdba7a354408, + 0x001fbd52a7b9f826, 0x001fbce663c6201a, 0x001fbc757d2c4de4, 0x001fbbffbf63b7aa, + 0x001fbb84f23fe6a2, 0x001fbb04d9a0d18c, 0x001fba7f351a70ac, 0x001fb9f3bf92b618, + 0x001fb9622ed4abfc, 0x001fb8ca33174a16, 0x001fb82b76765b54, 0x001fb7859c5b895c, + 0x001fb6d840d55594, 0x001fb622f7d96942, 0x001fb5654c6f37e0, 0x001fb49ebfbf69d2, + 0x001fb3cec803e746, 0x001fb2f4cf539c3e, 0x001fb21032442852, 0x001fb1203e5a9604, + 0x001fb0243042e1c2, 0x001faf1b31c479a6, 0x001fae045767e104, 0x001facde9dbf2d72, + 0x001faba8e640060a, 0x001faa61f399ff28, 0x001fa908656f66a2, 0x001fa79ab3508d3c, + 0x001fa61726d1f214, 0x001fa47bd48bea00, 0x001fa2c693c5c094, 0x001fa0f4f47df314, + 0x001f9f04336bbe0a, 0x001f9cf12b79f9bc, 0x001f9ab84415abc4, 0x001f98555b782fb8, + 0x001f95c3abd03f78, 0x001f92fda9cef1f2, 0x001f8ffcda9ae41c, 0x001f8cb99e7385f8, + 0x001f892aec479606, 0x001f8545f904db8e, 0x001f80fdc336039a, 0x001f7c427839e926, + 0x001f7700a3582acc, 0x001f71200f1a241c, 0x001f6a8234b7352a, 0x001f630000a8e266, + 0x001f5a66904fe3c4, 0x001f50724ece1172, 0x001f44c7665c6fda, 0x001f36e5a38a59a2, + 0x001f26143450340a, 0x001f113e047b0414, 0x001ef6aefa57cbe6, 0x001ed38ca188151e, + 0x001ea2a61e122db0, 0x001e5961c78b267c, 0x001dddf62bac0bb0, 0x001cdb4dd9e4e8c0, + ]; + var WE = [ + 9.655740063209182975e-16, 7.089014243955414331e-18, 1.163941249669122378e-17, + 1.524391512353216015e-17, 1.833284885723743916e-17, 2.108965109464486630e-17, + 2.361128077843138196e-17, 2.595595772310893952e-17, 2.816173554197752338e-17, + 3.025504130321382330e-17, 3.225508254836375280e-17, 3.417632340185027033e-17, + 3.602996978734452488e-17, 3.782490776869649048e-17, 3.956832198097553231e-17, + 4.126611778175946428e-17, 4.292321808442525631e-17, 4.454377743282371417e-17, + 4.613133981483185932e-17, 4.768895725264635940e-17, 4.921928043727962847e-17, + 5.072462904503147014e-17, 5.220704702792671737e-17, 5.366834661718192181e-17, + 5.511014372835094717e-17, 5.653388673239667134e-17, 5.794088004852766616e-17, + 5.933230365208943081e-17, 6.070922932847179572e-17, 6.207263431163193485e-17, + 6.342341280303076511e-17, 6.476238575956142121e-17, 6.609030925769405241e-17, + 6.740788167872722244e-17, 6.871574991183812442e-17, 7.001451473403929616e-17, + 7.130473549660643409e-17, 7.258693422414648352e-17, 7.386159921381791997e-17, + 7.512918820723728089e-17, 7.639013119550825792e-17, 7.764483290797848102e-17, + 7.889367502729790548e-17, 8.013701816675454434e-17, 8.137520364041762206e-17, + 8.260855505210038174e-17, 8.383737972539139383e-17, 8.506196999385323132e-17, + 8.628260436784112996e-17, 8.749954859216182511e-17, 8.871305660690252281e-17, + 8.992337142215357066e-17, 9.113072591597909173e-17, 9.233534356381788123e-17, + 9.353743910649128938e-17, 9.473721916312949566e-17, 9.593488279457997317e-17, + 9.713062202221521206e-17, 9.832462230649511362e-17, 9.951706298915071878e-17, + 1.007081177024294931e-16, 1.018979547484694078e-16, 1.030867374515421954e-16, + 1.042746244856188556e-16, 1.054617701794576406e-16, 1.066483248011914702e-16, + 1.078344348241948498e-16, 1.090202431758350473e-16, 1.102058894705578110e-16, + 1.113915102286197502e-16, 1.125772390816567488e-16, 1.137632069661684705e-16, + 1.149495423059009298e-16, 1.161363711840218308e-16, 1.173238175059045788e-16, + 1.185120031532669434e-16, 1.197010481303465158e-16, 1.208910707027385520e-16, + 1.220821875294706151e-16, 1.232745137888415193e-16, 1.244681632985112523e-16, + 1.256632486302898513e-16, 1.268598812200397542e-16, 1.280581714730749379e-16, + 1.292582288654119552e-16, 1.304601620412028847e-16, 1.316640789066572582e-16, + 1.328700867207380889e-16, 1.340782921828999433e-16, 1.352888015181175458e-16, + 1.365017205594397770e-16, 1.377171548282880964e-16, 1.389352096127063919e-16, + 1.401559900437571538e-16, 1.413796011702485188e-16, 1.426061480319665444e-16, + 1.438357357315790180e-16, 1.450684695053687684e-16, 1.463044547929475721e-16, + 1.475437973060951633e-16, 1.487866030968626066e-16, 1.500329786250736949e-16, + 1.512830308253539427e-16, 1.525368671738125550e-16, 1.537945957544996933e-16, + 1.550563253257577148e-16, 1.563221653865837505e-16, 1.575922262431176140e-16, + 1.588666190753684151e-16, 1.601454560042916733e-16, 1.614288501593278662e-16, + 1.627169157465130500e-16, 1.640097681172717950e-16, 1.653075238380036909e-16, + 1.666103007605742067e-16, 1.679182180938228863e-16, 1.692313964762022267e-16, + 1.705499580496629830e-16, 1.718740265349031656e-16, 1.732037273081008369e-16, + 1.745391874792533975e-16, 1.758805359722491379e-16, 1.772279036068006489e-16, + 1.785814231823732619e-16, 1.799412295642463721e-16, 1.813074597718501559e-16, + 1.826802530695252266e-16, 1.840597510598587828e-16, 1.854460977797569461e-16, + 1.868394397994192684e-16, 1.882399263243892051e-16, 1.896477093008616722e-16, + 1.910629435244376536e-16, 1.924857867525243818e-16, 1.939163998205899420e-16, + 1.953549467624909132e-16, 1.968015949351037382e-16, 1.982565151475019047e-16, + 1.997198817949342081e-16, 2.011918729978734671e-16, 2.026726707464198289e-16, + 2.041624610503588774e-16, 2.056614340951917875e-16, 2.071697844044737034e-16, + 2.086877110088159721e-16, 2.102154176219292789e-16, 2.117531128241075913e-16, + 2.133010102535779087e-16, 2.148593288061663316e-16, 2.164282928437604723e-16, + 2.180081324120784027e-16, 2.195990834682870728e-16, 2.212013881190495942e-16, + 2.228152948696180545e-16, 2.244410588846308588e-16, 2.260789422613173739e-16, + 2.277292143158621037e-16, 2.293921518837311354e-16, 2.310680396348213318e-16, + 2.327571704043534613e-16, 2.344598455404957859e-16, 2.361763752697773994e-16, + 2.379070790814276700e-16, 2.396522861318623520e-16, 2.414123356706293277e-16, + 2.431875774892255956e-16, 2.449783723943070217e-16, 2.467850927069288738e-16, + 2.486081227895851719e-16, 2.504478596029557040e-16, 2.523047132944217013e-16, + 2.541791078205812227e-16, 2.560714816061770759e-16, 2.579822882420530896e-16, + 2.599119972249746917e-16, 2.618610947423924219e-16, 2.638300845054942823e-16, + 2.658194886341845120e-16, 2.678298485979525166e-16, 2.698617262169488933e-16, + 2.719157047279818500e-16, 2.739923899205814823e-16, 2.760924113487617126e-16, + 2.782164236246436081e-16, 2.803651078006983464e-16, 2.825391728480253184e-16, + 2.847393572388174091e-16, 2.869664306419817679e-16, 2.892211957417995598e-16, + 2.915044901905293183e-16, 2.938171887070028633e-16, 2.961602053345465687e-16, + 2.985344958730045276e-16, 3.009410605012618141e-16, 3.033809466085003416e-16, + 3.058552518544860874e-16, 3.083651274815310004e-16, 3.109117819034266344e-16, + 3.134964845996663118e-16, 3.161205703467105734e-16, 3.187854438219713117e-16, + 3.214925846206797361e-16, 3.242435527309451638e-16, 3.270399945182240440e-16, + 3.298836492772283149e-16, 3.327763564171671408e-16, 3.357200633553244075e-16, + 3.387168342045505162e-16, 3.417688593525636996e-16, 3.448784660453423890e-16, + 3.480481301037442286e-16, 3.512804889222979418e-16, 3.545783559224791863e-16, + 3.579447366604276541e-16, 3.613828468219060593e-16, 3.648961323764542545e-16, + 3.684882922095621322e-16, 3.721633036080207290e-16, 3.759254510416256532e-16, + 3.797793587668874387e-16, 3.837300278789213687e-16, 3.877828785607895292e-16, + 3.919437984311428867e-16, 3.962191980786774996e-16, 4.006160751056541688e-16, + 4.051420882956573177e-16, 4.098056438903062509e-16, 4.146159964290904582e-16, + 4.195833672073398926e-16, 4.247190841824385048e-16, 4.300357481667470702e-16, + 4.355474314693952008e-16, 4.412699169036069903e-16, 4.472209874259932285e-16, + 4.534207798565834480e-16, 4.598922204905932469e-16, 4.666615664711475780e-16, + 4.737590853262492027e-16, 4.812199172829237933e-16, 4.890851827392209900e-16, + 4.974034236191939753e-16, 5.062325072144159699e-16, 5.156421828878082953e-16, + 5.257175802022274839e-16, 5.365640977112021618e-16, 5.483144034258703912e-16, + 5.611387454675159622e-16, 5.752606481503331688e-16, 5.909817641652102998e-16, + 6.087231416180907671e-16, 6.290979034877557049e-16, 6.530492053564040799e-16, + 6.821393079028928626e-16, 7.192444966089361564e-16, 7.706095350032096755e-16, + 8.545517038584027421e-16, + ]; + var FE = [ + 1.000000000000000000e+00, 9.381436808621747003e-01, 9.004699299257464817e-01, + 8.717043323812035949e-01, 8.477855006239896074e-01, 8.269932966430503241e-01, + 8.084216515230083777e-01, 7.915276369724956185e-01, 7.759568520401155522e-01, + 7.614633888498962833e-01, 7.478686219851951034e-01, 7.350380924314234843e-01, + 7.228676595935720206e-01, 7.112747608050760117e-01, 7.001926550827881623e-01, + 6.895664961170779872e-01, 6.793505722647653622e-01, 6.695063167319247333e-01, + 6.600008410789997004e-01, 6.508058334145710999e-01, 6.418967164272660897e-01, + 6.332519942143660652e-01, 6.248527387036659775e-01, 6.166821809152076561e-01, + 6.087253820796220127e-01, 6.009689663652322267e-01, 5.934009016917334289e-01, + 5.860103184772680329e-01, 5.787873586028450257e-01, 5.717230486648258170e-01, + 5.648091929124001709e-01, 5.580382822625874484e-01, 5.514034165406412891e-01, + 5.448982376724396115e-01, 5.385168720028619127e-01, 5.322538802630433219e-01, + 5.261042139836197284e-01, 5.200631773682335979e-01, 5.141263938147485613e-01, + 5.082897764106428795e-01, 5.025495018413477233e-01, 4.969019872415495476e-01, + 4.913438695940325340e-01, 4.858719873418849144e-01, 4.804833639304542103e-01, + 4.751751930373773747e-01, 4.699448252839599771e-01, 4.647897562504261781e-01, + 4.597076156421376902e-01, 4.546961574746155033e-01, 4.497532511627549967e-01, + 4.448768734145485126e-01, 4.400651008423538957e-01, 4.353161032156365740e-01, + 4.306281372884588343e-01, 4.259995411430343437e-01, 4.214287289976165751e-01, + 4.169141864330028757e-01, 4.124544659971611793e-01, 4.080481831520323954e-01, + 4.036940125305302773e-01, 3.993906844752310725e-01, 3.951369818332901573e-01, + 3.909317369847971069e-01, 3.867738290841376547e-01, 3.826621814960098344e-01, + 3.785957594095807899e-01, 3.745735676159021588e-01, 3.705946484351460013e-01, + 3.666580797815141568e-01, 3.627629733548177748e-01, 3.589084729487497794e-01, + 3.550937528667874599e-01, 3.513180164374833381e-01, 3.475804946216369817e-01, + 3.438804447045024082e-01, 3.402171490667800224e-01, 3.365899140286776059e-01, + 3.329980687618089852e-01, 3.294409642641363267e-01, 3.259179723935561879e-01, + 3.224284849560891675e-01, 3.189719128449572394e-01, 3.155476852271289490e-01, + 3.121552487741795501e-01, 3.087940669345601852e-01, 3.054636192445902565e-01, + 3.021634006756935276e-01, 2.988929210155817917e-01, 2.956517042812611962e-01, + 2.924392881618925744e-01, 2.892552234896777485e-01, 2.860990737370768255e-01, + 2.829704145387807457e-01, 2.798688332369729248e-01, 2.767939284485173568e-01, + 2.737453096528029706e-01, 2.707225967990600224e-01, 2.677254199320447947e-01, + 2.647534188350622042e-01, 2.618062426893629779e-01, 2.588835497490162285e-01, + 2.559850070304153791e-01, 2.531102900156294577e-01, 2.502590823688622956e-01, + 2.474310756653276266e-01, 2.446259691318921070e-01, 2.418434693988772144e-01, + 2.390832902624491774e-01, 2.363451524570596429e-01, 2.336287834374333461e-01, + 2.309339171696274118e-01, 2.282602939307167011e-01, 2.256076601166840667e-01, + 2.229757680581201940e-01, 2.203643758433594946e-01, 2.177732471487005272e-01, + 2.152021510753786837e-01, 2.126508619929782795e-01, 2.101191593889882581e-01, + 2.076068277242220372e-01, 2.051136562938377095e-01, 2.026394390937090173e-01, + 2.001839746919112650e-01, 1.977470661050988732e-01, 1.953285206795632167e-01, + 1.929281499767713515e-01, 1.905457696631953912e-01, 1.881811994042543179e-01, + 1.858342627621971110e-01, 1.835047870977674633e-01, 1.811926034754962889e-01, + 1.788975465724783054e-01, 1.766194545904948843e-01, 1.743581691713534942e-01, + 1.721135353153200598e-01, 1.698854013025276610e-01, 1.676736186172501919e-01, + 1.654780418749360049e-01, 1.632985287519018169e-01, 1.611349399175920349e-01, + 1.589871389693142123e-01, 1.568549923693652315e-01, 1.547383693844680830e-01, + 1.526371420274428570e-01, 1.505511850010398944e-01, 1.484803756438667910e-01, + 1.464245938783449441e-01, 1.443837221606347754e-01, 1.423576454324722018e-01, + 1.403462510748624548e-01, 1.383494288635802039e-01, 1.363670709264288572e-01, + 1.343990717022136294e-01, 1.324453279013875218e-01, 1.305057384683307731e-01, + 1.285802045452281717e-01, 1.266686294375106714e-01, 1.247709185808309612e-01, + 1.228869795095451356e-01, 1.210167218266748335e-01, 1.191600571753276827e-01, + 1.173168992115555670e-01, 1.154871635786335338e-01, 1.136707678827443141e-01, + 1.118676316700562973e-01, 1.100776764051853845e-01, 1.083008254510337970e-01, + 1.065370040500016602e-01, 1.047861393065701724e-01, 1.030481601712577161e-01, + 1.013229974259536315e-01, 9.961058367063713170e-02, 9.791085331149219917e-02, + 9.622374255043279756e-02, 9.454918937605585882e-02, 9.288713355604354127e-02, + 9.123751663104015530e-02, 8.960028191003285847e-02, 8.797537446727021759e-02, + 8.636274114075691288e-02, 8.476233053236811865e-02, 8.317409300963238272e-02, + 8.159798070923741931e-02, 8.003394754231990538e-02, 7.848194920160642130e-02, + 7.694194317048050347e-02, 7.541388873405840965e-02, 7.389774699236474620e-02, + 7.239348087570873780e-02, 7.090105516237182881e-02, 6.942043649872875477e-02, + 6.795159342193660135e-02, 6.649449638533977414e-02, 6.504911778675374900e-02, + 6.361543199980733421e-02, 6.219341540854099459e-02, 6.078304644547963265e-02, + 5.938430563342026597e-02, 5.799717563120065922e-02, 5.662164128374287675e-02, + 5.525768967669703741e-02, 5.390531019604608703e-02, 5.256449459307169225e-02, + 5.123523705512628146e-02, 4.991753428270637172e-02, 4.861138557337949667e-02, + 4.731679291318154762e-02, 4.603376107617516977e-02, 4.476229773294328196e-02, + 4.350241356888818328e-02, 4.225412241331623353e-02, 4.101744138041481941e-02, + 3.979239102337412542e-02, 3.857899550307485742e-02, 3.737728277295936097e-02, + 3.618728478193142251e-02, 3.500903769739741045e-02, 3.384258215087432992e-02, + 3.268796350895953468e-02, 3.154523217289360859e-02, 3.041444391046660423e-02, + 2.929566022463739317e-02, 2.818894876397863569e-02, 2.709438378095579969e-02, + 2.601204664513421735e-02, 2.494202641973178314e-02, 2.388442051155817078e-02, + 2.283933540638524023e-02, 2.180688750428358066e-02, 2.078720407257811723e-02, + 1.978042433800974303e-02, 1.878670074469603046e-02, 1.780620041091136169e-02, + 1.683910682603994777e-02, 1.588562183997316302e-02, 1.494596801169114850e-02, + 1.402039140318193759e-02, 1.310916493125499106e-02, 1.221259242625538123e-02, + 1.133101359783459695e-02, 1.046481018102997894e-02, 9.614413642502209895e-03, + 8.780314985808975251e-03, 7.963077438017040002e-03, 7.163353183634983863e-03, + 6.381905937319179087e-03, 5.619642207205483020e-03, 4.877655983542392333e-03, + 4.157295120833795314e-03, 3.460264777836904049e-03, 2.788798793574076128e-03, + 2.145967743718906265e-03, 1.536299780301572356e-03, 9.672692823271745359e-04, + 4.541343538414967652e-04, + ]; + + var api = { + syntheticFloor: syntheticFloor, + statement: statement, + judgment: judgment, + isDistinguishable: isDistinguishable, + fixed4: fixed4, + // Exposed for the parity check, not for the page. + _internal: { + Pcg64: Pcg64, + generator: generator, + seedStates: SEED_STATES, + standardNormal: standardNormal, + standardExponential: standardExponential, + standardGamma: standardGamma, + beta: beta, + percentile: percentile, + }, + }; + + if (typeof module === "object" && module.exports) { + module.exports = api; + } else { + root.PlumblineFloor = api; + } +})(typeof globalThis !== "undefined" ? globalThis : this); diff --git a/site/index.html b/site/index.html new file mode 100644 index 0000000..aede9e1 --- /dev/null +++ b/site/index.html @@ -0,0 +1,261 @@ + + + + + +plumbline: ECE floor calculator + + + + + + +
+

The ECE floor

+

A perfectly calibrated model does not score an Expected Calibration Error of zero + on a finite sample. It scores something above zero, and how far above depends on how many rows + you have. Enter a configuration to see that floor, and optionally a measured ECE to see whether + it clears it.

+ +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + +
+
+ + + +
+

What computes this

+

This calculator is a JavaScript reimplementation of plumbline's + synthetic_floor(n, n_bins, accuracy), not the Python itself. It reproduces + numpy's seeded random stream draw for draw (the PCG64 generator and numpy's own normal, + gamma, and beta samplers), so with the same seeds it redraws exactly the outcomes the Python + draws. It is checked against values exported from the Python on every change, in CI, and the + site does not deploy if they disagree by more than 1e-9. The checked values are in + floor-golden.json; the check is + scripts/check_floor_parity.mjs.

+

The floor here is built from summary numbers: it invents n predicted + probabilities averaging your accuracy. A plumbline report uses + calibration_floor on your actual predictions instead, which conditions on how + they are spread, so its floor for the same row count can differ. Use this page to plan and to + sanity-check; use the report for the verdict on a real run.

+
+
+ + + +