diff --git a/CHANGELOG.md b/CHANGELOG.md
index bac0c8c..cc9e4e3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -49,6 +49,16 @@ different event from one that moved because it was wrong.
ranking (`site/search.js`) runs in the browser with no library.
`scripts/check_search.mjs` holds the ranking to its cases and every index
entry to a page and id that exist.
+- The explainer opens with the example report's result drawn as a card (the
+ measured ECE against its floor's 95th percentile, and the verdict), written
+ at build time from the report's own line so it reads the same with scripts
+ off, and keeps its contents in a rail beside the text on wide screens.
+- The calculator and planner each run in their own worker and can be
+ cancelled; their inputs are kept in the address, so a link reproduces a
+ result, and each result has "Copy link" and "Copy result" buttons.
+- Every page carries a Content-Security-Policy that allows nothing but the
+ site's own files (no inline script or style), and `check_site_links.mjs`
+ fails a page that lacks it or carries anything it would block.
### Changed
@@ -59,6 +69,17 @@ different event from one that moved because it was wrong.
### Fixed
+- Every validation message on the site's calculator and planner read
+ "[object Object]" (#25). Messages now say what is wrong, mark the field
+ invalid, and are tied to it for screen readers; a result is announced as one
+ line rather than the whole result block (#53).
+- A failure to draw the worked example was reported as the example being
+ missing; the page now says which happened. `floor.js` turned a bin count of 0
+ into 10 where the Python refuses it, and now refuses it too. The parity
+ check holds the worked example's floor to 1e-9, not only to the four
+ decimals the report prints (#55).
+- A link to a later section of the explainer (`#planner`) stopped short of it,
+ because the worked example grows the page after it loads.
- The inconclusive verdict read as a pass. "Not distinguishable from a
perfectly calibrated model at this sample size" is what the arithmetic
establishes and close to the opposite of what it means, and readers took it
diff --git a/scripts/build_site.py b/scripts/build_site.py
index 2c3903a..5e5414b 100644
--- a/scripts/build_site.py
+++ b/scripts/build_site.py
@@ -56,7 +56,7 @@
from typing import Any, TypedDict
from plumbline.cli import app
-from plumbline.metrics.calibration import synthetic_floor
+from plumbline.metrics.calibration import calibration_floor, synthetic_floor
from plumbline.runner.execute import RunResult
ROOT = Path(__file__).resolve().parent.parent
@@ -292,6 +292,7 @@ def build_example() -> dict[str, Any]:
n_bins = int(ece_match["bins"])
accuracy = sum(outcomes) / len(outcomes)
summary = synthetic_floor(len(probabilities), n_bins=n_bins, accuracy=accuracy)["ece"]
+ real = calibration_floor(run.probabilities(), n_bins=n_bins)["ece"]
return {
"about": (
@@ -317,6 +318,10 @@ def build_example() -> dict[str, Any]:
"python": {
"accuracy": accuracy,
"summary_floor": {"mean": summary.mean, "p95": summary.p95},
+ # The floor from the rows themselves, unrounded, so the page's
+ # calibrationFloor is held to 1e-9 and not only to the 4 decimals
+ # the report prints.
+ "calibration_floor": {"mean": real.mean, "p95": real.p95},
},
}
@@ -572,10 +577,22 @@ def social_meta(title: str, description: str, url: str) -> str:
)
+#: The Content Security Policy every page carries. Everything the site loads is
+#: its own: no inline script or style, no other origin, and data: only for the
+#: inline favicon. ``site/index.html`` carries the same tag, which the build
+#: requires, and ``scripts/check_site_links.mjs`` checks every page for it.
+CSP = (
+ "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; "
+ "font-src 'self'; connect-src 'self'; worker-src 'self'; frame-src 'none'; "
+ "object-src 'none'; base-uri 'none'; form-action 'none'"
+)
+CSP_META = f''
+
PAGE = """
+{csp}
{title} | plumbline
@@ -622,6 +639,7 @@ def _page(
) -> str:
"""One doc-shaped page: header, docs sidebar, the body, and its contents list."""
return PAGE.format(
+ csp=CSP_META,
title=html.escape(title),
description=html.escape(description, quote=True),
meta=meta,
@@ -702,11 +720,71 @@ def write_404(out: Path) -> None:
HEADER_SLOT = ""
-def explainer_page(source: str) -> str:
- """The explainer as written, with the shared header in its slot."""
- if source.count(HEADER_SLOT) != 1:
- raise BuildError(f"site/index.html must contain {HEADER_SLOT} exactly once")
- return source.replace(HEADER_SLOT, site_header("", "calculator"))
+#: Where the example's result card goes, in the explainer's opening block.
+CARD_SLOT = ""
+
+
+def example_card(report: dict[str, Any]) -> str:
+ """The example report's headline result, drawn at build time.
+
+ ``report`` is the ``report`` block of ``example-run.json``: the figures parsed
+ from the report's own ECE line, so the card says exactly what the report
+ says, and says it with scripts off. The gauge puts the measured ECE against
+ the calibrated-model floor's 95th percentile on one scale.
+ """
+ ece, p95, n = float(report["ece"]), float(report["floor_p95"]), int(report["n"])
+ inconclusive = "INCONCLUSIVE" in report["ece_line"]
+ width, scale = 320.0, max(ece, p95) * 1.3
+ x_ece, x_p95 = width * ece / scale, width * p95 / scale
+ anchor = "start" if x_ece < 60 else "end" if x_ece > width - 60 else "middle"
+ gauge = (
+ f'"
+ )
+ if inconclusive:
+ chip = 'INCONCLUSIVE'
+ says = (
+ f"The measured figure sits inside what a perfectly calibrated model scores on "
+ f"{n} rows, so this dataset establishes nothing either way."
+ )
+ else:
+ chip = 'Distinguishable'
+ says = (
+ f"The measured figure is above what a perfectly calibrated model scores on {n} "
+ "rows nineteen times in twenty."
+ )
+ return (
+ '\n'
+ f' The example report, {n} rows\n'
+ '
\n'
+ ""
+ )
+
+
+def explainer_page(source: str, report: dict[str, Any]) -> str:
+ """The explainer as written, with the shared header and the example card in their slots."""
+ for slot in (HEADER_SLOT, CARD_SLOT):
+ if source.count(slot) != 1:
+ raise BuildError(f"site/index.html must contain {slot} exactly once")
+ if source.count(CSP_META) != 1:
+ raise BuildError("site/index.html must carry the site's Content-Security-Policy tag")
+ page = source.replace(HEADER_SLOT, site_header("", "calculator"))
+ return page.replace(CARD_SLOT, example_card(report))
#: The explainer's name in search results.
@@ -789,7 +867,7 @@ def main() -> int:
prov = provenance()
rendered = render_docs(prov)
source = (SITE / "index.html").read_text(encoding="utf-8")
- explainer = explainer_page(source)
+ explainer = explainer_page(source, example["report"])
index = search_index(rendered, source)
except BuildError as problem:
print(f"site build refused: {problem}", file=sys.stderr)
diff --git a/scripts/check_floor_parity.mjs b/scripts/check_floor_parity.mjs
index 3929909..d94d945 100644
--- a/scripts/check_floor_parity.mjs
+++ b/scripts/check_floor_parity.mjs
@@ -72,6 +72,10 @@ if (examplePath) {
for (const key of ["mean", "p95"]) {
const gap = Math.abs(summary[key] - example.python.summary_floor[key]);
if (!(gap <= tolerance)) failures.push(`worked example: summary floor ${key} off by ${gap}`);
+ // The line above agrees to 4 decimals; this holds the floor from the rows
+ // to the same 1e-9 as every other case.
+ const real = Math.abs(band[key] - example.python.calibration_floor[key]);
+ if (!(real <= tolerance)) failures.push(`worked example: calibration floor ${key} off by ${real}`);
}
console.log(`worked example: ${derived.slice(0, 110)}...`);
}
diff --git a/scripts/check_site_links.mjs b/scripts/check_site_links.mjs
index b2d7c46..e7aaa19 100644
--- a/scripts/check_site_links.mjs
+++ b/scripts/check_site_links.mjs
@@ -13,6 +13,9 @@
// - a page loads anything (script, stylesheet, image, frame) from another origin,
// - a page lacks its canonical, Open Graph, or Twitter card tags, or its
// canonical and og:url do not name the page itself,
+// - a page does not carry exactly the site's Content-Security-Policy tag, or
+// has an inline script, an inline style block, or a style attribute (which
+// that policy would block, so the page would silently break),
// - og.png is not a 1200x630 PNG,
// - the 404 page is missing, links anywhere that does not resolve, or (live)
// a missing path is not answered with it and a 404 status.
@@ -23,6 +26,13 @@ import path from "node:path";
const SITE_URL = "https://tmhsdigital.github.io/plumbline/";
const SITE_PATH = new URL(SITE_URL).pathname; // "/plumbline/"
+// Stated here rather than read from build_site.py, so a change to the policy
+// is a change to what this check requires.
+const CSP =
+ "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; " +
+ "font-src 'self'; connect-src 'self'; worker-src 'self'; frame-src 'none'; " +
+ "object-src 'none'; base-uri 'none'; form-action 'none'";
+
const target = process.argv[2];
if (!target) {
console.error("usage: check_site_links.mjs ");
@@ -119,6 +129,15 @@ function checkSocial(sitePath, html) {
}
}
+function checkPolicy(where, html) {
+ const tags = [...html.matchAll(//g)];
+ if (tags.length !== 1) problems.push(`${where}: ${tags.length} Content-Security-Policy tags, expected 1`);
+ else if (tags[0][1] !== CSP) problems.push(`${where}: Content-Security-Policy is not the site's policy`);
+ if (/
-
+
Skip to the argument
-
-
+
+
A calibration claim has a floor
A perfectly calibrated model does not score an Expected Calibration Error of zero.
On a finite sample it scores something above zero, and on a few hundred rows that something is
often larger than the differences people report. Below a certain row count, a calibration claim
is not measurable at all. This page lets you check where your own numbers sit, without installing
anything.