From 4d81646443e6549f0d5c58da7b22c06f047c261f Mon Sep 17 00:00:00 2001 From: TMHSDigital <154358121+TMHSDigital@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:42:38 -0400 Subject: [PATCH 1/2] refactor(site): move the explainer's style and script into their own files No change in behaviour. This is the first step toward a Content Security Policy with no 'unsafe-inline', and it makes the fixes that follow read as diffs of explainer.js rather than of an HTML file. Co-Authored-By: Claude Opus 5.5 (1M context) --- site/explainer.css | 54 +++++++ site/explainer.js | 312 ++++++++++++++++++++++++++++++++++++++ site/index.html | 369 +-------------------------------------------- 3 files changed, 368 insertions(+), 367 deletions(-) create mode 100644 site/explainer.css create mode 100644 site/explainer.js diff --git a/site/explainer.css b/site/explainer.css new file mode 100644 index 0000000..a2837ed --- /dev/null +++ b/site/explainer.css @@ -0,0 +1,54 @@ +/* The explainer (index.html): its calculator, worked example, and planner. */ +section { padding: 2rem 0; border-top: 1px solid var(--line); } +.lede { font-size: 1.05rem; } +.label { display: inline-block; font: 600 0.75rem var(--mono); text-transform: uppercase; letter-spacing: 0.05em; color: var(--warn-fg); background: var(--warn-bg); padding: 0.1rem 0.4rem; border-radius: 3px; margin-bottom: 0.5rem; } + +form { display: grid; grid-template-columns: repeat(auto-fit, minmax(9.5rem, 1fr)); gap: 1rem; margin: 1.25rem 0 1rem; } +label { display: block; font-size: 0.9rem; font-weight: 600; margin-bottom: 0.25rem; } +.hint { display: block; font-size: 0.8rem; font-weight: 400; color: var(--muted); } +input { width: 100%; font: 1rem var(--mono); padding: 0.5rem 0.6rem; color: var(--fg); background: var(--bg); border: 1px solid var(--control); border-radius: 4px; } +.actions { grid-column: 1 / -1; display: flex; gap: 1rem; align-items: center; flex-wrap: wrap; } +button { font: 600 1rem var(--sans); padding: 0.55rem 1.1rem; color: var(--bg); background: var(--fg); border: 0; border-radius: 4px; cursor: pointer; } +button[disabled] { opacity: 0.55; cursor: progress; } +progress { flex: 1 1 8rem; height: 0.5rem; } +.error { color: var(--error); } + +.result { border: 1px solid var(--line); border-radius: 6px; padding: 1rem; background: var(--panel); } +.result[hidden], [hidden] { display: none !important; } +.result h3:first-child { margin-top: 0; } +dl { display: grid; grid-template-columns: max-content 1fr; gap: 0.35rem 1rem; margin: 0 0 1rem; } +dt { color: var(--muted); } +dd { margin: 0; font-family: var(--mono); overflow-wrap: anywhere; } +.verdict { padding: 0.75rem; border-radius: 4px; margin: 0 0 1rem; } +.verdict.inconclusive { background: var(--warn-bg); color: var(--warn-fg); } +.verdict.distinguishable { background: var(--ok-bg); color: var(--ok-fg); } +.verdict strong { display: block; margin-bottom: 0.25rem; } +.big { font: 600 1.6rem var(--mono); margin: 0 0 0.5rem; } + +.table-wrap { overflow-x: auto; margin: 0 0 1rem; } +table { border-collapse: collapse; width: 100%; font-size: 0.9rem; } +caption { text-align: left; color: var(--muted); font-size: 0.85rem; padding-bottom: 0.4rem; } +th, td { padding: 0.35rem 0.5rem; border-bottom: 1px solid var(--line); text-align: right; font-family: var(--mono); white-space: nowrap; } +th { font-family: var(--sans); font-weight: 600; color: var(--muted); } +th:first-child, td:first-child { text-align: left; } +tfoot td { border-bottom: 0; font-weight: 600; } +@media (max-width: 30rem) { + table { font-size: 0.8rem; } + th, td { padding: 0.3rem 0.25rem; } + th { white-space: normal; } +} + +figure { margin: 0 0 1rem; } +figcaption { font-size: 0.85rem; color: var(--muted); margin-top: 0.25rem; } +svg.hist { width: 100%; height: auto; display: block; } +svg.hist .bar { fill: var(--bar); } +svg.hist .axis { stroke: var(--line); } +svg.hist text { fill: var(--muted); font: 13px var(--mono); } +svg.hist .measured { stroke: var(--mark-measured); stroke-width: 2; } +svg.hist .p95 { stroke: var(--mark-p95); stroke-width: 2; stroke-dasharray: 5 3; } +svg.hist text.measured-label { fill: var(--mark-measured); } +svg.hist text.p95-label { fill: var(--mark-p95); } + +ol.steps { padding-left: 1.25rem; } +ol.steps > li { margin-bottom: 1.25rem; } +.trace { font: 0.85rem var(--mono); color: var(--muted); padding-left: 1.25rem; margin: 0 0 1rem; } diff --git a/site/explainer.js b/site/explainer.js new file mode 100644 index 0000000..dc107b6 --- /dev/null +++ b/site/explainer.js @@ -0,0 +1,312 @@ +// The explainer's calculator, worked example, and planner. The arithmetic is +// floor.js; this file only reads the forms and draws the results. +(function () { + "use strict"; + var F = window.PlumblineFloor; + var fmt = F.fixed4; + function $(id) { return document.getElementById(id); } + function text(id, value) { $(id).textContent = value; } + function el(tag, content, className) { + var node = document.createElement(tag); + if (content !== undefined) node.textContent = content; + if (className) node.className = className; + return node; + } + function row(cells) { + var tr = document.createElement("tr"); + cells.forEach(function (cell) { tr.appendChild(el("td", cell)); }); + return tr; + } + function binLabel(lower, upper) { + var digits = upper - lower < 0.1 - 1e-9 ? 2 : 1; + return lower.toFixed(digits) + "-" + upper.toFixed(digits); + } + + /* ---- worker ---------------------------------------------------------- */ + + // A worker keeps the page responsive on large n. Where the browser refuses + // one (a page opened from disk, for example), the same code runs here. + var worker = null; + var pending = {}; + var nextId = 0; + try { + worker = new Worker("floor-worker.js"); + worker.onmessage = function (event) { + var m = event.data; + var job = pending[m.id]; + if (!job) return; + if (m.progress !== undefined) job.onProgress(m.progress); + else if (m.step !== undefined) job.onStep(m.step); + else { + delete pending[m.id]; + if (m.error) job.reject(new Error(m.error)); + else job.resolve(m.result); + } + }; + worker.onerror = function (event) { + event.preventDefault(); + worker = null; + Object.keys(pending).forEach(function (id) { runHere(pending[id]); delete pending[id]; }); + }; + } catch (e) { + worker = null; + } + + function runHere(job) { + setTimeout(function () { + try { + if (job.message.kind === "synthetic") { + job.resolve(F.syntheticFloor(job.message.n, job.message.nBins, job.message.accuracy)); + } else { + job.resolve(F.planRows(job.message.target, job.message.nBins, job.message.accuracy, { onStep: job.onStep })); + } + } catch (error) { job.reject(error); } + }, 20); + } + + function submit(message, onProgress, onStep) { + return new Promise(function (resolve, reject) { + var job = { message: message, resolve: resolve, reject: reject, onProgress: onProgress || function () {}, onStep: onStep || function () {} }; + if (!worker) { runHere(job); return; } + message.id = ++nextId; + pending[message.id] = job; + worker.postMessage(message); + }); + } + + function number(input, check, message) { + var value = Number(input.value); + return check(value) ? { value: value } : { error: message, field: input }; + } + + function busy(prefix, on, statusText) { + $(prefix + "-go").disabled = on; + $(prefix + "-progress").hidden = !on; + $(prefix + "-progress").value = 0; + var status = $(prefix + "-status"); + status.classList.remove("error"); + status.textContent = statusText || ""; + } + + function fail(prefix, error) { + var status = $(prefix + "-status"); + status.textContent = error.message || String(error); + status.classList.add("error"); + if (error.field) error.field.focus(); + } + + var isWhole = function (lo, hi) { return function (v) { return Number.isInteger(v) && v >= lo && v <= hi; }; }; + var isOpenUnit = function (v) { return v > 0 && v < 1; }; + + /* ---- calculator ------------------------------------------------------ */ + + $("calc").addEventListener("submit", function (event) { + event.preventDefault(); + var form = event.target; + var fields = [ + number(form.n, isWhole(1, 100000), "Rows must be a whole number from 1 to 100,000."), + number(form.bins, isWhole(1, 1000), "Bins must be a whole number from 1 to 1,000."), + number(form.accuracy, isOpenUnit, "Accuracy must lie strictly between 0 and 1."), + ]; + var measuredText = form.measured.value.trim(); + var measured = null; + if (measuredText !== "") { + var m = number(form.measured, function (v) { return v >= 0 && v <= 1; }, "Measured ECE must lie between 0 and 1."); + fields.push(m); + measured = m.value; + } + for (var i = 0; i < fields.length; i++) { + if (fields[i].error) { busy("calc", false); fail("calc", fields[i]); return; } + } + var input = { n: fields[0].value, nBins: fields[1].value, accuracy: fields[2].value, measured: measured }; + busy("calc", true, "Running 2,000 bootstrap resamples..."); + submit({ kind: "synthetic", n: input.n, nBins: input.nBins, accuracy: input.accuracy }, + function (f) { $("calc-progress").value = f; }) + .then(function (band) { showCalc(band, input); busy("calc", false, "Done."); }) + .catch(function (error) { busy("calc", false); fail("calc", error); }); + }); + + function showCalc(band, input) { + text("calc-mean", fmt(band.mean)); + text("calc-p95", fmt(band.p95)); + text("calc-config", band.n + " rows, " + band.nBins + " bins, accuracy " + input.accuracy); + text("calc-read", "A perfectly calibrated model scores " + fmt(band.mean) + " on average at this size, and above " + + fmt(band.p95) + " one time in twenty. A measured ECE at or below " + fmt(band.p95) + + " cannot be told apart from calibrated on this many rows."); + var verdict = $("calc-verdict"); + if (input.measured === null) { + verdict.hidden = true; + } else { + var clears = F.isDistinguishable(input.measured, band); + verdict.className = "verdict " + (clears ? "distinguishable" : "inconclusive"); + verdict.replaceChildren( + el("strong", clears ? "Distinguishable from sampling noise" : "Inconclusive, which is not a pass"), + el("span", F.statement(input.measured, band)) + ); + verdict.hidden = false; + } + $("calc-result").hidden = false; + } + + /* ---- worked example -------------------------------------------------- */ + + function histogram(svg, draws, marks) { + var left = 6, right = 394, top = 40, bottom = 172; + var hi = Math.max.apply(null, draws.concat(marks.map(function (m) { return m.value; }))) * 1.08; + var nb = 40; + var counts = new Array(nb).fill(0); + draws.forEach(function (d) { counts[Math.min(nb - 1, Math.floor(d / hi * nb))]++; }); + var peak = Math.max.apply(null, counts); + var x = function (v) { return left + (v / hi) * (right - left); }; + var ns = "http://www.w3.org/2000/svg"; + function add(tag, attrs, content) { + var node = document.createElementNS(ns, tag); + Object.keys(attrs).forEach(function (k) { node.setAttribute(k, attrs[k]); }); + if (content !== undefined) node.textContent = content; + svg.appendChild(node); + return node; + } + svg.replaceChildren(); + var bw = (right - left) / nb; + counts.forEach(function (c, i) { + if (!c) return; + var h = (c / peak) * (bottom - top); + add("rect", { class: "bar", x: left + i * bw + 0.5, y: bottom - h, width: Math.max(1, bw - 1), height: h }); + }); + add("line", { class: "axis", x1: left, x2: right, y1: bottom, y2: bottom }); + [0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3].forEach(function (t) { + if (t > hi) return; + add("line", { class: "axis", x1: x(t), x2: x(t), y1: bottom, y2: bottom + 4 }); + add("text", { x: x(t), y: bottom + 20, "text-anchor": t === 0 ? "start" : "middle" }, t.toFixed(2)); + }); + marks.forEach(function (m, i) { + add("line", { class: m.className, x1: x(m.value), x2: x(m.value), y1: 6 + i * 17, y2: bottom }); + // Start the label at its line when it fits to the right, else end it there. + var anchor = x(m.value) + 4 + m.label.length * 8 < right ? "start" : "end"; + var dx = anchor === "end" ? -4 : 4; + add("text", { class: m.className + "-label", x: x(m.value) + dx, y: 16 + i * 17, "text-anchor": anchor }, m.label); + }); + } + + function loadExample() { + fetch("example-run.json", { cache: "no-cache" }) + .then(function (response) { + if (!response.ok) throw new Error("HTTP " + response.status); + return response.json(); + }) + .then(showExample) + .catch(function () { + $("ex-report-line").replaceChildren(el("p", "The example is not available in this copy of the page.", "muted")); + $("ex-missing").hidden = false; + }); + } + + function showExample(ex) { + var p = ex.probabilities, correct = ex.correct, nBins = ex.n_bins, n = p.length; + $("ex-report-line").replaceChildren(el("p", ex.report.accuracy_line), el("p", ex.report.ece_line)); + + var right = correct.reduce(function (a, b) { return a + b; }, 0); + var accuracy = right / n; + $("ex-rows").replaceChildren( + document.createTextNode(n + " predictions and outcomes from the seeded mock adapter, a deterministic stand-in rather than a vendor. They were regenerated when this site was deployed, by running the command the report records: "), + el("code", ex.command), + document.createTextNode(". " + right + " of " + n + " were right, an accuracy of " + fmt(accuracy) + ".") + ); + + var bins = F.reliability(p, correct, nBins); + var tbody = $("ex-bins"); + tbody.replaceChildren(); + bins.forEach(function (b) { + tbody.appendChild(row([binLabel(b.lower, b.upper), String(b.count), fmt(b.meanProbability), fmt(b.accuracy), fmt(b.gap)])); + }); + var measured = F.ece(p, correct, nBins); + text("ex-n", String(n)); + text("ex-ece", fmt(measured)); + + var band = F.calibrationFloor(p, nBins, { keepDraws: true }); + histogram($("ex-hist"), band.draws, [ + { value: measured, label: "measured " + fmt(measured), className: "measured" }, + { value: band.p95, label: "p95 " + fmt(band.p95), className: "p95" }, + ]); + var beyond = band.draws.filter(function (d) { return d >= measured; }).length; + text("ex-hist-desc", "Histogram of 2,000 resampled ECEs from a perfectly calibrated model with these 105 probabilities. Mean " + + fmt(band.mean) + ", 95th percentile " + fmt(band.p95) + ". " + beyond + " of the 2,000 (" + + Math.round(beyond / 20) + "%) scored at least the measured " + fmt(measured) + "."); + + var derived = F.statement(measured, band); + var clears = F.isDistinguishable(measured, band); + $("ex-verdict").className = "verdict " + (clears ? "distinguishable" : "inconclusive"); + $("ex-verdict").replaceChildren(el("strong", clears ? "Distinguishable from sampling noise" : "Inconclusive, which is not a pass"), el("span", derived)); + var same = derived === ex.report.ece_line; + text("ex-match", same + ? "Derived here, and identical to the line in the report, character for character." + : "This derivation does not match the report line. The deploy check should have prevented this; please open an issue."); + $("ex-steps").hidden = false; + + var summary = F.syntheticFloor(n, nBins, accuracy, { keepProbabilities: true }); + text("ex-real-floor", fmt(band.mean) + " (95th percentile " + fmt(band.p95) + ")"); + text("ex-summary-floor", fmt(summary.mean) + " (95th percentile " + fmt(summary.p95) + ")"); + var realCounts = new Array(nBins).fill(0); + var inventedCounts = new Array(nBins).fill(0); + bins.forEach(function (b) { realCounts[b.index] = b.count; }); + summary.probabilities.forEach(function (q) { inventedCounts[Math.min(nBins - 1, Math.floor(q * nBins))]++; }); + var spread = $("ex-spread"); + spread.replaceChildren(); + for (var k = 0; k < nBins; k++) { + if (!realCounts[k] && !inventedCounts[k]) continue; + spread.appendChild(row([binLabel(k / nBins, (k + 1) / nBins), String(realCounts[k]), String(inventedCounts[k])])); + } + $("ex-lesson").hidden = false; + } + + loadExample(); + + /* ---- planner --------------------------------------------------------- */ + + $("plan").addEventListener("submit", function (event) { + event.preventDefault(); + var form = event.target; + var fields = [ + number(form.target, isOpenUnit, "The ECE to detect must lie strictly between 0 and 1."), + number(form.bins, isWhole(1, 1000), "Bins must be a whole number from 1 to 1,000."), + number(form.accuracy, isOpenUnit, "Accuracy must lie strictly between 0 and 1."), + ]; + for (var i = 0; i < fields.length; i++) { + if (fields[i].error) { busy("plan", false); fail("plan", fields[i]); return; } + } + var target = fields[0].value, nBins = fields[1].value, accuracy = fields[2].value; + var trace = $("plan-trace"); + trace.replaceChildren(); + $("plan-result").hidden = true; + busy("plan", true, "Searching. Large row counts take several seconds each..."); + submit({ kind: "plan", target: target, nBins: nBins, accuracy: accuracy }, + function (f) { $("plan-progress").value = f; }, + function (steps) { showTrace(steps); }) + .then(function (plan) { showPlan(plan, target, nBins, accuracy); busy("plan", false, "Done."); }) + .catch(function (error) { busy("plan", false); fail("plan", error); }); + }); + + function showTrace(steps) { + var trace = $("plan-trace"); + trace.replaceChildren(); + steps.forEach(function (s) { + trace.appendChild(el("li", s.n.toLocaleString("en-US") + " rows: 95th percentile " + fmt(s.p95))); + }); + } + + function showPlan(plan, target, nBins, accuracy) { + showTrace(plan.evaluations); + if (plan.exceeds) { + text("plan-rows", "More than " + plan.maxRows.toLocaleString("en-US") + " rows"); + text("plan-read", "At " + plan.maxRows.toLocaleString("en-US") + " rows the summary floor's 95th percentile is " + + fmt(plan.band.p95) + ", still above " + target + ". Extrapolating at one over the square root of n suggests roughly " + + plan.extrapolated.toLocaleString("en-US") + " rows; this page does not compute floors that large."); + } else { + text("plan-rows", "About " + plan.rows.toLocaleString("en-US") + " rows"); + text("plan-read", "At " + plan.rows.toLocaleString("en-US") + " rows, " + nBins + " bins, and accuracy " + accuracy + + ", the summary floor has mean " + fmt(plan.band.mean) + " and 95th percentile " + fmt(plan.band.p95) + + ", so a measured ECE of " + target + " would clear it. On materially fewer rows, the same measurement would read as INCONCLUSIVE."); + } + $("plan-result").hidden = false; + } +})(); diff --git a/site/index.html b/site/index.html index 1b5606a..774e4c0 100644 --- a/site/index.html +++ b/site/index.html @@ -24,61 +24,7 @@ - + @@ -309,317 +255,6 @@

What computes these numbers

- + From 32ae3259e3bd6ca9b794d00b082fa2e0cefbf3de Mon Sep 17 00:00:00 2001 From: TMHSDigital <154358121+TMHSDigital@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:52:34 -0400 Subject: [PATCH 2/2] feat(site): the explainer's opening, a CSP on every page, and the calculator fixes (phase 7) The explainer opens with the example report's result as a card: the measured ECE against its floor's 95th percentile on one gauge, and the verdict. build_site.py writes it from the report's own ECE line, so it says what the report says and reads the same with scripts off. The contents list becomes a rail beside the text on wide screens. Every page now carries a Content-Security-Policy that allows only the site's own files. The explainer's inline style and script moved out in the previous commit, so nothing needs 'unsafe-inline' or a hash. The build refuses an explainer without the tag, and check_site_links.mjs fails any page that lacks it or carries an inline script, style block, or style attribute the policy would block. The calculator and planner: - validation messages said "[object Object]"; they now say what is wrong, set aria-invalid and aria-describedby on the field, and focus it (fixes #25); - the result blocks are no longer live regions; the status line announces one summary line (fixes the rest of #53); - each tool has its own worker and a Cancel button, so the planner's long search never queues the calculator; - inputs are kept in the query string, a link carrying them runs the tool on load, and each result has Copy link and Copy result (#57). The worked example says whether its rows were missing or failed to draw; floor.js refuses a bin count of 0 as the Python does, where it used to turn it into 10; and the parity check holds the example's floor to 1e-9, not only the four decimals the report prints (fixes #55). A link to a later section now lands on it after the example has grown the page. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 21 +++ scripts/build_site.py | 92 +++++++++- scripts/check_floor_parity.mjs | 4 + scripts/check_site_links.mjs | 22 ++- site/explainer.css | 48 ++++++ site/explainer.js | 296 ++++++++++++++++++++++++++------- site/floor.js | 8 +- site/index.html | 44 +++-- tests/test_build_site.py | 37 ++++- 9 files changed, 483 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bac0c8c..cc9e4e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,16 @@ different event from one that moved because it was wrong. ranking (`site/search.js`) runs in the browser with no library. `scripts/check_search.mjs` holds the ranking to its cases and every index entry to a page and id that exist. +- The explainer opens with the example report's result drawn as a card (the + measured ECE against its floor's 95th percentile, and the verdict), written + at build time from the report's own line so it reads the same with scripts + off, and keeps its contents in a rail beside the text on wide screens. +- The calculator and planner each run in their own worker and can be + cancelled; their inputs are kept in the address, so a link reproduces a + result, and each result has "Copy link" and "Copy result" buttons. +- Every page carries a Content-Security-Policy that allows nothing but the + site's own files (no inline script or style), and `check_site_links.mjs` + fails a page that lacks it or carries anything it would block. ### Changed @@ -59,6 +69,17 @@ different event from one that moved because it was wrong. ### Fixed +- Every validation message on the site's calculator and planner read + "[object Object]" (#25). Messages now say what is wrong, mark the field + invalid, and are tied to it for screen readers; a result is announced as one + line rather than the whole result block (#53). +- A failure to draw the worked example was reported as the example being + missing; the page now says which happened. `floor.js` turned a bin count of 0 + into 10 where the Python refuses it, and now refuses it too. The parity + check holds the worked example's floor to 1e-9, not only to the four + decimals the report prints (#55). +- A link to a later section of the explainer (`#planner`) stopped short of it, + because the worked example grows the page after it loads. - The inconclusive verdict read as a pass. "Not distinguishable from a perfectly calibrated model at this sample size" is what the arithmetic establishes and close to the opposite of what it means, and readers took it diff --git a/scripts/build_site.py b/scripts/build_site.py index 2c3903a..5e5414b 100644 --- a/scripts/build_site.py +++ b/scripts/build_site.py @@ -56,7 +56,7 @@ from typing import Any, TypedDict from plumbline.cli import app -from plumbline.metrics.calibration import synthetic_floor +from plumbline.metrics.calibration import calibration_floor, synthetic_floor from plumbline.runner.execute import RunResult ROOT = Path(__file__).resolve().parent.parent @@ -292,6 +292,7 @@ def build_example() -> dict[str, Any]: n_bins = int(ece_match["bins"]) accuracy = sum(outcomes) / len(outcomes) summary = synthetic_floor(len(probabilities), n_bins=n_bins, accuracy=accuracy)["ece"] + real = calibration_floor(run.probabilities(), n_bins=n_bins)["ece"] return { "about": ( @@ -317,6 +318,10 @@ def build_example() -> dict[str, Any]: "python": { "accuracy": accuracy, "summary_floor": {"mean": summary.mean, "p95": summary.p95}, + # The floor from the rows themselves, unrounded, so the page's + # calibrationFloor is held to 1e-9 and not only to the 4 decimals + # the report prints. + "calibration_floor": {"mean": real.mean, "p95": real.p95}, }, } @@ -572,10 +577,22 @@ def social_meta(title: str, description: str, url: str) -> str: ) +#: The Content Security Policy every page carries. Everything the site loads is +#: its own: no inline script or style, no other origin, and data: only for the +#: inline favicon. ``site/index.html`` carries the same tag, which the build +#: requires, and ``scripts/check_site_links.mjs`` checks every page for it. +CSP = ( + "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; " + "font-src 'self'; connect-src 'self'; worker-src 'self'; frame-src 'none'; " + "object-src 'none'; base-uri 'none'; form-action 'none'" +) +CSP_META = f'' + PAGE = """ +{csp} {title} | plumbline @@ -622,6 +639,7 @@ def _page( ) -> str: """One doc-shaped page: header, docs sidebar, the body, and its contents list.""" return PAGE.format( + csp=CSP_META, title=html.escape(title), description=html.escape(description, quote=True), meta=meta, @@ -702,11 +720,71 @@ def write_404(out: Path) -> None: HEADER_SLOT = "" -def explainer_page(source: str) -> str: - """The explainer as written, with the shared header in its slot.""" - if source.count(HEADER_SLOT) != 1: - raise BuildError(f"site/index.html must contain {HEADER_SLOT} exactly once") - return source.replace(HEADER_SLOT, site_header("", "calculator")) +#: Where the example's result card goes, in the explainer's opening block. +CARD_SLOT = "" + + +def example_card(report: dict[str, Any]) -> str: + """The example report's headline result, drawn at build time. + + ``report`` is the ``report`` block of ``example-run.json``: the figures parsed + from the report's own ECE line, so the card says exactly what the report + says, and says it with scripts off. The gauge puts the measured ECE against + the calibrated-model floor's 95th percentile on one scale. + """ + ece, p95, n = float(report["ece"]), float(report["floor_p95"]), int(report["n"]) + inconclusive = "INCONCLUSIVE" in report["ece_line"] + width, scale = 320.0, max(ece, p95) * 1.3 + x_ece, x_p95 = width * ece / scale, width * p95 / scale + anchor = "start" if x_ece < 60 else "end" if x_ece > width - 60 else "middle" + gauge = ( + f'' + f'' + f'' + f'' + f'' + f"measured {ece:.4f}" + f'' + f"floor, 95th percentile {p95:.4f}" + ) + if inconclusive: + chip = 'INCONCLUSIVE' + says = ( + f"The measured figure sits inside what a perfectly calibrated model scores on " + f"{n} rows, so this dataset establishes nothing either way." + ) + else: + chip = 'Distinguishable' + says = ( + f"The measured figure is above what a perfectly calibrated model scores on {n} " + "rows nineteen times in twenty." + ) + return ( + '
\n' + f'
The example report, {n} rows
\n' + '
\n' + f'

Measured ECE {ece:.4f}

\n' + '

Calibrated-model floor, 95th percentile ' + f'{p95:.4f}

\n' + "
\n" + f" {gauge}\n" + f'

{chip} {says} ' + 'See it derived from the rows.

\n' + "
" + ) + + +def explainer_page(source: str, report: dict[str, Any]) -> str: + """The explainer as written, with the shared header and the example card in their slots.""" + for slot in (HEADER_SLOT, CARD_SLOT): + if source.count(slot) != 1: + raise BuildError(f"site/index.html must contain {slot} exactly once") + if source.count(CSP_META) != 1: + raise BuildError("site/index.html must carry the site's Content-Security-Policy tag") + page = source.replace(HEADER_SLOT, site_header("", "calculator")) + return page.replace(CARD_SLOT, example_card(report)) #: The explainer's name in search results. @@ -789,7 +867,7 @@ def main() -> int: prov = provenance() rendered = render_docs(prov) source = (SITE / "index.html").read_text(encoding="utf-8") - explainer = explainer_page(source) + explainer = explainer_page(source, example["report"]) index = search_index(rendered, source) except BuildError as problem: print(f"site build refused: {problem}", file=sys.stderr) diff --git a/scripts/check_floor_parity.mjs b/scripts/check_floor_parity.mjs index 3929909..d94d945 100644 --- a/scripts/check_floor_parity.mjs +++ b/scripts/check_floor_parity.mjs @@ -72,6 +72,10 @@ if (examplePath) { for (const key of ["mean", "p95"]) { const gap = Math.abs(summary[key] - example.python.summary_floor[key]); if (!(gap <= tolerance)) failures.push(`worked example: summary floor ${key} off by ${gap}`); + // The line above agrees to 4 decimals; this holds the floor from the rows + // to the same 1e-9 as every other case. + const real = Math.abs(band[key] - example.python.calibration_floor[key]); + if (!(real <= tolerance)) failures.push(`worked example: calibration floor ${key} off by ${real}`); } console.log(`worked example: ${derived.slice(0, 110)}...`); } diff --git a/scripts/check_site_links.mjs b/scripts/check_site_links.mjs index b2d7c46..e7aaa19 100644 --- a/scripts/check_site_links.mjs +++ b/scripts/check_site_links.mjs @@ -13,6 +13,9 @@ // - a page loads anything (script, stylesheet, image, frame) from another origin, // - a page lacks its canonical, Open Graph, or Twitter card tags, or its // canonical and og:url do not name the page itself, +// - a page does not carry exactly the site's Content-Security-Policy tag, or +// has an inline script, an inline style block, or a style attribute (which +// that policy would block, so the page would silently break), // - og.png is not a 1200x630 PNG, // - the 404 page is missing, links anywhere that does not resolve, or (live) // a missing path is not answered with it and a 404 status. @@ -23,6 +26,13 @@ import path from "node:path"; const SITE_URL = "https://tmhsdigital.github.io/plumbline/"; const SITE_PATH = new URL(SITE_URL).pathname; // "/plumbline/" +// Stated here rather than read from build_site.py, so a change to the policy +// is a change to what this check requires. +const CSP = + "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; " + + "font-src 'self'; connect-src 'self'; worker-src 'self'; frame-src 'none'; " + + "object-src 'none'; base-uri 'none'; form-action 'none'"; + const target = process.argv[2]; if (!target) { console.error("usage: check_site_links.mjs "); @@ -119,6 +129,15 @@ function checkSocial(sitePath, html) { } } +function checkPolicy(where, html) { + const tags = [...html.matchAll(//g)]; + if (tags.length !== 1) problems.push(`${where}: ${tags.length} Content-Security-Policy tags, expected 1`); + else if (tags[0][1] !== CSP) problems.push(`${where}: Content-Security-Policy is not the site's policy`); + if (/]*\bsrc=)[^>]*>/i.test(html)) problems.push(`${where}: an inline