From 9ed91bb196818948c00432d7e43bb0ecbd022a70 Mon Sep 17 00:00:00 2001 From: TMHSDigital <154358121+TMHSDigital@users.noreply.github.com> Date: Wed, 23 Sep 2026 12:40:51 -0400 Subject: [PATCH] ci(site): test the pages in a real browser, check the live site, widen the em-dash check scripts/smoke_site.mjs serves the assembled site at /plumbline/, as Pages does, and drives the runner's own Chrome over the DevTools protocol with Node 22's built-in WebSocket, so the site workflow still installs nothing. It checks the flows a reader uses: the explainer's result card and derived example, an invalid input's message and aria-invalid, a calculator run writing its inputs into the address, a link carrying them reproducing the result, and search from a doc page. It fails on any console error, uncaught exception, or CSP violation on every page. Checked against a copy of the build with the old "[object Object]" bug and an inline script put back: it names both. A live job after deploy runs check_site_links.mjs against the Pages URL, which is the only way to see that a missing path really returns the 404 page with a 404 status. It retries once, since the CDN can lag a deploy. The em-dash check now covers site/, scripts/, and .github/, and the HTML entity and JS escape spellings, written with bracketed characters so the workflow does not match itself. The quickstart's traceback guard used grep -qv, which succeeds on any multi-line output, so it could never fail. It now fails when the word appears (fixes #49). Part of #54. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 16 +- .github/workflows/site.yml | 25 +++ CHANGELOG.md | 13 +- scripts/smoke_site.mjs | 319 +++++++++++++++++++++++++++++++++++++ 4 files changed, 367 insertions(+), 6 deletions(-) create mode 100644 scripts/smoke_site.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41c0161..565feff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,11 +124,15 @@ jobs: --results results 2>&1)" set -e echo "$out" | grep -q "requires" || { echo "expected a named refusal, got:"; echo "$out"; exit 1; } - echo "$out" | grep -qv "Traceback" || { echo "traceback leaked"; exit 1; } + if echo "$out" | grep -q "Traceback"; then echo "traceback leaked"; echo "$out"; exit 1; fi # The house style has no em dashes, and the report's own strings are prose a - # reader sees. datasets/public/ is excluded because it holds vendored - # third-party rows that are reproduced as published. + # reader sees, as are the site's pages and the issue templates. Besides the + # character itself it catches the spellings that render as one in HTML or JS + # (the named and numeric HTML entities, and the JS unicode escape); each is + # written in the pattern with a bracketed character so this file does not + # match itself. datasets/public/ holds vendored third-party rows reproduced as + # published, and site/vendor/ a vendored library, so both are excluded. prose: name: no em dashes in the docs or the report strings runs-on: ubuntu-latest @@ -136,9 +140,11 @@ jobs: steps: - uses: actions/checkout@v5 - - name: Tracked markdown and src/ contain no em dash + - name: Tracked prose, source, site, and templates contain no em dash run: | - if git grep -n -P '\x{2014}' -- '*.md' 'src/' ':!datasets/public/'; then + pattern='\x{2014}|&m[d]ash;|̵[2];|ȁ[4];|\\u201[4]' + if git grep -n -i -P "$pattern" -- '*.md' 'src/' 'site/' 'scripts/' '.github/' \ + ':!datasets/public/' ':!site/vendor/'; then echo "::error::em dash found above; use a comma, colon, parentheses, or a new sentence" exit 1 fi diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml index d679816..ef40a24 100644 --- a/.github/workflows/site.yml +++ b/.github/workflows/site.yml @@ -103,6 +103,13 @@ jobs: tar -xf artifact/artifact.tar -C site-as-published node scripts/check_site_links.mjs site-as-published + # The runner's own Chrome, driven over the DevTools protocol by the + # runner's own Node; nothing is installed. It fails on a broken flow (the + # calculator, a shared link, search) and on any console error or CSP + # violation, on every page. + - name: The pages work in a real browser + run: node scripts/smoke_site.mjs site-as-published + deploy: name: deploy to GitHub Pages needs: [parity, docs] @@ -123,3 +130,21 @@ jobs: - name: Publish the checked site id: deployment uses: actions/deploy-pages@v5 + + live: + name: the live site resolves as the checked one did + needs: deploy + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + # What Pages actually serves, including a real 404 for a missing path, + # which only a request to the live site can show. The CDN can take a + # moment to serve the new build, so a failure is retried once. + - name: Every link, anchor, meta tag, and policy resolves on the live site + run: | + node scripts/check_site_links.mjs https://tmhsdigital.github.io/plumbline/ || { + sleep 60 + node scripts/check_site_links.mjs https://tmhsdigital.github.io/plumbline/ + } diff --git a/CHANGELOG.md b/CHANGELOG.md index cc9e4e3..bcc9e24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,13 +59,21 @@ different event from one that moved because it was wrong. - Every page carries a Content-Security-Policy that allows nothing but the site's own files (no inline script or style), and `check_site_links.mjs` fails a page that lacks it or carries anything it would block. +- The site is checked in a real browser before it deploys. + `scripts/smoke_site.mjs` drives the runner's own Chrome over the DevTools + protocol with Node's standard library (nothing is installed), runs the + calculator, a shared link, and search, and fails on any console error or CSP + violation on any page. After a deploy, a new job checks the live site's + links, anchors, meta tags, policy, and 404. ### Changed - Report bullets separate the label from the figure with a colon instead of an em dash (`- **Cost**: not reported.`), and the docs no longer use em dashes. A CI job now fails on an em dash in tracked markdown or `src/`. **This - changes report text, not any number.** + changes report text, not any number.** The job also covers `site/`, + `scripts/`, and `.github/`, and the HTML entity and JavaScript escape + spellings of the character. ### Fixed @@ -94,6 +102,9 @@ different event from one that moved because it was wrong. `FileNotFoundError` traceback instead of refusing with a reason. - An adapter built without a required setting raised a bare `TypeError` from `__init__` instead of naming the setting. +- CI's guard against a traceback leaking from the quickstart refusal could + never fail: `grep -qv Traceback` succeeds on any output with one line + without the word (#49). - CI ran every pull request twice, because both the `push` and `pull_request` triggers fired on a branch pushed to origin. diff --git a/scripts/smoke_site.mjs b/scripts/smoke_site.mjs new file mode 100644 index 0000000..7df3d6d --- /dev/null +++ b/scripts/smoke_site.mjs @@ -0,0 +1,319 @@ +// Drive the assembled site in a real browser and fail if a reader would meet a +// broken page. Runs on the runner's own Node (22 or later, for its built-in +// WebSocket) and the Chrome the runner already has; nothing is installed. The +// browser is driven over the Chrome DevTools Protocol directly. +// +// node scripts/smoke_site.mjs _site +// +// Set CHROME to the browser's path if it is not in a usual place. +// +// The site is served at /plumbline/, as Pages serves it, and the check fails on +// any console error, uncaught exception, or Content-Security-Policy violation, +// on any page, and if any of these does not hold: +// +// - the explainer opens with its result card, and the worked example derives +// the report's line in the browser; +// - an invalid input says what is wrong (not "[object Object]") and marks the +// field invalid; +// - the calculator runs, and writes its inputs into the address; +// - a link carrying those inputs runs the calculator as the page opens; +// - search, opened from the header on a doc page, finds a section of that doc. + +import { spawn } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +const root = process.argv[2]; +if (!root || !existsSync(path.join(root, "index.html"))) { + console.error("usage: smoke_site.mjs "); + process.exit(2); +} +if (typeof WebSocket !== "function") { + console.error(`smoke_site: Node ${process.version} has no built-in WebSocket; Node 22 or later is needed`); + process.exit(2); +} + +// ---- serve the site where Pages serves it -------------------------------- + +const TYPES = { + ".html": "text/html; charset=utf-8", + ".css": "text/css; charset=utf-8", + ".js": "text/javascript; charset=utf-8", + ".json": "application/json; charset=utf-8", + ".png": "image/png", + ".md": "text/markdown; charset=utf-8", +}; +const PREFIX = "/plumbline/"; + +const server = createServer((request, response) => { + const url = new URL(request.url, "http://localhost"); + if (!url.pathname.startsWith(PREFIX)) { + response.writeHead(404).end(); + return; + } + let file = path.join(root, ...decodeURIComponent(url.pathname.slice(PREFIX.length)).split("/")); + if (!path.resolve(file).startsWith(path.resolve(root))) { + response.writeHead(403).end(); + return; + } + if (existsSync(file) && statSync(file).isDirectory()) file = path.join(file, "index.html"); + if (!existsSync(file)) { + response.writeHead(404, { "content-type": TYPES[".html"] }).end(readFileSync(path.join(root, "404.html"))); + return; + } + response.writeHead(200, { "content-type": TYPES[path.extname(file)] ?? "application/octet-stream" }); + response.end(readFileSync(file)); +}); +await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); +const origin = `http://127.0.0.1:${server.address().port}`; +const site = `${origin}${PREFIX}`; + +// ---- start the browser ---------------------------------------------------- + +function findChrome() { + const candidates = [ + process.env.CHROME, + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", + "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe", + ]; + return candidates.find((candidate) => candidate && existsSync(candidate)); +} +const chromePath = findChrome(); +if (!chromePath) { + console.error("smoke_site: no Chrome found; set CHROME to its path"); + process.exit(2); +} + +const profile = mkdtempSync(path.join(tmpdir(), "plumbline-smoke-")); +const chrome = spawn(chromePath, [ + "--headless=new", + "--remote-debugging-port=0", + `--user-data-dir=${profile}`, + "--no-first-run", + "--no-default-browser-check", + "--disable-gpu", + "--window-size=1280,900", + "about:blank", +], { stdio: ["ignore", "ignore", "pipe"] }); + +const browserUrl = await new Promise((resolve, reject) => { + let seen = ""; + const timer = setTimeout(() => reject(new Error(`Chrome did not report a DevTools address:\n${seen}`)), 20000); + chrome.stderr.on("data", (chunk) => { + seen += chunk; + const match = /DevTools listening on (ws:\/\/\S+)/.exec(seen); + if (match) { + clearTimeout(timer); + resolve(match[1]); + } + }); + chrome.on("exit", (code) => reject(new Error(`Chrome exited with ${code}:\n${seen}`))); +}); + +// ---- a minimal DevTools client --------------------------------------------- + +const socket = new WebSocket(browserUrl); +await new Promise((resolve, reject) => { + socket.addEventListener("open", resolve, { once: true }); + socket.addEventListener("error", reject, { once: true }); +}); +let nextId = 0; +const waiting = new Map(); +const listeners = []; +socket.addEventListener("message", (event) => { + const message = JSON.parse(event.data); + if (message.id !== undefined && waiting.has(message.id)) { + const { resolve, reject } = waiting.get(message.id); + waiting.delete(message.id); + if (message.error) reject(new Error(`${message.error.message} (${message.error.code})`)); + else resolve(message.result); + } else if (message.method) { + for (const listener of listeners) listener(message); + } +}); +function send(method, params = {}, sessionId) { + const id = ++nextId; + socket.send(JSON.stringify({ id, method, params, sessionId })); + return new Promise((resolve, reject) => waiting.set(id, { resolve, reject })); +} + +const { targetId } = await send("Target.createTarget", { url: "about:blank" }); +const { sessionId } = await send("Target.attachToTarget", { targetId, flatten: true }); +const page = (method, params) => send(method, params, sessionId); + +// Everything a reader's console would show as a problem. +const problems = []; +let where = "about:blank"; +listeners.push(({ method, params, sessionId: from }) => { + if (from !== sessionId) return; + if (method === "Runtime.exceptionThrown") { + const details = params.exceptionDetails; + problems.push(`${where}: uncaught ${details.exception?.description ?? details.text}`); + } else if (method === "Runtime.consoleAPICalled" && (params.type === "error" || params.type === "assert")) { + problems.push(`${where}: console.${params.type}: ${params.args.map((a) => a.value ?? a.description).join(" ")}`); + } else if (method === "Log.entryAdded" && params.entry.level === "error") { + problems.push(`${where}: ${params.entry.source}: ${params.entry.text}`); + } +}); +await page("Runtime.enable"); +await page("Log.enable"); +await page("Page.enable"); +// Recorded on every page before its own scripts run, so a violation the +// console might word differently still counts. +await page("Page.addScriptToEvaluateOnNewDocument", { + source: `window.__violations = []; + document.addEventListener("securitypolicyviolation", (e) => + window.__violations.push(e.violatedDirective + " " + (e.blockedURI || "inline")));`, +}); + +async function evaluate(expression) { + const { result, exceptionDetails } = await page("Runtime.evaluate", { + expression, + awaitPromise: true, + returnByValue: true, + }); + if (exceptionDetails) throw new Error(`${expression.slice(0, 80)}: ${exceptionDetails.exception?.description ?? exceptionDetails.text}`); + return result.value; +} + +// Polls a condition in the page until it holds. +async function until(condition, what, timeout = 30000) { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + if (await evaluate(`Boolean(${condition})`)) return; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error(`timed out waiting for ${what}`); +} + +// Loads a page and waits for its load event. A navigation to the address the +// page already has is a same-document jump with no load event, so each check +// opens an address of its own; the timeout turns a mistake there into a +// failure rather than a hang. +async function open(sitePath) { + where = sitePath || "index.html"; + let listener; + const loaded = new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`${where} did not finish loading`)), 20000); + listener = ({ method, sessionId: from }) => { + if (method === "Page.loadEventFired" && from === sessionId) { + clearTimeout(timer); + resolve(); + } + }; + listeners.push(listener); + }); + try { + await page("Page.navigate", { url: site + sitePath }); + await loaded; + } finally { + listeners.splice(listeners.indexOf(listener), 1); + } +} + +async function violations() { + const found = await evaluate("window.__violations || []"); + for (const v of found) problems.push(`${where}: CSP violation: ${v}`); +} + +const failures = []; +async function check(name, body) { + try { + await body(); + console.log(` ok ${name}`); + } catch (error) { + failures.push(`${name}: ${error.message}`); + console.log(` -- ${name}: ${error.message}`); + } +} +const expect = (condition, what) => { + if (!condition) throw new Error(what); +}; +const $ = (id) => `document.getElementById(${JSON.stringify(id)})`; + +// ---- the checks ------------------------------------------------------------ + +console.log(`smoke_site: ${chromePath}\n on ${site}`); + +await check("the explainer opens with its result card and derives the example", async () => { + await open(""); + expect(await evaluate("!!document.querySelector('.hero-card .gauge')"), "no result card"); + await until(`!${$("ex-steps")}.hidden`, "the worked example to be drawn"); + const match = await evaluate(`${$("ex-match")}.textContent`); + expect(match.startsWith("Derived here, and identical"), `the derived line does not match: ${match}`); + expect(await evaluate("!document.querySelector('.search-open').hidden"), "site.js did not run: search is still hidden"); + await violations(); +}); + +await check("an invalid input says what is wrong and marks the field", async () => { + await evaluate(`${$("n")}.value = "0"; ${$("calc-go")}.click();`); + const status = await evaluate(`${$("calc-status")}.textContent`); + expect(status === "Rows must be a whole number from 1 to 100,000.", `status reads ${JSON.stringify(status)}`); + expect((await evaluate(`${$("n")}.getAttribute("aria-invalid")`)) === "true", "the field is not marked invalid"); +}); + +await check("the calculator runs and keeps its inputs in the address", async () => { + await evaluate(`${$("n")}.value = "200"; ${$("measured")}.value = "0.2"; ${$("calc-go")}.click();`); + await until(`!${$("calc-result")}.hidden && !${$("calc-go")}.disabled`, "a calculator result"); + const status = await evaluate(`${$("calc-status")}.textContent`); + expect(status.startsWith("Floor mean "), `status reads ${JSON.stringify(status)}`); + const search = await evaluate("location.search"); + expect(search.includes("n=200") && search.includes("measured=0.2"), `address is ${search}`); + await violations(); +}); + +await check("a link carrying the inputs runs the calculator as the page opens", async () => { + // Not the address the previous check left behind (see open()). + await open("?n=150&bins=10&accuracy=0.8&measured=0.25#calculator"); + await until(`!${$("calc-result")}.hidden`, "the linked result"); + const verdict = await evaluate(`${$("calc-verdict")}.textContent`); + expect(verdict.includes("ECE 0.2500 over 150 rows"), `verdict reads ${JSON.stringify(verdict.slice(0, 80))}`); + await violations(); +}); + +await check("search on a doc page finds a section of it", async () => { + await open("docs/methodology.html"); + await evaluate("document.querySelector('.search-open').click()"); + await until("document.querySelector('dialog.search')?.open", "the search dialog"); + await evaluate(`(() => { const input = document.querySelector('dialog.search input'); + input.value = 'temperature'; input.dispatchEvent(new Event('input')); })()`); + await until("document.querySelectorAll('.search-results a').length > 0", "search results"); + const first = await evaluate("document.querySelector('.search-results a').href"); + expect(first.includes("/docs/methodology.html#"), `the first result is ${first}`); + await violations(); +}); + +await check("every doc page loads without an error", async () => { + const index = JSON.parse(readFileSync(path.join(root, "search-index.json"), "utf8")); + const pages = [...new Set(index.map((entry) => entry.u.split("#")[0]).filter((u) => u.startsWith("docs/")))]; + for (const sitePath of [...pages, "docs/", "404.html"]) { + await open(sitePath); + await violations(); + } +}); + +// ---- report ---------------------------------------------------------------- + +socket.close(); +chrome.kill(); +server.close(); +await new Promise((resolve) => chrome.once("exit", resolve).once("error", resolve)); +try { + rmSync(profile, { recursive: true, force: true }); +} catch { + // Chrome can hold the profile a moment after it exits; the OS cleans temp. +} + +const all = [...failures, ...problems]; +if (all.length) { + console.error(`\nsmoke_site: ${all.length} problem(s):\n ${all.join("\n ")}`); + process.exit(1); +} +console.log("smoke_site: every flow works, with no console error and no CSP violation.");