From 2578ee61d8c6006e774111d88dde1e6f308221bc Mon Sep 17 00:00:00 2001 From: joshuarg007 Date: Sun, 15 Nov 2026 15:04:59 -0700 Subject: [PATCH] Harden rendergap against hostile pages: 0.2.0 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/action-self-test.yml | 7 + CHANGELOG.md | 24 ++ README.md | 23 ++ action.yml | 30 +- bin/rendergap.js | 61 +++- package.json | 2 +- src/agents.js | 5 +- src/crawlers.js | 104 ++++-- src/extract.js | 90 ++++- src/fetch.js | 128 ++++++- src/render.js | 283 ++++++++++++--- src/report.js | 7 +- src/robots.js | 33 +- test/hostile.test.js | 469 +++++++++++++++++++++++++ test/units.test.js | 78 ++++ 15 files changed, 1221 insertions(+), 123 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 test/hostile.test.js create mode 100644 test/units.test.js diff --git a/.github/workflows/action-self-test.yml b/.github/workflows/action-self-test.yml index e566e34..70e5692 100644 --- a/.github/workflows/action-self-test.yml +++ b/.github/workflows/action-self-test.yml @@ -21,3 +21,10 @@ jobs: https://www.axiondeepdigital.com/research/ai-visibility-study check: both fail-on: critical + - uses: ./ + with: + urls: https://www.axiondeepdigital.com/las-cruces/seo + check: gap + viewport: mobile + headers: | + x-rendergap-self-test: 1 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..534cb01 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,24 @@ +# Changelog + +## 0.2.0 + +Hardening pass: every protection below has a hostile-page test, and each was verified by breaking it on purpose and watching a test fail. + +- Bot challenge detection names vendors (Cloudflare, DataDome, Imperva, PerimeterX, AWS WAF, SiteGround, DDoS-Guard, Akamai). reCAPTCHA on a contact form is no longer a false challenge. +- Responses are decoded by their declared charset; bodies over 15 MB are capped. +- When the crawler is blocked, challenged, sent non-HTML or caught in a redirect loop, that is reported on its own instead of a noisy content diff. +- Cookie banners, consent tools, dialogs and chat widgets are excluded from the comparison. +- Text is normalized (quotes, dashes, zero-width characters, Unicode compatibility forms). +- Reads open shadow roots and scrolls to trigger lazy-loaded sections. +- Extraction runs in an isolated JavaScript world. +- Dialogs are dismissed, `debugger;` statements skipped, infinite loops stopped, and every step has a deadline. The CLI has a hard ceiling. +- JavaScript and meta refresh redirects are flagged. +- New: `--viewport mobile`, repeatable `--header`, basic auth from URL credentials (never printed). +- Crawler check: robots.txt fetched as each crawler, crawler-specific `X-Robots-Tag`, one retry on HTTP 429, `rate limited` and `inconclusive` statuses, `--fail-on blocked` counts only real blocks. +- robots.txt: exact product-token matching, percent-encoding normalization, BOM handling, robots.txt always allowed. +- Input validation with clear errors and exit code 2 for usage mistakes. +- Action: `viewport` and `headers` inputs, safe URL trimming, collapsible job summary. + +## 0.1.0 + +First release: render gap and crawler access checks, CLI and GitHub Action. diff --git a/README.md b/README.md index c4ae7c5..f9aced7 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,8 @@ Options: - `--agent ` the crawler whose raw fetch to compare. Default `gptbot`. - `--fail-on ` exit 1 when a finding is at least `critical` (default), `warning` or `info`. Use `never` to always exit 0. +- `--viewport mobile` render the page as a phone instead of a desktop browser. +- `--header "Name: value"` send an extra header on every request, repeatable. Use it for preview deployments behind a bypass token. Credentials in the URL (`https://user:pass@staging.example.com/`) are sent as basic auth and never printed. - `--json` machine-readable output. - `--timeout ` page load timeout. @@ -74,6 +76,8 @@ CCBot sent a bot challenge (HTTP 200, 5210 bytes) Google-Extended allowed (robots.txt token only, not fetched) ``` +robots.txt is also fetched as each crawler, because some servers hand AI crawlers a stricter file than browsers get; when that happens the crawler is judged by its own copy and the report says so. Other statuses: `rate limited` (HTTP 429 even after one polite retry), `inconclusive` (a normal browser request was blocked too, so the crawler block proves nothing), and `served with X-Robots-Tag noindex`, which honors crawler-specific headers such as `X-Robots-Tag: GPTBot: noindex`. + Crawlers checked: GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, PerplexityBot, CCBot, Googlebot, Bingbot, plus the robots.txt-only tokens Google-Extended and Applebot-Extended. Use `--only gptbot,claudebot` to narrow it, and `--fail-on blocked` to exit 1 when any of them is blocked. What it can and cannot tell you: requests come from your machine with each crawler's user agent. Some firewalls verify real crawlers by IP address or reverse DNS, so the real crawler can be treated differently, in either direction. A block here is worth investigating; an "allowed" here is not proof the real crawler gets through. Run it against sites you own or have permission to test. @@ -91,6 +95,9 @@ Check preview deployments before they ship: check: both # gap, crawlers, or both fail-on: critical # gap check threshold fail-on-blocked: false + viewport: desktop # or mobile + headers: | # optional, one per line + x-vercel-protection-bypass: ${{ secrets.VERCEL_BYPASS }} ``` Results are printed to the log and to the job summary. @@ -101,6 +108,22 @@ Results are printed to the log and to the job summary. - **Rendered side:** a normal page load with a desktop browser user agent, waiting for the network to go quiet, then one more second. - **Extraction:** one function reads both sides: title, meta tags, canonical, headings, JSON-LD types, phone numbers (tel links and visible text), email links, prices, FAQ items, forms, word count and text blocks. +## What it is built to survive + +Real pages fight tools. Each of these has a test in `test/hostile.test.js`, and each protection is checked by deliberately breaking it and confirming a test fails. + +- Bot challenge pages (Cloudflare, DataDome, Imperva, PerimeterX, AWS WAF and others) are reported as blocks. A contact form that merely loads reCAPTCHA is not. +- Pages in legacy encodings such as windows-1252 are decoded from the header or the meta tag, so accented text does not show up as a false gap. +- Cookie banners, consent tools, modal dialogs and chat widgets are left out of the comparison, including banners built only from utility classes. A bakery's "cookies" section still counts. +- Content inside web components (open shadow roots) and sections that load on scroll are read. +- Curly quotes, dashes, zero-width characters and odd whitespace compare equal to their plain forms. +- The extractor runs in an isolated JavaScript world, so a page that overwrites `JSON`, `Set`, `Array.from` or `querySelectorAll` cannot break or fool it. +- `alert()` and `confirm()` are dismissed, `debugger;` statements are skipped, and a page stuck in an infinite loop is stopped and read as it stood. +- Pages that never go network-idle are read as loaded. Every in-page step has a deadline and the CLI has a hard ceiling, so a CI job cannot hang. +- When the crawler is blocked, served a challenge, sent a PDF or caught in a redirect loop, that is the finding. You do not get a page of false "missing content" noise. +- JavaScript and meta refresh redirects are flagged, since the crawler and the browser are then reading different pages. +- robots.txt follows RFC 9309: exact product-token matching, longest match wins, `*` and `$` patterns, percent-encoding, byte order marks, and "server error means disallow all". + Findings are about what is present, not whether a crawler ranks or cites you. A clean report means your content is in the first HTML response. It does not mean an AI system will recommend you. ## Contributing diff --git a/action.yml b/action.yml index ae15628..8c6bc34 100644 --- a/action.yml +++ b/action.yml @@ -25,6 +25,14 @@ inputs: description: "Crawler check: fail when any crawler is blocked (true or false)" required: false default: "false" + viewport: + description: "desktop or mobile for the rendered side of the gap check" + required: false + default: "desktop" + headers: + description: "Extra request headers, one 'Name: value' per line (for example a preview bypass token from secrets)" + required: false + default: "" runs: using: "composite" @@ -46,24 +54,32 @@ runs: RG_AGENT: ${{ inputs.agent }} RG_FAIL_ON: ${{ inputs.fail-on }} RG_FAIL_BLOCKED: ${{ inputs.fail-on-blocked }} + RG_VIEWPORT: ${{ inputs.viewport }} + RG_HEADERS: ${{ inputs.headers }} RG_BIN: ${{ github.action_path }}/bin/rendergap.js run: | if [ -x /usr/bin/google-chrome ]; then export CHROME_PATH=/usr/bin/google-chrome; fi export NO_COLOR=1 + trim() { local s="$1"; s="${s#"${s%%[![:space:]]*}"}"; s="${s%"${s##*[![:space:]]}"}"; printf '%s' "$s"; } + hdr=() + while IFS= read -r h; do + h="$(trim "$h")"; [ -n "$h" ] && hdr+=(--header "$h") + done <<< "$RG_HEADERS" + summary() { { echo "
$1"; echo; echo '````text'; echo "$2"; echo '````'; echo "
"; } >> "$GITHUB_STEP_SUMMARY"; } status=0 while IFS= read -r url; do - url="$(echo "$url" | xargs)" + url="$(trim "$url")" [ -z "$url" ] && continue if [ "$RG_CHECK" = "gap" ] || [ "$RG_CHECK" = "both" ]; then - out="$(node "$RG_BIN" "$url" --agent "$RG_AGENT" --fail-on "$RG_FAIL_ON")" || status=1 - echo "$out" - { echo '```'; echo "$out"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + code=0; out="$(node "$RG_BIN" "$url" --agent "$RG_AGENT" --fail-on "$RG_FAIL_ON" --viewport "$RG_VIEWPORT" "${hdr[@]}" 2>&1)" || code=$? + echo "$out"; summary "render gap: $url" "$out" + if [ "$code" -ne 0 ]; then status=1; fi fi if [ "$RG_CHECK" = "crawlers" ] || [ "$RG_CHECK" = "both" ]; then flag="never"; [ "$RG_FAIL_BLOCKED" = "true" ] && flag="blocked" - out="$(node "$RG_BIN" crawlers "$url" --fail-on "$flag")" || status=1 - echo "$out" - { echo '```'; echo "$out"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + code=0; out="$(node "$RG_BIN" crawlers "$url" --fail-on "$flag" "${hdr[@]}" 2>&1)" || code=$? + echo "$out"; summary "crawler access: $url" "$out" + if [ "$code" -ne 0 ]; then status=1; fi fi done <<< "$RG_URLS" exit $status diff --git a/bin/rendergap.js b/bin/rendergap.js index 8c009ac..a52e5d4 100755 --- a/bin/rendergap.js +++ b/bin/rendergap.js @@ -2,7 +2,7 @@ import { parseArgs } from "node:util"; import { readFileSync } from "node:fs"; import { AGENTS, findAgent } from "../src/agents.js"; -import { checkCrawlers } from "../src/crawlers.js"; +import { checkCrawlers, BLOCKED } from "../src/crawlers.js"; import { meetsThreshold } from "../src/diff.js"; import { formatGap, formatCrawlers } from "../src/report.js"; @@ -17,6 +17,7 @@ Render gap options --fail-on exit 1 if a finding is at least: critical | warning | info | never (default: critical) --timeout page load timeout (default: 30000) + --viewport desktop or mobile for the rendered side (default: desktop) Crawler check options --only check only these crawlers (ids or names) @@ -24,6 +25,7 @@ Crawler check options (default: never) Both + --header "K: V" extra request header, repeatable (e.g. a preview bypass token) --json print JSON instead of text -h, --help show this help -v, --version show the version @@ -31,18 +33,31 @@ Both Crawlers: ${AGENTS.map((a) => a.id).join(", ")} Set CHROME_PATH to use an installed Chrome instead of the bundled one.`; -const { values, positionals } = parseArgs({ +function fail(msg) { + console.error(`rendergap: ${msg}`); + process.exit(2); +} + +let parsed; +try { + parsed = parseArgs({ allowPositionals: true, options: { agent: { type: "string", default: "gptbot" }, "fail-on": { type: "string" }, timeout: { type: "string", default: "30000" }, + viewport: { type: "string", default: "desktop" }, + header: { type: "string", multiple: true, default: [] }, only: { type: "string" }, json: { type: "boolean", default: false }, help: { type: "boolean", short: "h", default: false }, version: { type: "boolean", short: "v", default: false }, }, -}); + }); +} catch (err) { + fail(`${err.message}\nRun rendergap --help for usage.`); +} +const { values, positionals } = parsed; if (values.version) { const pkg = JSON.parse(readFileSync(new URL("../package.json", import.meta.url))); @@ -59,13 +74,30 @@ if (values.help || !target) { let url; try { - url = new URL(/^https?:\/\//i.test(target) ? target : `https://${target}`).href; + const u = new URL(/^[a-z][a-z0-9+.-]*:\/\//i.test(target) ? target : `https://${target}`); + if (!/^https?:$/.test(u.protocol)) throw new Error(); + url = u.href; } catch { - console.error(`Not a valid URL: ${target}`); - process.exit(2); + fail(`not a valid http(s) URL: ${target}`); } -const timeoutMs = Number(values.timeout) || 30000; +const timeoutMs = Number(values.timeout); +if (!Number.isFinite(timeoutMs) || timeoutMs < 1000) fail(`--timeout must be a number of milliseconds, at least 1000`); +if (!["desktop", "mobile"].includes(values.viewport)) fail(`--viewport must be desktop or mobile`); + +const headers = {}; +for (const h of values.header) { + const i = h.indexOf(":"); + if (i < 1) fail(`--header must look like "Name: value", got: ${h}`); + headers[h.slice(0, i).trim()] = h.slice(i + 1).trim(); +} + +// A hard ceiling so a stuck page or browser can never hang a CI job. +const ceiling = Math.max(timeoutMs * 6, 180000); +setTimeout(() => { + console.error(`rendergap: gave up after ${Math.round(ceiling / 1000)} s`); + process.exit(2); +}, ceiling).unref(); try { if (isCrawlers) { @@ -77,17 +109,20 @@ try { return a; }); } - const result = await checkCrawlers(url, { agents, timeoutMs }); - console.log(values.json ? JSON.stringify(result, null, 2) : formatCrawlers(result)); const failOn = values["fail-on"] || "never"; - if (failOn === "blocked" && result.results.some((r) => r.status !== "allowed")) process.exit(1); + if (!["blocked", "never"].includes(failOn)) fail(`for crawlers, --fail-on must be blocked or never`); + const result = await checkCrawlers(url, { agents, timeoutMs, headers }); + console.log(values.json ? JSON.stringify(result, null, 2) : formatCrawlers(result)); + if (failOn === "blocked" && result.results.some((r) => BLOCKED.has(r.status))) process.exit(1); } else { + const failOn = values["fail-on"] || "critical"; + if (!["critical", "warning", "info", "never"].includes(failOn)) fail(`--fail-on must be critical, warning, info or never`); const agent = findAgent(values.agent); - if (!agent || agent.fetchable === false) throw new Error(`Not a fetchable crawler: ${values.agent}`); + if (!agent || agent.fetchable === false) fail(`not a fetchable crawler: ${values.agent}`); const { renderGap } = await import("../src/render.js"); - const result = await renderGap(url, { agent, timeoutMs }); + const result = await renderGap(url, { agent, timeoutMs, viewport: values.viewport, headers }); console.log(values.json ? JSON.stringify(result, null, 2) : formatGap(result)); - if (meetsThreshold(result.findings, values["fail-on"] || "critical")) process.exit(1); + if (meetsThreshold(result.findings, failOn)) process.exit(1); } } catch (err) { console.error(`rendergap: ${err.message}`); diff --git a/package.json b/package.json index 1fa877d..f8c57fc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rendergap", - "version": "0.1.0", + "version": "0.2.0", "description": "See what AI crawlers miss on your page: compare raw HTML with the rendered page, and check which crawlers can reach it at all.", "type": "module", "bin": { diff --git a/src/agents.js b/src/agents.js index b0ce383..0ce4220 100644 --- a/src/agents.js +++ b/src/agents.js @@ -8,6 +8,9 @@ export const BROWSER_UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"; +export const MOBILE_UA = + "Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Mobile Safari/537.36"; + export const AGENTS = [ { id: "gptbot", @@ -92,6 +95,6 @@ export const AGENTS = [ ]; export function findAgent(idOrToken) { - const k = String(idOrToken).toLowerCase(); + const k = String(idOrToken || "").trim().toLowerCase(); return AGENTS.find((a) => a.id === k || a.token.toLowerCase() === k); } diff --git a/src/crawlers.js b/src/crawlers.js index b035e52..4a47e9b 100644 --- a/src/crawlers.js +++ b/src/crawlers.js @@ -6,32 +6,55 @@ // differ from what the real crawler gets, in either direction. import { AGENTS, BROWSER_UA } from "./agents.js"; -import { fetchRaw } from "./fetch.js"; +import { fetchRaw, xRobotsFor, splitCredentials } from "./fetch.js"; import { parseRobots, isAllowed } from "./robots.js"; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); -export async function checkCrawlers(url, { agents = AGENTS, timeoutMs, delayMs = 400 } = {}) { +// Statuses that mean "this crawler is kept out". Used by --fail-on blocked. +export const BLOCKED = new Set(["blocked-robots", "blocked-server", "challenged", "noindex-header"]); + +async function fetchWithBackoff(url, opts, { retries = 1, delayMs = 1500 } = {}) { + let res = await fetchRaw(url, opts); + for (let i = 0; i < retries && res.ok && res.status === 429; i++) { + const wait = Math.min(Number(res.retryAfter) * 1000 || delayMs, 10000); + await sleep(wait); + res = await fetchRaw(url, opts); + } + return res; +} + +export async function checkCrawlers(url, { agents = AGENTS, timeoutMs, delayMs = 400, headers = {} } = {}) { + ({ url, headers } = splitCredentials(url, headers)); const target = new URL(url); const pathWithQuery = target.pathname + target.search; const robotsUrl = new URL("/robots.txt", target).href; - const robotsRes = await fetchRaw(robotsUrl, { userAgent: BROWSER_UA, timeoutMs }); - let robots = { url: robotsUrl, status: robotsRes.ok ? robotsRes.status : null, error: robotsRes.error }; - let groups = []; - if (robotsRes.ok && robotsRes.status >= 200 && robotsRes.status < 300) { - groups = parseRobots(robotsRes.body); - } else if (robotsRes.ok && robotsRes.status >= 500) { - // RFC 9309: an unreachable robots.txt (5xx) means assume complete disallow. - robots.note = "robots.txt returned a server error; crawlers treat that as disallow all"; - groups = parseRobots("User-agent: *\nDisallow: /"); - } + const robotsRes = await fetchRaw(robotsUrl, { userAgent: BROWSER_UA, timeoutMs, headers }); + const robots = { url: robotsUrl, status: robotsRes.ok ? robotsRes.status : null, error: robotsRes.error }; + const groups = robotsGroups(robotsRes, robots); + // Real crawlers request robots.txt with their own user agent, and some servers answer + // differently. Fetch it as each crawler and use that version when it differs. + const robotsByUa = new Map(); - const baseline = await fetchRaw(url, { userAgent: BROWSER_UA, timeoutMs }); + const baseline = await fetchWithBackoff(url, { userAgent: BROWSER_UA, timeoutMs, headers }); const results = []; for (const agent of agents) { - const verdict = isAllowed(groups, agent.token, pathWithQuery); + let agentGroups = groups; + if (agent.fetchable !== false) { + const r = await fetchRaw(robotsUrl, { userAgent: agent.ua, timeoutMs, headers }); + if (r.ok) { + const info = {}; + const g = robotsGroups(r, info); + // Compare parsed rules, not bytes: a timestamp comment should not count as a difference. + if (r.status !== robotsRes.status || JSON.stringify(g) !== JSON.stringify(groups)) { + agentGroups = g; + robotsByUa.set(agent.token, { status: r.status, ...info }); + } + } + } + const verdict = isAllowed(agentGroups, agent.token, pathWithQuery); const row = { id: agent.id, token: agent.token, @@ -46,15 +69,25 @@ export async function checkCrawlers(url, { agents = AGENTS, timeoutMs, delayMs = continue; } await sleep(delayMs); - const res = await fetchRaw(url, { userAgent: agent.ua, timeoutMs }); + const res = await fetchWithBackoff(url, { userAgent: agent.ua, timeoutMs, headers }); row.http = res.ok - ? { status: res.status, bytes: res.bytes, challenge: res.challenge, xRobotsTag: res.xRobotsTag, ms: res.ms } + ? { + status: res.status, + bytes: res.bytes, + challenge: res.challenge, + xRobotsTag: res.xRobotsTag, + finalUrl: res.finalUrl, + ms: res.ms, + } : { error: res.error, ms: res.ms }; - row.status = classify(verdict, res, baseline); + row.status = classify(verdict, res, baseline, agent.token); + if (robotsByUa.has(agent.token)) row.robotsDiffers = robotsByUa.get(agent.token); results.push(row); } + if (robotsByUa.size) robots.note = [robots.note, `robots.txt is different for: ${[...robotsByUa.keys()].join(", ")}`].filter(Boolean).join("; "); return { + version: 1, url, robots, baseline: baseline.ok @@ -64,14 +97,37 @@ export async function checkCrawlers(url, { agents = AGENTS, timeoutMs, delayMs = }; } -export function classify(verdict, res, baseline) { +function robotsGroups(res, info) { + if (res.ok && res.status >= 200 && res.status < 300) { + if (/^\s*= 500 || res.status === 429)) { + // RFC 9309: an unreachable robots.txt means assume complete disallow. + info.note = `robots.txt returned HTTP ${res.status}; crawlers treat that as disallow all`; + return parseRobots("User-agent: *\nDisallow: /"); + } + if (res.ok) { + info.note = `robots.txt returned HTTP ${res.status}; that means no rules, everything allowed`; + return []; + } + info.note = "robots.txt could not be fetched"; + return []; +} + +export function classify(verdict, res, baseline, token = "") { if (!verdict.allowed) return "blocked-robots"; if (!res.ok) return "error"; - const baseOk = baseline.ok && baseline.status < 400; - if (res.status >= 400 && baseOk) return "blocked-server"; - if (res.challenge && !(baseline.ok && baseline.challenge)) return "challenged"; - if (res.status >= 400) return "error"; - if (/noindex|none/i.test(res.xRobotsTag)) return "noindex-header"; + const baseOk = baseline.ok && baseline.status < 400 && !baseline.challenge; + if (res.status === 429) return "rate-limited"; + if (!baseOk && (res.status >= 400 || res.challenge)) return "inconclusive"; + if (res.status >= 400) return "blocked-server"; + if (res.challenge) return "challenged"; + const directives = xRobotsFor(res.xRobotsTag, token); + if (directives.includes("noindex") || directives.includes("none")) return "noindex-header"; if (baseOk && baseline.bytes > 2000 && res.bytes < baseline.bytes * 0.2) return "different-response"; return "allowed"; } @@ -83,5 +139,7 @@ export const STATUS_TEXT = { challenged: "sent a bot challenge", "noindex-header": "served with X-Robots-Tag noindex", "different-response": "got a much smaller response than a browser", + "rate-limited": "rate limited (HTTP 429), try again later", + inconclusive: "inconclusive: a normal browser request was blocked too", error: "request failed", }; diff --git a/src/extract.js b/src/extract.js index 664de3b..ea03dba 100644 --- a/src/extract.js +++ b/src/extract.js @@ -3,25 +3,85 @@ // measured the same way. Keep it self-contained: no closures over module scope. export function extractSignals() { - const clean = (s) => (s || "").replace(/\s+/g, " ").trim(); + // Normalize so the same words compare equal: compatibility forms, curly quotes, + // zero-width characters and every kind of whitespace. + const clean = (s) => + (s || "") + .normalize("NFKC") + .replace(/[\u200B-\u200D\u2060\uFEFF\u00AD]/g, "") + .replace(/[\u2018\u2019\u201A\u2032]/g, "'") + .replace(/[\u201C\u201D\u201E\u2033]/g, '"') + .replace(/[\u2013\u2014]/g, "-") + .replace(/\s+/g, " ") + .trim(); + // Overlays that are not page content: cookie banners, consent tools, modal dialogs, + // chat widgets. Text inside them is ignored on both sides. + // Names only, so a bakery's "cookies" section is never mistaken for a banner. + const OVERLAY = /(^|[\s_-])(cookies?[-_]?(banner|consent|notice|bar|popup|law|modal|wall)|consent[-_]?(banner|manager|modal|bar)?|cmp[-_]|gdpr|ccpa|onetrust|ot-sdk|cookiebot|cybot|osano|termly|truste|iubenda|intercom|drift-|hubspot-messages|tidio|crisp-client|zendesk|livechat|tawk)/i; + // Custom banners often carry only utility classes. A fixed-position box whose text is + // a consent prompt is a banner too. Sticky is left out: that is how headers stay put. + // Computed style only exists on the rendered + // side, which is the only side a JavaScript-injected banner appears on. + // Consent prompts, not mentions: "we use cookies", "accept cookies", "cookie settings". + const CONSENT_TEXT = /\b(we use cookies|uses cookies|accept (all )?cookies|cookie (settings|preferences|policy)|manage (cookies|preferences|consent)|your (privacy )?consent|consent to (our|the) use)\b/i; + const floating = new WeakMap(); + const isConsentBox = (n) => { + if (!floating.has(n)) { + let v = false; + const pos = getComputedStyle(n).position; + if (pos === "fixed" && n.tagName !== "HEADER" && n.tagName !== "NAV") { + v = CONSENT_TEXT.test((n.innerText || n.textContent || "").slice(0, 600)); + } + floating.set(n, v); + } + return floating.get(n); + }; + const inOverlay = (el) => { + for (let n = el; n && n.nodeType === 1; n = n.parentNode || n.host) { + if (n === document.body || n === document.documentElement) return false; + if (n.getAttribute("role") === "dialog" || n.getAttribute("aria-modal") === "true" || n.getAttribute("role") === "alertdialog") return true; + if (OVERLAY.test((n.id || "") + " " + (typeof n.className === "string" ? n.className : ""))) return true; + if (isConsentBox(n)) return true; + } + return false; + }; + // querySelectorAll that also looks inside open shadow roots (web components). + const roots = [document]; + for (let i = 0; i < roots.length; i++) { + for (const el of roots[i].querySelectorAll("*")) if (el.shadowRoot) roots.push(el.shadowRoot); + } + const all = (sel) => roots.flatMap((r) => [...r.querySelectorAll(sel)]); + const content = (sel) => all(sel).filter((e) => !inOverlay(e)); const attr = (sel, name) => { const el = document.querySelector(sel); return el ? clean(el.getAttribute(name)) : ""; }; const texts = (sel, cap = 50) => - [...document.querySelectorAll(sel)].map((e) => clean(e.textContent)).filter(Boolean).slice(0, cap); + content(sel).map((e) => clean(e.textContent)).filter(Boolean).slice(0, cap); // Visible text only. Script, style and template contents are never page text, and // falling back to textContent on an empty page would count JavaScript source as content. - const bodyText = (() => { - if (!document.body) return ""; + const { text: bodyText, overlayWords } = (() => { + if (!document.body) return { text: "", overlayWords: 0 }; const copy = document.body.cloneNode(true); copy.querySelectorAll("script, style, template, noscript[data-ignore]").forEach((n) => n.remove()); - return clean(document.body.innerText) || clean(copy.textContent); + const main = clean(document.body.innerText) || clean(copy.textContent); + const shadow = roots.slice(1).map((r) => { + const c = document.createElement("div"); + for (const n of r.childNodes) c.appendChild(n.cloneNode(true)); + c.querySelectorAll("script, style, template").forEach((n) => n.remove()); + return clean(c.textContent); + }); + // Subtract the words of cookie banners, dialogs and chat widgets. + const overlayText = [...document.body.querySelectorAll("*")] + .filter((el) => inOverlay(el) && !(el.parentElement && inOverlay(el.parentElement))) + .map((el) => clean(el.innerText || "")) + .join(" "); + return { text: clean([main, ...shadow].join(" ")), overlayWords: overlayText ? overlayText.split(" ").length : 0 }; })(); const jsonLdTypes = []; - for (const s of document.querySelectorAll('script[type="application/ld+json"]')) { + for (const s of all('script[type="application/ld+json" i]')) { try { const walk = (node) => { if (!node || typeof node !== "object") return; @@ -39,7 +99,7 @@ export function extractSignals() { const phoneRe = /(?:\+?1[\s.-]?)?\(?\b\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}\b/g; const phones = new Set(); - for (const a of document.querySelectorAll('a[href^="tel:"]')) { + for (const a of all('a[href^="tel:" i]')) { const d = a.getAttribute("href").replace(/\D/g, "").slice(-10); if (d.length === 10) phones.add(d); } @@ -50,11 +110,11 @@ export function extractSignals() { const prices = [...new Set(bodyText.match(/[$€£]\s?\d[\d,]*(?:\.\d{2})?/g) || [])].slice(0, 30); - const blocks = [...document.querySelectorAll("p, li, dd, td, blockquote, summary")] + const blocks = content("p, li, dd, td, blockquote, summary") .map((e) => clean(e.textContent)) .filter((t) => t.length >= 40); - const links = [...document.querySelectorAll("a[href]")]; + const links = all("a[href]"); const unnamedLinks = links.filter((a) => { const text = clean(a.textContent); // title is not counted: it is not a reliable accessible name. @@ -71,17 +131,17 @@ export function extractSignals() { metaRobots: attr('meta[name="robots" i]', "content"), h1: texts("h1", 10), h2: texts("h2", 50), - wordCount: bodyText ? bodyText.split(" ").length : 0, + wordCount: Math.max(0, (bodyText ? bodyText.split(" ").length : 0) - overlayWords), linkCount: links.length, unnamedLinks, jsonLdTypes: [...new Set(jsonLdTypes)].sort(), phones: [...phones].sort(), - emails: [...new Set([...document.querySelectorAll('a[href^="mailto:"]')].map((a) => + emails: [...new Set(all('a[href^="mailto:" i]').map((a) => a.getAttribute("href").slice(7).split("?")[0].toLowerCase()))].sort(), prices, - forms: document.querySelectorAll("form").length, - faqItems: document.querySelectorAll("details > summary").length, - imagesWithAlt: document.querySelectorAll("img[alt]:not([alt=''])").length, - blocks: blocks.slice(0, 400), + forms: all("form").length, + faqItems: all("details > summary").length, + imagesWithAlt: all("img[alt]:not([alt=''])").length, + blocks: blocks.slice(0, 5000), }; } diff --git a/src/fetch.js b/src/fetch.js index fdcbad5..7f6ad97 100644 --- a/src/fetch.js +++ b/src/fetch.js @@ -1,15 +1,78 @@ // Plain HTTP fetch, the way a crawler that does not run JavaScript sees a page. +export const MAX_BYTES = 15 * 1024 * 1024; + +// Bot-challenge interstitials. Each pattern names a vendor's challenge page, not just a +// mention of a captcha: a contact form that loads reCAPTCHA is not a challenge page. const CHALLENGE_PATTERNS = [ - /\s*just a moment/i, - /cf-chl-|challenge-platform/i, - /<title>\s*attention required/i, - /captcha/i, - /<title>\s*access denied/i, - /px-captcha|_incapsula_resource|ddos-guard/i, + /<title>\s*just a moment\.{0,3}\s*<\/title>/i, // Cloudflare + /\/cdn-cgi\/challenge-platform\//i, // Cloudflare + /<title>\s*attention required!?\s*\|\s*cloudflare/i, // Cloudflare block + /px-captcha|perimeterx/i, // HUMAN / PerimeterX + /_incapsula_resource/i, // Imperva + /<title>\s*pardon our interruption/i, // Imperva / Distil + /captcha-delivery\.com/i, // DataDome + /ddos-guard/i, // DDoS-Guard + /sgcaptcha/i, // SiteGround + /awswaf|aws-waf-token/i, // AWS WAF challenge + /<title>\s*access denied\s*<\/title>[\s\S]{0,4000}reference\s*#/i, // Akamai ]; -export async function fetchRaw(url, { userAgent, timeoutMs = 20000 } = {}) { +export function looksLikeChallenge(body) { + const head = body.slice(0, 30000); + if (CHALLENGE_PATTERNS.some((re) => re.test(head))) return true; + // Generic: a small page whose title asks the visitor to prove they are human. + if (body.length < 30000 && /<title>[^<]*(verify you are human|checking your browser|are you a robot|security check)/i.test(head)) return true; + return false; +} + +function charsetFrom(contentType, bytes) { + const header = /charset\s*=\s*["']?([\w-]+)/i.exec(contentType || ""); + if (header) return header[1]; + const head = new TextDecoder("latin1").decode(bytes.subarray(0, 4096)); + const meta = /<meta[^>]+charset\s*=\s*["']?([\w-]+)/i.exec(head); + return meta ? meta[1] : "utf-8"; +} + +function decode(bytes, contentType) { + // BOM wins over everything. + if (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) return new TextDecoder("utf-8").decode(bytes.subarray(3)); + const label = charsetFrom(contentType, bytes); + try { + return new TextDecoder(label).decode(bytes); + } catch { + return new TextDecoder("utf-8").decode(bytes); + } +} + +async function readCapped(res) { + const reader = res.body?.getReader(); + if (!reader) return { bytes: new Uint8Array(0), truncated: false }; + const chunks = []; + let total = 0; + let truncated = false; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > MAX_BYTES) { + truncated = true; + chunks.push(value.subarray(0, value.byteLength - (total - MAX_BYTES))); + await reader.cancel(); + break; + } + chunks.push(value); + } + const out = new Uint8Array(Math.min(total, MAX_BYTES)); + let off = 0; + for (const c of chunks) { + out.set(c, off); + off += c.byteLength; + } + return { bytes: out, truncated }; +} + +export async function fetchRaw(url, { userAgent, timeoutMs = 20000, headers = {} } = {}) { const started = Date.now(); const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); @@ -18,31 +81,68 @@ export async function fetchRaw(url, { userAgent, timeoutMs = 20000 } = {}) { redirect: "follow", signal: controller.signal, headers: { - "user-agent": userAgent, accept: "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8", "accept-language": "en-US,en;q=0.9", + ...headers, + "user-agent": userAgent, }, }); - const body = await res.text(); + const { bytes, truncated } = await readCapped(res); + const contentType = res.headers.get("content-type") || ""; + const body = decode(bytes, contentType); return { ok: true, status: res.status, - finalUrl: res.url, - contentType: res.headers.get("content-type") || "", + finalUrl: res.url || url, + redirected: res.redirected, + contentType, + isHtml: !contentType || /html|xml/i.test(contentType), xRobotsTag: res.headers.get("x-robots-tag") || "", + retryAfter: res.headers.get("retry-after") || "", server: res.headers.get("server") || "", - bytes: Buffer.byteLength(body), + bytes: bytes.byteLength, + truncated, body, - challenge: CHALLENGE_PATTERNS.some((re) => re.test(body.slice(0, 20000))), + challenge: looksLikeChallenge(body), ms: Date.now() - started, }; } catch (err) { + const cause = err.cause?.code || err.cause?.message; return { ok: false, - error: err.name === "AbortError" ? `timed out after ${timeoutMs} ms` : err.message, + error: err.name === "AbortError" ? `timed out after ${timeoutMs} ms` : cause ? `${err.message} (${cause})` : err.message, ms: Date.now() - started, }; } finally { clearTimeout(timer); } } + +// X-Robots-Tag can be general ("noindex") or aimed at one crawler ("GPTBot: noindex"). +// Returns the directives that apply to this crawler token. +export function xRobotsFor(value, token) { + if (!value) return []; + const out = []; + const t = token.toLowerCase(); + for (const part of value.split(/,(?=\s*[\w-]+\s*:)|\n/)) { + const m = /^\s*([\w-]+)\s*:\s*(.+)$/.exec(part); + if (m && !/^(unavailable_after|max-snippet|max-image-preview|max-video-preview)$/i.test(m[1])) { + if (m[1].toLowerCase() === t) out.push(...m[2].split(",").map((d) => d.trim().toLowerCase())); + } else { + out.push(...part.split(",").map((d) => d.trim().toLowerCase()).filter(Boolean)); + } + } + return out; +} + +// https://user:pass@staging.example.com/ is common for protected previews, but fetch() +// refuses URLs with credentials. Move them into an Authorization header instead. +export function splitCredentials(url, headers = {}) { + const u = new URL(url); + if (!u.username && !u.password) return { url: u.href, headers }; + const auth = Buffer.from(`${decodeURIComponent(u.username)}:${decodeURIComponent(u.password)}`).toString("base64"); + u.username = ""; + u.password = ""; + const has = Object.keys(headers).some((k) => k.toLowerCase() === "authorization"); + return { url: u.href, headers: has ? headers : { ...headers, Authorization: `Basic ${auth}` } }; +} diff --git a/src/render.js b/src/render.js index 8459b3f..9db213a 100644 --- a/src/render.js +++ b/src/render.js @@ -5,25 +5,97 @@ // blocked; the rendered side is a normal page load. import puppeteer from "puppeteer"; -import { BROWSER_UA } from "./agents.js"; -import { fetchRaw } from "./fetch.js"; +import { BROWSER_UA, MOBILE_UA } from "./agents.js"; +import { fetchRaw, xRobotsFor, splitCredentials } from "./fetch.js"; import { extractSignals } from "./extract.js"; import { compare } from "./diff.js"; +// Every in-page step gets a deadline: a page script stuck in a loop blocks the page's +// thread, and an await on it would otherwise never return. +export function withTimeout(promise, ms, what) { + let timer; + return Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(Object.assign(new Error(`${what} did not finish within ${ms} ms`), { name: "StepTimeout" })), ms); + }), + ]).finally(() => clearTimeout(timer)); +} + +// A DevTools session attached before navigation, so it can still interrupt the page +// later: a session opened while the page's thread is spinning can never attach. +// skipAllPauses keeps `debugger;` statements in site code from pausing the page. +async function attachGuard(page) { + try { + const session = await page.createCDPSession(); + session.unsticking = false; + // skipAllPauses does not always survive a navigation to a new process, so also + // resume any pause we did not ask for. + session.on("Debugger.paused", () => { + if (!session.unsticking) session.send("Debugger.resume").catch(() => {}); + }); + await session.send("Debugger.enable"); + await session.send("Debugger.setSkipAllPauses", { skip: true }); + return session; + } catch { + return null; + } +} + +// If the page's own script is spinning, stop it so the DOM can be read. +async function unstick(page, guard, notes) { + const alive = await withTimeout(page.evaluate(() => 1), 2000, "page check").then(() => true, () => false); + if (alive || !guard) return; + guard.unsticking = true; + guard.send("Debugger.pause").catch(() => {}); + guard.send("Runtime.terminateExecution").catch(() => {}); + guard.send("Debugger.resume").catch(() => {}); + guard.unsticking = false; + const back = await withTimeout(page.evaluate(() => 1), 5000, "page check").then(() => true, () => false); + notes.push( + back + ? "the page's script was stuck in a loop; rendergap stopped it and read the page as it was" + : "the page's script was stuck in a loop and could not be stopped", + ); +} + +async function closeQuietly(page) { + await withTimeout(page.close(), 5000, "closing the page").catch(() => {}); +} + +const VIEWPORTS = { + desktop: { width: 1366, height: 900, deviceScaleFactor: 1, isMobile: false, hasTouch: false }, + mobile: { width: 412, height: 915, deviceScaleFactor: 2.6, isMobile: true, hasTouch: true }, +}; + export async function launchBrowser() { const args = []; if (process.env.CI || process.getuid?.() === 0) args.push("--no-sandbox", "--disable-setuid-sandbox"); - return puppeteer.launch({ - headless: true, - executablePath: process.env.CHROME_PATH || undefined, - args, - }); + try { + return await puppeteer.launch({ + headless: true, + executablePath: process.env.CHROME_PATH || undefined, + args, + }); + } catch (err) { + if (/Could not find|Failed to launch|ENOENT/i.test(err.message)) { + throw new Error( + "could not start Chrome. Set CHROME_PATH to an installed Chrome, or run `npx puppeteer browsers install chrome`." + + (process.env.CHROME_PATH ? ` (CHROME_PATH is ${process.env.CHROME_PATH})` : ""), + ); + } + throw err; + } } -function withBase(html, baseUrl) { - if (/<base\s/i.test(html)) return html; - const tag = `<base href="${baseUrl.replace(/"/g, """)}">`; - return /<head[^>]*>/i.test(html) ? html.replace(/<head[^>]*>/i, (m) => m + tag) : tag + html; +export function withBase(html, baseUrl) { + // Only a real <base> in the head counts; one inside a script or comment does not. + const head = html.replace(/<!--[\s\S]*?-->/g, "").replace(/<script[\s\S]*?<\/script>/gi, ""); + if (/<base\s[^>]*href/i.test(head)) return html; + const tag = `<base href="${baseUrl.replace(/&/g, "&").replace(/"/g, """)}">`; + if (/<head[^>]*>/i.test(html)) return html.replace(/<head[^>]*>/i, (m) => m + tag); + if (/<html[^>]*>/i.test(html)) return html.replace(/<html[^>]*>/i, (m) => m + `<head>${tag}</head>`); + return tag + html; } async function readRaw(browser, html, baseUrl) { @@ -32,55 +104,182 @@ async function readRaw(browser, html, baseUrl) { await page.setJavaScriptEnabled(false); await page.setRequestInterception(true); page.on("request", (req) => req.abort()); - await page.setContent(withBase(html, baseUrl), { waitUntil: "domcontentloaded", timeout: 30000 }); - return await page.evaluate(extractSignals); + await page.setContent(withBase(html, baseUrl), { waitUntil: "domcontentloaded", timeout: 60000 }); + return await withTimeout(page.evaluate(extractSignals), 30000, "reading the raw HTML"); } finally { - await page.close(); + await closeQuietly(page); + } +} + +// Run the extractor in a fresh isolated world (public DevTools protocol), so a page that +// overwrites JSON, Set, Array.from or String.prototype cannot break or fool it. Falls +// back to the page's own world if the protocol call is not available. +async function extractIsolated(page) { + let session; + try { + session = await page.createCDPSession(); + const { frameTree } = await session.send("Page.getFrameTree"); + const { executionContextId } = await session.send("Page.createIsolatedWorld", { + frameId: frameTree.frame.id, + worldName: "rendergap", + grantUniveralAccess: false, + }); + const { result, exceptionDetails } = await session.send("Runtime.evaluate", { + expression: `(${extractSignals.toString()})()`, + contextId: executionContextId, + returnByValue: true, + awaitPromise: true, + }); + if (exceptionDetails) throw new Error(exceptionDetails.exception?.description || exceptionDetails.text); + return result.value; + } catch (err) { + if (/Execution context|Target closed|detached/i.test(err.message)) throw err; + return page.evaluate(extractSignals); + } finally { + await session?.detach().catch(() => {}); + } +} + +// A page can navigate away while we read it (late redirects, reloads). Wait and retry. +async function extractStable(page) { + for (let attempt = 0; ; attempt++) { + try { + return await extractIsolated(page); + } catch (err) { + if (attempt >= 2 || !/Execution context|context was destroyed|detached|Cannot find context/i.test(err.message)) throw err; + await page.waitForNavigation({ waitUntil: "domcontentloaded", timeout: 10000 }).catch(() => {}); + } } } -async function readRendered(browser, url, { timeoutMs, settleMs }) { +async function autoScroll(page, maxMs = 4000) { + // Trigger lazy-loaded sections so the rendered side is what a visitor scrolling sees. + await page.evaluate(async (limit) => { + const start = Date.now(); + let last = -1; + while (Date.now() - start < limit) { + window.scrollBy(0, Math.max(400, window.innerHeight * 0.8)); + await new Promise((r) => setTimeout(r, 120)); + const y = window.scrollY + window.innerHeight; + if (y >= document.documentElement.scrollHeight - 2 && y === last) break; + last = y; + } + window.scrollTo(0, 0); + }, maxMs); +} + +async function readRendered(browser, url, { timeoutMs, settleMs, viewport, headers }) { const page = await browser.newPage(); + const notes = []; + const guard = await attachGuard(page); + // alert(), confirm() and prompt() freeze a headless page until answered. + page.on("dialog", (d) => { + notes.push(`page opened a ${d.type()} dialog; dismissed it`); + d.dismiss().catch(() => {}); + }); try { - await page.setUserAgent(BROWSER_UA); - await page.setViewport({ width: 1366, height: 900 }); - const res = await page.goto(url, { waitUntil: "networkidle2", timeout: timeoutMs }); + await page.setUserAgent(viewport === "mobile" ? MOBILE_UA : BROWSER_UA); + await page.setViewport(VIEWPORTS[viewport] || VIEWPORTS.desktop); + if (Object.keys(headers).length) await page.setExtraHTTPHeaders(headers); + let res = null; + try { + res = await page.goto(url, { waitUntil: "networkidle2", timeout: timeoutMs }); + } catch (err) { + // Pages that keep a connection open (chat widgets, long polling) never go idle. + // Fall back to whatever has loaded instead of failing the whole check. + if (err.name !== "TimeoutError") throw err; + notes.push(`network never went idle within ${timeoutMs} ms; read the page as loaded so far`); + if (page.url() === "about:blank") throw new Error(`page did not load within ${timeoutMs} ms`); + } + await unstick(page, guard, notes); + await withTimeout(autoScroll(page), 8000, "scrolling").catch(() => notes.push("could not scroll the page")); if (settleMs) await new Promise((r) => setTimeout(r, settleMs)); - const signals = await page.evaluate(extractSignals); - return { status: res ? res.status() : null, finalUrl: page.url(), signals }; + await unstick(page, guard, notes); + const signals = await withTimeout(extractStable(page), 30000, "reading the rendered page"); + return { status: res ? res.status() : null, finalUrl: page.url(), signals, notes }; } finally { - await page.close(); + await guard?.detach().catch(() => {}); + await closeQuietly(page); } } -export async function renderGap(url, { agent, timeoutMs = 30000, settleMs = 1000, browser } = {}) { - const raw = await fetchRaw(url, { userAgent: agent.ua, timeoutMs }); - if (!raw.ok) throw new Error(`Raw fetch as ${agent.token} failed: ${raw.error}`); +const samePage = (a, b) => { + try { + const x = new URL(a); + const y = new URL(b); + const strip = (u) => u.host.replace(/^www\./, "") + u.pathname.replace(/\/+$/, ""); + return strip(x) === strip(y); + } catch { + return a === b; + } +}; + +export async function renderGap(url, { agent, timeoutMs = 30000, settleMs = 1000, viewport = "desktop", headers = {}, browser } = {}) { + ({ url, headers } = splitCredentials(url, headers)); + const raw = await fetchRaw(url, { userAgent: agent.ua, timeoutMs, headers }); + if (!raw.ok) { + // Redirect loops (cookie walls) and resets are what the crawler gets, so they are + // findings. A name that does not resolve is the caller's mistake, so it throws. + if (/ENOTFOUND|EAI_AGAIN|Invalid URL/i.test(raw.error)) throw new Error(`cannot reach ${url}: ${raw.error}`); + return { + version: 1, url, agent: agent.token, viewport, + raw: { error: raw.error }, rendered: null, notes: [], + findings: [{ severity: "critical", signal: "access", message: `${agent.token} could not load the page: ${raw.error}.` }], + }; + } + + const result = { + version: 1, + url, + agent: agent.token, + viewport, + raw: { status: raw.status, finalUrl: raw.finalUrl, bytes: raw.bytes, contentType: raw.contentType, xRobotsTag: raw.xRobotsTag }, + rendered: null, + findings: [], + notes: [], + }; + + // When the crawler never gets the page, a content diff is noise. Report the block. + const blockers = []; + if (raw.status >= 400) blockers.push(`${agent.token} got HTTP ${raw.status}, so it never sees this page's content.`); + if (raw.challenge) blockers.push(`${agent.token} was served a bot challenge page instead of the content.`); + if (!raw.isHtml) blockers.push(`${agent.token} got ${raw.contentType || "a non-HTML response"}, not HTML.`); + if (blockers.length) { + result.findings = blockers.map((message) => ({ severity: "critical", signal: "access", message })); + return result; + } + if (raw.truncated) result.notes.push("raw response was larger than 15 MB and was cut off"); + const own = !browser; browser = browser || (await launchBrowser()); try { const rawSignals = await readRaw(browser, raw.body, raw.finalUrl); - const rendered = await readRendered(browser, url, { timeoutMs, settleMs }); - const findings = compare(rawSignals, rendered.signals); - if (raw.status >= 400) { - findings.unshift({ - severity: "critical", - signal: "http", - message: `${agent.token} got HTTP ${raw.status}; a browser got ${rendered.status}.`, - }); + const rendered = await readRendered(browser, url, { timeoutMs, settleMs, viewport, headers }); + result.notes.push(...rendered.notes); + result.raw.signals = strip(rawSignals); + result.rendered = { status: rendered.status, finalUrl: rendered.finalUrl, signals: strip(rendered.signals) }; + result.findings = compare(rawSignals, rendered.signals); + + const directives = xRobotsFor(raw.xRobotsTag, agent.token); + if (directives.includes("noindex") || directives.includes("none")) { + result.findings.unshift({ severity: "critical", signal: "xRobotsTag", message: `The response tells ${agent.token} not to index this page (X-Robots-Tag: ${raw.xRobotsTag}).` }); } - if (raw.challenge) { - findings.unshift({ severity: "critical", signal: "challenge", message: `${agent.token} was served a bot challenge page.` }); + if (rendered.status && rendered.status >= 400) { + result.notes.push(`the browser got HTTP ${rendered.status}; check the URL or pass --header for protected previews`); } - return { - url, - agent: agent.token, - raw: { status: raw.status, finalUrl: raw.finalUrl, bytes: raw.bytes, signals: strip(rawSignals) }, - rendered: { status: rendered.status, finalUrl: rendered.finalUrl, signals: strip(rendered.signals) }, - findings, - }; + if (!samePage(raw.finalUrl, rendered.finalUrl)) { + result.findings.unshift({ + severity: "warning", + signal: "redirect", + message: "The crawler and the browser ended up on different URLs, so they are not reading the same page.", + items: [`raw: ${raw.finalUrl}`, `rendered: ${rendered.finalUrl}`], + }); + } + return result; } finally { - if (own) await browser.close(); + if (own) { + await withTimeout(browser.close(), 10000, "closing the browser").catch(() => browser.process()?.kill("SIGKILL")); + } } } diff --git a/src/report.js b/src/report.js index 105d5a2..b8a6b66 100644 --- a/src/report.js +++ b/src/report.js @@ -13,8 +13,10 @@ const MARK = { critical: red("CRITICAL"), warning: yellow("WARNING "), info: dim export function formatGap(r) { const out = []; out.push(bold(`rendergap: ${r.url}`)); - out.push(dim(`raw fetch as ${r.agent}: HTTP ${r.raw.status}, ${r.raw.bytes} bytes, ${r.raw.signals.wordCount} words`)); - out.push(dim(`rendered in Chrome: HTTP ${r.rendered.status}, ${r.rendered.signals.wordCount} words`)); + const words = r.raw.signals ? `, ${r.raw.signals.wordCount} words` : ""; + out.push(dim(r.raw.error ? `raw fetch as ${r.agent}: failed` : `raw fetch as ${r.agent}: HTTP ${r.raw.status}, ${r.raw.bytes} bytes${words}`)); + if (r.rendered) out.push(dim(`rendered in Chrome (${r.viewport}): HTTP ${r.rendered.status}, ${r.rendered.signals.wordCount} words`)); + for (const n of r.notes || []) out.push(dim(`note: ${n}`)); out.push(""); if (!r.findings.length) { out.push(green("No gap found. What a browser shows is also in the raw HTML.")); @@ -47,6 +49,7 @@ export function formatCrawlers(r) { else if (x.http?.error) detail = ` (${x.http.error})`; else if (x.note) detail = ` (${x.note})`; out.push(`${x.token.padEnd(width)} ${colored}${dim(detail)}`); + if (x.robotsDiffers) out.push(dim(`${"".padEnd(width)} robots.txt served to this crawler differs from the browser's (HTTP ${x.robotsDiffers.status})`)); } out.push(""); out.push(dim("Requests come from your IP with each crawler's user agent. Firewalls that verify crawlers by IP may treat the real crawler differently.")); diff --git a/src/robots.js b/src/robots.js index b53d236..da3e379 100644 --- a/src/robots.js +++ b/src/robots.js @@ -7,7 +7,7 @@ export function parseRobots(text) { const groups = []; let current = null; let lastWasAgent = false; - for (const rawLine of String(text || "").split(/\r?\n/)) { + for (const rawLine of String(text || "").replace(/^\uFEFF/, "").split(/\r\n|\r|\n/)) { const line = rawLine.replace(/#.*$/, "").trim(); if (!line) continue; const idx = line.indexOf(":"); @@ -19,12 +19,13 @@ export function parseRobots(text) { current = { agents: [], rules: [] }; groups.push(current); } - current.agents.push(value.toLowerCase()); + // "GPTBot/1.1" and "GPTBot" name the same product token. + current.agents.push(value.split("/")[0].trim().toLowerCase()); lastWasAgent = true; } else if (key === "allow" || key === "disallow") { lastWasAgent = false; if (!current) continue; - current.rules.push({ allow: key === "allow", path: value }); + current.rules.push({ allow: key === "allow", path: normalizePath(value) }); } else { lastWasAgent = false; } @@ -32,9 +33,28 @@ export function parseRobots(text) { return groups; } +// RFC 9309 compares percent-encoded forms: decode what is safe to decode, then encode +// everything outside the unreserved set the same way on both sides. +export function normalizePath(p) { + let out = ""; + for (let i = 0; i < p.length; i++) { + const ch = p[i]; + if (ch === "%" && /^[0-9a-f]{2}$/i.test(p.slice(i + 1, i + 3))) { + const decoded = String.fromCharCode(parseInt(p.slice(i + 1, i + 3), 16)); + out += /[A-Za-z0-9\-._~]/.test(decoded) ? decoded : "%" + p.slice(i + 1, i + 3).toUpperCase(); + i += 2; + } else if (/[A-Za-z0-9\-._~!$&'()*+,;=:@\/?#[\]]/.test(ch)) { + out += ch; + } else { + out += [...new TextEncoder().encode(ch)].map((b) => "%" + b.toString(16).toUpperCase().padStart(2, "0")).join(""); + } + } + return out; +} + function selectGroups(groups, token) { const t = token.toLowerCase(); - const exact = groups.filter((g) => g.agents.some((a) => a !== "*" && t.startsWith(a))); + const exact = groups.filter((g) => g.agents.some((a) => a !== "*" && a === t)); if (exact.length) return { groups: exact, matchedBy: "name" }; const star = groups.filter((g) => g.agents.includes("*")); return { groups: star, matchedBy: star.length ? "wildcard" : "none" }; @@ -55,11 +75,14 @@ function ruleToRegex(path) { } export function isAllowed(groups, token, pathWithQuery) { + const path = normalizePath(pathWithQuery || "/"); + // robots.txt itself is always allowed. + if (path === "/robots.txt") return { allowed: true, matchedBy: "implicit", rule: null }; const { groups: selected, matchedBy } = selectGroups(groups, token); const rules = selected.flatMap((g) => g.rules).filter((r) => r.path !== ""); let best = null; for (const r of rules) { - if (!ruleToRegex(r.path).test(pathWithQuery)) continue; + if (!ruleToRegex(r.path).test(path)) continue; const len = r.path.length; if (!best || len > best.len || (len === best.len && r.allow && !best.rule.allow)) { best = { rule: r, len }; diff --git a/test/hostile.test.js b/test/hostile.test.js new file mode 100644 index 0000000..fa1e912 --- /dev/null +++ b/test/hostile.test.js @@ -0,0 +1,469 @@ +// A local server that misbehaves in the ways real sites do. Each route exercises one +// weakness. Browser tests need Chrome (CHROME_PATH or the Puppeteer download). + +import { test, before, after } from "node:test"; +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { existsSync } from "node:fs"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { findAgent } from "../src/agents.js"; +import { fetchRaw, MAX_BYTES } from "../src/fetch.js"; +import { checkCrawlers } from "../src/crawlers.js"; + +const run = promisify(execFile); +const BIN = new URL("../bin/rendergap.js", import.meta.url).pathname; +const gptbot = findAgent("gptbot"); + +const PARA = "We repair water heaters, leaks and drains for homes and small businesses across the valley, usually the same day you call."; +const STATIC_BODY = `<h1>Acme Plumbing</h1><h2>Services</h2><p>${PARA}</p><p>Call <a href="tel:+15755550100">(575) 555-0100</a> any time for a licensed plumber.</p>`; +const page = (body, head = "<title>Acme Plumbing") => `${head}${body}`; + +const routes = { + "/static": (req, res) => html(res, page(STATIC_BODY)), + + // windows-1252 bytes, declared only in a meta tag. + "/latin1": (req, res) => { + const doc = `Café Olé

Café Olé

Fresh piñon coffee roasted here every morning, with pastries baked before sunrise.

`; + res.writeHead(200, { "content-type": "text/html" }); + res.end(Buffer.from(doc, "latin1")); + }, + + // Section that only appears once it scrolls into view. + "/lazy": (req, res) => + html(res, page(`${STATIC_BODY}
+`)), + + // Content inside a web component's open shadow root. + "/shadow": (req, res) => + html(res, page(`

${PARA}

+`)), + + // Never goes network-idle: a request that never finishes. + "/longpoll": (req, res) => html(res, page(`${STATIC_BODY}`)), + "/hang": () => {}, + + // Browser navigates elsewhere with JavaScript. + "/jsredirect": (req, res) => html(res, page(`

${PARA}

`)), + + // Bot gets 403, browsers get the page. + "/botwall": (req, res) => { + if (/GPTBot/.test(req.headers["user-agent"])) { + res.writeHead(403, { "content-type": "text/html" }); + return res.end("Forbidden"); + } + html(res, page(STATIC_BODY)); + }, + + // JS adds a cookie banner and a chat widget: not content, should not be findings. + "/overlays": (req, res) => + html(res, page(`${STATIC_BODY}`)), + + // A custom consent banner with only utility classes, fixed to the bottom. + "/tailwindbanner": (req, res) => + html(res, page(`${STATIC_BODY}`)), + + // A sticky header with real navigation text must still count. + "/stickyheader": (req, res) => + html(res, page(`

${PARA}

`)), + + // A page that sabotages the globals an extractor would lean on. + "/sabotage": (req, res) => + html(res, page(`

${PARA}

`)), + + // alert() on load would freeze a headless page. + "/alert": (req, res) => + html(res, page(`${STATIC_BODY}`)), + + // HTTP basic auth, as on many staging servers. + "/basic": (req, res) => { + const expected = "Basic " + Buffer.from("stage:s3cret!").toString("base64"); + if (req.headers.authorization !== expected) { + res.writeHead(401, { "www-authenticate": 'Basic realm="staging"' }); + return res.end(); + } + html(res, page(STATIC_BODY)); + }, + + // Meta refresh to another page. + "/metarefresh": (req, res) => + html(res, page(`

${PARA}

`, `Moved`)), + + // Content first, then a script that never yields. + "/spin": (req, res) => + html(res, page(`${STATIC_BODY}
`)), + + // A stray debugger statement in production code. + "/debugger": (req, res) => + html(res, page(`

${PARA}

`)), + + // A bakery's real "cookies" section, added by JS: this IS content. + "/bakery": (req, res) => + html(res, page(`

${PARA}

`)), + + // Same words, typed differently by the server and by the client script. + "/quotes": (req, res) => + html(res, page(`

We\u2019re open \u2014 call today

Placeholder

`)), + + // Endless redirect for clients without cookies. + "/loop": (req, res) => { + res.writeHead(302, { location: "/loop" }); + res.end(); + }, + + "/pdf": (req, res) => { + res.writeHead(200, { "content-type": "application/pdf" }); + res.end("%PDF-1.4 fake"); + }, + + "/xrobots": (req, res) => { + res.writeHead(200, { "content-type": "text/html", "x-robots-tag": "GPTBot: noindex" }); + res.end(page(STATIC_BODY)); + }, + + // Staging preview that needs a bypass header on every request. + "/protected": (req, res) => { + if (req.headers["x-preview-token"] !== "letmein") { + res.writeHead(401, { "content-type": "text/html" }); + return res.end("Unauthorized"); + } + html(res, page(STATIC_BODY)); + }, + + "/huge": (req, res) => { + res.writeHead(200, { "content-type": "text/html" }); + const chunk = Buffer.alloc(1024 * 1024, 97); + let sent = 0; + const pump = () => { + while (sent < 20) { + sent++; + if (!res.write(chunk)) return res.once("drain", pump); + } + res.end(); + }; + pump(); + }, + + // Rate limits every crawler request once, then lets it through. + "/ratelimited": (() => { + const seen = new Set(); + return (req, res) => { + const ua = req.headers["user-agent"]; + if (/bot/i.test(ua) && !seen.has(ua)) { + seen.add(ua); + res.writeHead(429, { "retry-after": "0" }); + return res.end(); + } + html(res, page(STATIC_BODY)); + }; + })(), +}; + +function html(res, body) { + res.writeHead(200, { "content-type": "text/html; charset=utf-8" }); + res.end(body); +} + +let server; +let origin; +let robotsBody = "User-agent: *\nAllow: /\n"; +let robotsStatus = 200; +let cloakRobots = false; + +before(async () => { + server = createServer((req, res) => { + const path = req.url.split("?")[0]; + if (path === "/robots.txt") { + res.writeHead(robotsStatus, { "content-type": "text/plain" }); + // Some servers hand AI crawlers a stricter robots.txt than browsers. + if (cloakRobots && /GPTBot/.test(req.headers["user-agent"])) return res.end("User-agent: GPTBot\nDisallow: /\n"); + return res.end(robotsBody + `# generated ${Date.now()}\n`); + } + (routes[path] || ((q, r) => { r.writeHead(404); r.end(); }))(req, res); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + origin = `http://127.0.0.1:${server.address().port}`; +}); + +after(() => server.closeAllConnections?.() ?? server.close()); + +async function hasChrome() { + if (process.env.CHROME_PATH) return existsSync(process.env.CHROME_PATH); + try { + const { default: p } = await import("puppeteer"); + return existsSync(p.executablePath()); + } catch { + return false; + } +} + +const gap = async (path, opts = {}) => { + const { renderGap } = await import("../src/render.js"); + return renderGap(origin + path, { agent: gptbot, settleMs: 100, ...opts }); +}; +const signals = (r) => r.findings.map((f) => f.signal); + +test("fetch decodes windows-1252 from the meta tag", async () => { + const r = await fetchRaw(`${origin}/latin1`, { userAgent: "t" }); + assert.match(r.body, /Café Olé/); +}); + +test("fetch caps huge responses", async () => { + const r = await fetchRaw(`${origin}/huge`, { userAgent: "t" }); + assert.equal(r.truncated, true); + assert.equal(r.bytes, MAX_BYTES); +}); + +test("crawler check retries once after a 429", async () => { + const r = await checkCrawlers(`${origin}/ratelimited`, { delayMs: 0, agents: [gptbot] }); + assert.equal(r.results[0].status, "allowed"); +}); + +test("robots.txt server error means disallow all", async () => { + robotsStatus = 503; + try { + const r = await checkCrawlers(`${origin}/static`, { delayMs: 0, agents: [gptbot] }); + assert.equal(r.results[0].status, "blocked-robots"); + } finally { + robotsStatus = 200; + } +}); + +test("robots.txt served as an HTML page applies no rules", async () => { + robotsBody = "Disallow: /"; + try { + const r = await checkCrawlers(`${origin}/static`, { delayMs: 0, agents: [gptbot] }); + assert.equal(r.results[0].status, "allowed"); + assert.match(r.robots.note, /HTML/); + } finally { + robotsBody = "User-agent: *\nAllow: /\n"; + } +}); + +test("crawler-specific X-Robots-Tag is reported", async () => { + const r = await checkCrawlers(`${origin}/xrobots`, { delayMs: 0, agents: [gptbot, findAgent("claudebot")] }); + const by = Object.fromEntries(r.results.map((x) => [x.id, x.status])); + assert.equal(by.gptbot, "noindex-header"); + assert.equal(by.claudebot, "allowed"); +}); + +test("gap: charset is handled, no false gap", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/latin1"); + assert.deepEqual(r.findings.filter((f) => f.severity !== "info"), []); +}); + +test("gap: lazy-loaded section is found by scrolling", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/lazy"); + const prices = r.findings.find((f) => f.signal === "prices"); + assert.ok(prices, `expected prices in ${signals(r)}`); + assert.ok(signals(r).includes("h2")); +}); + +test("gap: content in shadow DOM is seen", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/shadow"); + for (const s of ["h1", "phone"]) assert.ok(signals(r).includes(s), `expected ${s} in ${signals(r)}`); +}); + +test("gap: a page that never goes idle still gets checked", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/longpoll", { timeoutMs: 3000 }); + assert.ok(r.notes.some((n) => /idle/.test(n)), JSON.stringify(r.notes)); + assert.deepEqual(r.findings.filter((f) => f.severity === "critical"), []); +}); + +test("gap: a JavaScript redirect is flagged", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/jsredirect"); + assert.ok(signals(r).includes("redirect"), `expected redirect in ${signals(r)}`); +}); + +test("gap: a blocked crawler reports the block, not a content diff", async (t) => { + const r = await gap("/botwall"); + assert.deepEqual(signals(r), ["access"]); + assert.match(r.findings[0].message, /403/); +}); + +test("gap: non-HTML is reported, not diffed", async () => { + const r = await gap("/pdf"); + assert.deepEqual(signals(r), ["access"]); +}); + +test("gap: X-Robots-Tag noindex for this crawler is critical", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/xrobots"); + assert.equal(r.findings[0].signal, "xRobotsTag"); +}); + +test("gap: --header reaches both the raw fetch and the browser", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/protected", { headers: { "x-preview-token": "letmein" } }); + assert.equal(r.raw.status, 200); + assert.equal(r.rendered.status, 200); + assert.deepEqual(r.findings.filter((f) => f.severity !== "info"), []); +}); + +test("gap: mobile viewport renders", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/static", { viewport: "mobile" }); + assert.equal(r.viewport, "mobile"); + assert.deepEqual(r.findings.filter((f) => f.severity !== "info"), []); +}); + +test("cli: exit codes", async (t) => { + const bad = await run("node", [BIN, "--fail-on", "nope", `${origin}/static`]).catch((e) => e); + assert.equal(bad.code, 2); + const crawlersClean = await run("node", [BIN, "crawlers", `${origin}/static`, "--only", "gptbot", "--fail-on", "blocked"]); + assert.match(crawlersClean.stdout, /GPTBot\s+allowed/); + const blocked = await run("node", [BIN, "crawlers", `${origin}/xrobots`, "--only", "gptbot", "--fail-on", "blocked"]).catch((e) => e); + assert.equal(blocked.code, 1); + const json = await run("node", [BIN, "crawlers", `${origin}/static`, "--only", "gptbot", "--json"]); + assert.equal(JSON.parse(json.stdout).results[0].status, "allowed"); + if (!(await hasChrome())) return; + const clean = await run("node", [BIN, `${origin}/static`]); + assert.match(clean.stdout, /No gap found/); + const crit = await run("node", [BIN, `${origin}/botwall`]).catch((e) => e); + assert.equal(crit.code, 1); +}); + +test("robots.txt served differently to a crawler is caught", async () => { + cloakRobots = true; + try { + const r = await checkCrawlers(`${origin}/static`, { delayMs: 0, agents: [gptbot, findAgent("claudebot")] }); + const by = Object.fromEntries(r.results.map((x) => [x.id, x])); + assert.equal(by.gptbot.status, "blocked-robots"); + assert.ok(by.gptbot.robotsDiffers); + assert.equal(by.claudebot.status, "allowed"); + assert.equal(by.claudebot.robotsDiffers, undefined, "timestamp comment alone must not count as a difference"); + } finally { + cloakRobots = false; + } +}); + +test("gap: cookie banners and dialogs are not findings", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/overlays"); + assert.deepEqual(r.findings.filter((f) => f.severity !== "info"), [], JSON.stringify(r.findings)); +}); + +test("gap: a bakery's cookie section is still content", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/bakery"); + const h2 = r.findings.find((f) => f.signal === "h2"); + assert.ok(h2 && h2.items.includes("Our cookies"), JSON.stringify(r.findings)); +}); + +test("gap: curly quotes and dashes compare equal", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/quotes"); + assert.ok(!r.findings.some((f) => f.signal === "h1"), JSON.stringify(r.findings)); +}); + +test("gap: a redirect loop is a finding, not a crash", async () => { + const r = await gap("/loop"); + assert.equal(r.findings[0].signal, "access"); + assert.match(r.findings[0].message, /could not load/); +}); + +test("gap: an unresolvable host throws", async () => { + const { renderGap } = await import("../src/render.js"); + await assert.rejects(renderGap("http://does-not-exist.invalid/", { agent: gptbot }), /cannot reach/); +}); + +test("gap: a utility-class consent banner is not a finding", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/tailwindbanner"); + assert.deepEqual(r.findings.filter((f) => f.signal === "blocks" || f.signal === "h2"), [], JSON.stringify(r.findings)); +}); + +test("gap: a sticky header that mentions cookies is still content", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/stickyheader"); + const h2 = r.findings.find((f) => f.signal === "h2"); + assert.ok(h2 && h2.items.includes("Emergency plumbing in Las Cruces"), JSON.stringify(r.findings)); +}); + +test("gap: a page that sabotages globals cannot break extraction", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/sabotage"); + for (const sig of ["h1", "phone"]) assert.ok(signals(r).includes(sig), `expected ${sig} in ${signals(r)}`); + assert.deepEqual(r.rendered.signals.jsonLdTypes, ["Plumber"]); +}); + +test("gap: alert() and confirm() do not hang the check", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/alert", { timeoutMs: 15000 }); + assert.ok(r.notes.some((n) => /dialog/.test(n)), JSON.stringify(r.notes)); + assert.deepEqual(r.findings.filter((f) => f.severity !== "info"), []); +}); + +test("credentials in the URL become a header and never reach the output", async (t) => { + const port = new URL(origin).port; + const withCreds = `http://stage:s3cret%21@127.0.0.1:${port}/basic`; + const c = await checkCrawlers(withCreds, { delayMs: 0, agents: [gptbot] }); + assert.equal(c.results[0].status, "allowed"); + assert.ok(!JSON.stringify(c).includes("s3cret")); + if (!(await hasChrome())) return; + const r = await gap(`/basic`, { headers: {} }).catch(() => null); + const { renderGap } = await import("../src/render.js"); + const g = await renderGap(withCreds, { agent: gptbot, settleMs: 100 }); + assert.equal(g.raw.status, 200); + assert.equal(g.rendered.status, 200); + assert.ok(!JSON.stringify(g).includes("s3cret")); +}); + +test("gap: a meta refresh to another page is flagged", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const r = await gap("/metarefresh"); + assert.ok(signals(r).includes("redirect"), `expected redirect in ${signals(r)}`); +}); + +test("gap: a page stuck in an infinite loop still gets read", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const started = Date.now(); + const r = await gap("/spin", { timeoutMs: 5000 }); + assert.ok(Date.now() - started < 60000, "took too long"); + assert.ok(r.findings.some((f) => f.signal === "h2"), JSON.stringify(r.findings)); +}); + +test("gap: a debugger statement in site code does not pause the check", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + const started = Date.now(); + const r = await gap("/debugger"); + assert.ok(Date.now() - started < 20000, "paused on debugger statement"); + assert.ok(r.findings.some((f) => f.signal === "h2"), JSON.stringify(r.findings)); +}); diff --git a/test/units.test.js b/test/units.test.js new file mode 100644 index 0000000..05e01a5 --- /dev/null +++ b/test/units.test.js @@ -0,0 +1,78 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { looksLikeChallenge, xRobotsFor } from "../src/fetch.js"; +import { parseRobots, isAllowed, normalizePath } from "../src/robots.js"; +import { classify } from "../src/crawlers.js"; +import { withBase } from "../src/render.js"; + +test("a contact form with reCAPTCHA is not a challenge page", () => { + const page = `Contact us +
Please complete the captcha to send.
`; + assert.equal(looksLikeChallenge(page), false); +}); + +test("vendor challenge pages are detected", () => { + assert.equal(looksLikeChallenge("Just a moment..."), true); + assert.equal(looksLikeChallenge(''), true); + assert.equal(looksLikeChallenge(''), true); + assert.equal(looksLikeChallenge("Verify you are human"), true); +}); + +test("a long article that mentions 'checking your browser' is not a challenge", () => { + const long = "How bot checks work" + "Checking your browser is a phrase. ".repeat(2000) + ""; + assert.equal(looksLikeChallenge(long), false); +}); + +test("X-Robots-Tag is scoped to the named crawler", () => { + assert.deepEqual(xRobotsFor("GPTBot: noindex", "GPTBot"), ["noindex"]); + assert.deepEqual(xRobotsFor("GPTBot: noindex", "ClaudeBot"), []); + assert.deepEqual(xRobotsFor("noindex, nofollow", "ClaudeBot"), ["noindex", "nofollow"]); + assert.deepEqual(xRobotsFor("", "GPTBot"), []); + assert.ok(!xRobotsFor("max-snippet:-1", "GPTBot").includes("noindex")); +}); + +test("robots.txt with a byte order mark still parses", () => { + const g = parseRobots("User-agent: *\nDisallow: /secret"); + assert.equal(isAllowed(g, "GPTBot", "/secret/x").allowed, false); +}); + +test("user-agent matching is exact, not prefix", () => { + const g = parseRobots("User-agent: GPT\nDisallow: /\n\nUser-agent: *\nAllow: /"); + assert.equal(isAllowed(g, "GPTBot", "/").allowed, true); + const v = parseRobots("User-agent: GPTBot/1.1\nDisallow: /"); + assert.equal(isAllowed(v, "GPTBot", "/").allowed, false); +}); + +test("percent-encoding is normalized on both sides", () => { + const g = parseRobots("User-agent: *\nDisallow: /café\nDisallow: /a%2fb"); + assert.equal(isAllowed(g, "GPTBot", "/caf%C3%A9/menu").allowed, false); + assert.equal(isAllowed(g, "GPTBot", "/café/menu").allowed, false); + assert.equal(isAllowed(g, "GPTBot", "/a%2Fb").allowed, false); + assert.equal(normalizePath("/%7Euser"), "/~user"); +}); + +test("robots.txt itself is always allowed", () => { + const g = parseRobots("User-agent: *\nDisallow: /"); + assert.equal(isAllowed(g, "GPTBot", "/robots.txt").allowed, true); +}); + +test("classifier separates rate limits and blocked baselines", () => { + const allow = { allowed: true }; + const ok = { ok: true, status: 200, bytes: 50000, challenge: false, xRobotsTag: "" }; + assert.equal(classify(allow, { ...ok, status: 429 }, ok), "rate-limited"); + const blockedBase = { ...ok, status: 403 }; + assert.equal(classify(allow, { ...ok, status: 403 }, blockedBase), "inconclusive"); + const challengedBase = { ...ok, challenge: true }; + assert.equal(classify(allow, { ...ok, challenge: true }, challengedBase), "inconclusive"); + assert.equal(classify(allow, { ...ok, xRobotsTag: "GPTBot: noindex" }, ok, "ClaudeBot"), "allowed"); + assert.equal(classify(allow, { ...ok, xRobotsTag: "GPTBot: noindex" }, ok, "GPTBot"), "noindex-header"); +}); + +test("base tag injection ignores inside scripts and comments", () => { + const html = ''; + assert.match(withBase(html, "https://a.test/p"), //); + const real = ''; + assert.equal(withBase(real, "https://a.test/"), real); + assert.match(withBase("

no head

", "https://a.test/"), /^x", "https://a.test/?a=1&b=2"), /<\/head>/); +});