From 9be19dcbcc3fa1c1d146eb6e9fbd6ab909a66846 Mon Sep 17 00:00:00 2001 From: joshuarg007 Date: Sun, 15 Nov 2026 15:58:01 -0700 Subject: [PATCH] Scope secrets to the checked origin, separate server and JavaScript gaps, pass Google's robots.txt cases: 0.3.0 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 12 +++ README.md | 59 ++++++++----- action.yml | 2 +- package.json | 4 +- src/diff.js | 129 +++++++++++++++++----------- src/fetch.js | 42 ++++++--- src/render.js | 120 +++++++++++++++++++++++--- src/report.js | 37 ++++++-- src/robots.js | 54 ++++++++---- test/conformance.test.js | 180 +++++++++++++++++++++++++++++++++++++++ test/e2e.test.js | 3 +- test/hostile.test.js | 95 ++++++++++++++++++++- test/leak.test.js | 118 +++++++++++++++++++++++++ test/units.test.js | 3 +- 14 files changed, 732 insertions(+), 126 deletions(-) create mode 100644 test/conformance.test.js create mode 100644 test/leak.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 534cb01..cfb158f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.3.0 + +Changes prompted by an outside review. + +- **Security: custom headers and URL credentials are scoped to the checked origin.** Before this release, `--header` values (for example a preview bypass token) were sent with every request the browser made, including third-party scripts, images and beacons, and both the plain fetch and the browser carried them across cross-origin redirects. They are now added per request for the original origin only and dropped on any redirect that leaves it. `test/leak.test.js` proves no secret reaches a second origin. If you passed secrets with `--header` or the Action's `headers` input on 0.2.0 or earlier against pages that load third-party resources, rotate them. +- **Method: three readings instead of two.** The render gap used to compare the crawler's HTML with the rendered page, which changed two things at once (user agent and JavaScript), so a server sending bots a thinner page was reported as a JavaScript problem. rendergap now also reads the HTML a real browser receives and reports server differences and JavaScript differences separately. JSON output is version 2 with a `layer` on every finding and a `summary`. +- The rendered page and the plain browser request now send a user agent that matches the running Chrome, with matching client hints. Before, overriding the user agent dropped Chrome's client hints, and bot protection that checks them could block the rendered page. +- robots.txt parsing now passes a port of Google's reference conformance cases: unknown and Sitemap lines no longer split a group, user-agent values are read up to the first non-token character ("Foo Bar" means "foo"), lines without a colon and Google's documented misspellings are accepted, and percent-encoded characters are no longer decoded (`%62` does not match `b`). +- Clearer output grouped by cause, correct plurals, and a summary that does not report more than 100%. +- Test servers no longer keep the process alive when a name filter skips their tests. + + ## 0.2.0 Hardening pass: every protection below has a hostile-page test, and each was verified by breaking it on purpose and watching a test fail. diff --git a/README.md b/README.md index f9aced7..63c2dd1 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # rendergap -See what AI crawlers miss on your page. +See what a non-JavaScript crawler receives from your page, what only appears after rendering, and whether major crawler user agents are blocked. -Several major AI crawlers have been documented fetching pages without running JavaScript, and behavior varies by crawler. When a site builds its headings, prices, phone number or structured data in the browser, a crawler that reads only the first HTML response gets a thinner page than your visitors do. rendergap shows you that difference, and checks whether each crawler can reach the page at all. +Several major AI crawlers have been documented fetching pages without running JavaScript, and behavior varies by crawler. Some servers also send crawlers different HTML than they send browsers. When a site builds its headings, prices, phone number or structured data in the browser, or serves bots a thinner page, a crawler that reads only the first HTML response gets less than your visitors do. rendergap measures both causes separately, and checks whether each crawler can reach the page at all. In our study of 368 small business websites, 92 lost at least one answer-critical element between the rendered page and the raw HTML ([method and data](https://www.axiondeepdigital.com/research/ai-visibility-study)). This is the check we used, packaged so you can run it on your own pages and in CI. @@ -25,30 +25,43 @@ CHROME_PATH=/usr/bin/google-chrome rendergap https://example.com rendergap https://example.com/services ``` -rendergap fetches the page the way a crawler does (plain HTTP, the crawler's user agent, no JavaScript), then loads it in headless Chrome like a visitor. Both sides go through the same extractor, so the comparison is like for like. It reports what the rendered page has that the raw HTML does not: +rendergap takes three readings of the same URL and reads each one with the same extractor: -| Signal | Severity when missing from raw HTML | +| Reading | What it is | +|---|---| +| A, crawler HTML | Plain HTTP request with the crawler's user agent, no JavaScript | +| B, browser HTML | Plain HTTP request with a real Chrome user agent and client hints, no JavaScript | +| C, rendered page | Headless Chrome with JavaScript on, scrolled, settled | + +Comparing **A with B** shows how the server treats the crawler. Comparing **B with C** shows what JavaScript adds. Keeping them apart matters: if a server simply sends GPTBot a thinner page, that is not a rendering problem, and rendergap will not call it one. If bot protection blocks the plain browser request (B), rendergap says the two causes could not be separated and compares A with C directly. + +| Signal | Severity when missing | |---|---| | Title, canonical, H1, structured data (JSON-LD types), phone numbers | critical | -| Canonical or meta robots changed by JavaScript | critical | -| Raw HTML holds under half the rendered text | critical | +| Canonical or meta robots that differ between readings | critical | +| Under half the text of the later reading | critical | | H2 headings, meta description, prices, email links, FAQ items | warning | -| Forms, individual text blocks, links with no accessible name | info | +| The crawler and browser end on different URLs | warning | +| Forms, individual text blocks, links with no accessible name, a crawler that gets more text than browsers | info | ``` rendergap: https://example.com/ -raw fetch as GPTBot: HTTP 200, 1840 bytes, 3 words -rendered in Chrome: HTTP 200, 412 words +GPTBot HTML: HTTP 200, 1840 bytes, 96 words +browser HTML: HTTP 200, 6120 bytes, 288 words +rendered page (desktop): HTTP 200, 412 words +GPTBot's HTML carries 23% of the text a visitor sees (96 of 412 words). -CRITICAL Page title only appears after JavaScript runs. - - Acme Plumbing | Las Cruces -CRITICAL H1 text missing from the raw HTML. +How the server treats GPTBot (its HTML vs a browser's HTML, both before JavaScript) +CRITICAL H1 text missing from the HTML sent to GPTBot, though browsers get it. - Acme Plumbing in Las Cruces -CRITICAL Structured data types only present after JavaScript runs. - - Plumber -CRITICAL Phone numbers only visible after JavaScript runs. +CRITICAL A phone number is in the HTML sent to browsers but not in the HTML sent to GPTBot. - (575) 555-0100 -CRITICAL Raw HTML carries 1% of the rendered text (3 of 412 words). +CRITICAL The HTML sent to GPTBot carries 33% of the text browsers get (96 of 288 words). + +What JavaScript adds (browser HTML vs the rendered page) +WARNING Pricing only appears after JavaScript runs. + - $129 + - $1,450 ``` Options: @@ -56,7 +69,9 @@ Options: - `--agent ` the crawler whose raw fetch to compare. Default `gptbot`. - `--fail-on ` exit 1 when a finding is at least `critical` (default), `warning` or `info`. Use `never` to always exit 0. - `--viewport mobile` render the page as a phone instead of a desktop browser. -- `--header "Name: value"` send an extra header on every request, repeatable. Use it for preview deployments behind a bypass token. Credentials in the URL (`https://user:pass@staging.example.com/`) are sent as basic auth and never printed. +- `--header "Name: value"` send an extra header, repeatable. Use it for preview deployments behind a bypass token. Credentials in the URL (`https://user:pass@staging.example.com/`) are sent as basic auth and never printed. + +**Custom headers and URL credentials only ever go to the origin you are checking.** They are added per request, never to third-party scripts, fonts, analytics or images the page loads, and they are dropped the moment a redirect leaves that origin. `test/leak.test.js` runs a second server on another origin and fails if a secret ever reaches it. - `--json` machine-readable output. - `--timeout ` page load timeout. @@ -104,9 +119,11 @@ Results are printed to the log and to the job summary. ## How the comparison works -- **Raw side:** the HTTP response body for the crawler's user agent, loaded into Chrome with JavaScript disabled and every subresource blocked. Nothing runs, nothing loads. -- **Rendered side:** a normal page load with a desktop browser user agent, waiting for the network to go quiet, then one more second. -- **Extraction:** one function reads both sides: title, meta tags, canonical, headings, JSON-LD types, phone numbers (tel links and visible text), email links, prices, FAQ items, forms, word count and text blocks. +- **Crawler HTML (A):** the HTTP response for the crawler's user agent, loaded into Chrome with JavaScript disabled and every subresource blocked. Nothing runs, nothing loads. +- **Browser HTML (B):** the same, with the user agent and client hints of the Chrome that renders C, so A and B differ only in user agent, and B and C differ only in JavaScript. +- **Rendered page (C):** a normal page load, waiting for the network to go quiet, scrolling to trigger lazy content, then one more second. +- **Extraction:** one function reads all three: title, meta tags, canonical, headings, JSON-LD types, phone numbers (tel links and visible text), email links, prices, FAQ items, forms, word count and text blocks. +- **JSON output** (`--json`, version 2) includes all three readings, every finding tagged with its `layer` (`server`, `render`, `combined` or `access`), and a `summary` with word counts. ## What it is built to survive @@ -122,7 +139,7 @@ Real pages fight tools. Each of these has a test in `test/hostile.test.js`, and - Pages that never go network-idle are read as loaded. Every in-page step has a deadline and the CLI has a hard ceiling, so a CI job cannot hang. - When the crawler is blocked, served a challenge, sent a PDF or caught in a redirect loop, that is the finding. You do not get a page of false "missing content" noise. - JavaScript and meta refresh redirects are flagged, since the crawler and the browser are then reading different pages. -- robots.txt follows RFC 9309: exact product-token matching, longest match wins, `*` and `$` patterns, percent-encoding, byte order marks, and "server error means disallow all". +- robots.txt parsing passes a port of the conformance cases from [Google's open-source robots.txt parser](https://github.com/google/robotstxt) (`test/conformance.test.js`), with the one deliberate difference documented in that file. It follows RFC 9309: exact product-token matching, longest match wins, `*` and `$` patterns, percent-encoding, byte order marks, and "server error means disallow all". Findings are about what is present, not whether a crawler ranks or cites you. A clean report means your content is in the first HTML response. It does not mean an AI system will recommend you. diff --git a/action.yml b/action.yml index 8c6bc34..25d9a7b 100644 --- a/action.yml +++ b/action.yml @@ -1,5 +1,5 @@ name: "rendergap" -description: "Fail the build when key page content only appears after JavaScript runs, or when AI crawlers are blocked." +description: "Fail the build when crawlers get less than visitors: content that needs JavaScript, thinner HTML for bots, or blocked crawler user agents." author: "Axion Deep Labs" branding: icon: "eye" diff --git a/package.json b/package.json index f8c57fc..eab5222 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "rendergap", - "version": "0.2.0", - "description": "See what AI crawlers miss on your page: compare raw HTML with the rendered page, and check which crawlers can reach it at all.", + "version": "0.3.0", + "description": "See what a non-JavaScript crawler receives from your page, what only appears after rendering, and whether major crawler user agents are blocked.", "type": "module", "bin": { "rendergap": "bin/rendergap.js" diff --git a/src/diff.js b/src/diff.js index 86efc5b..49c273b 100644 --- a/src/diff.js +++ b/src/diff.js @@ -1,76 +1,105 @@ -// Compare signals from the raw HTML against the rendered page. -// A finding is something a person sees on the rendered page that a fetch without -// JavaScript does not get. +// Compare the signals of two readings of the same page. +// +// rendergap takes three readings: +// A the HTML the crawler's user agent receives (no JavaScript) +// B the HTML a normal browser receives (no JavaScript) +// C the page a browser shows after JavaScript runs +// compare(A, B, serverVoice) isolates how the server treats the crawler. +// compare(B, C, renderVoice) isolates what JavaScript adds. +// Each finding is something present in the later reading and absent from the earlier one. const SEVERITY = { critical: 3, warning: 2, info: 1 }; -function missingFrom(rawList, renderedList) { - const raw = new Set(rawList.map((s) => s.toLowerCase())); - return renderedList.filter((s) => !raw.has(s.toLowerCase())); +export const renderVoice = { + layer: "render", + only: (what, plural) => `${what} only ${plural ? "appear" : "appears"} after JavaScript runs.`, + missing: (what) => `${what} missing from the HTML before JavaScript runs.`, + changed: (what) => `JavaScript changes the ${what}.`, + ratio: (pct, a, b) => `The HTML before JavaScript carries ${pct}% of the rendered text (${a} of ${b} words).`, + from: "before JS", + to: "after JS", +}; + +export function serverVoice(token) { + return { + layer: "server", + only: (what, plural) => `${what} ${plural ? "are" : "is"} in the HTML sent to browsers but not in the HTML sent to ${token}.`, + missing: (what) => `${what} missing from the HTML sent to ${token}, though browsers get it.`, + changed: (what) => `The server sends ${token} a different ${what} than it sends browsers.`, + ratio: (pct, a, b) => `The HTML sent to ${token} carries ${pct}% of the text browsers get (${a} of ${b} words).`, + from: token, + to: "browser", + }; } -export function compare(raw, rendered) { +function missingFrom(earlier, later) { + const seen = new Set(earlier.map((s) => s.toLowerCase())); + return later.filter((s) => !seen.has(s.toLowerCase())); +} + +export function compare(a, b, voice = renderVoice) { const findings = []; const add = (severity, signal, message, items) => - findings.push({ severity, signal, message, ...(items && items.length ? { items } : {}) }); + findings.push({ severity, layer: voice.layer, signal, message, ...(items && items.length ? { items } : {}) }); - if (!raw.title && rendered.title) add("critical", "title", "Page title only appears after JavaScript runs.", [rendered.title]); - else if (raw.title && rendered.title && raw.title !== rendered.title) - add("info", "title", "JavaScript changes the page title.", [`raw: ${raw.title}`, `rendered: ${rendered.title}`]); + if (!a.title && b.title) add("critical", "title", voice.only("The page title"), [b.title]); + else if (a.title && b.title && a.title !== b.title) + add("info", "title", voice.changed("page title"), [`${voice.from}: ${a.title}`, `${voice.to}: ${b.title}`]); - if (!raw.metaDescription && rendered.metaDescription) - add("warning", "metaDescription", "Meta description only appears after JavaScript runs."); + if (!a.metaDescription && b.metaDescription) add("warning", "metaDescription", voice.only("The meta description")); - if (!raw.canonical && rendered.canonical) - add("critical", "canonical", "Canonical link only appears after JavaScript runs.", [rendered.canonical]); - else if (raw.canonical && rendered.canonical && raw.canonical !== rendered.canonical) - add("critical", "canonical", "JavaScript changes the canonical URL.", [`raw: ${raw.canonical}`, `rendered: ${rendered.canonical}`]); + if (!a.canonical && b.canonical) add("critical", "canonical", voice.only("The canonical link"), [b.canonical]); + else if (a.canonical && b.canonical && a.canonical !== b.canonical) + add("critical", "canonical", voice.changed("canonical URL"), [`${voice.from}: ${a.canonical}`, `${voice.to}: ${b.canonical}`]); - if (raw.metaRobots !== rendered.metaRobots && /noindex/i.test(raw.metaRobots + rendered.metaRobots)) - add("critical", "metaRobots", "Meta robots differs between raw and rendered, and one side says noindex.", - [`raw: ${raw.metaRobots || "(none)"}`, `rendered: ${rendered.metaRobots || "(none)"}`]); + if (a.metaRobots !== b.metaRobots && /noindex/i.test(a.metaRobots + b.metaRobots)) + add("critical", "metaRobots", voice.changed("meta robots tag") + " One side says noindex.", + [`${voice.from}: ${a.metaRobots || "(none)"}`, `${voice.to}: ${b.metaRobots || "(none)"}`]); - const h1 = missingFrom(raw.h1, rendered.h1); - if (h1.length) add(raw.h1.length ? "warning" : "critical", "h1", "H1 text missing from the raw HTML.", h1); + const h1 = missingFrom(a.h1, b.h1); + if (h1.length) add(a.h1.length ? "warning" : "critical", "h1", voice.missing("H1 text"), h1); - const h2 = missingFrom(raw.h2, rendered.h2); - if (h2.length) add("warning", "h2", `${h2.length} of ${rendered.h2.length} H2 headings are missing from the raw HTML.`, h2.slice(0, 15)); + const h2 = missingFrom(a.h2, b.h2); + if (h2.length) add("warning", "h2", voice.missing(`${h2.length} of ${b.h2.length} H2 headings`), h2.slice(0, 15)); - const ld = missingFrom(raw.jsonLdTypes, rendered.jsonLdTypes); - if (ld.length) add("critical", "jsonLd", "Structured data types only present after JavaScript runs.", ld); + const ld = missingFrom(a.jsonLdTypes, b.jsonLdTypes); + if (ld.length) add("critical", "jsonLd", voice.only("Structured data"), ld); - const phones = missingFrom(raw.phones, rendered.phones); - if (phones.length) add("critical", "phone", "Phone numbers only visible after JavaScript runs.", phones.map(fmtPhone)); + const phones = missingFrom(a.phones, b.phones); + if (phones.length) add("critical", "phone", voice.only("A phone number"), phones.map(fmtPhone)); - const emails = missingFrom(raw.emails, rendered.emails); - if (emails.length) add("warning", "email", "Email links only present after JavaScript runs.", emails); + const emails = missingFrom(a.emails, b.emails); + if (emails.length) add("warning", "email", voice.only("An email link"), emails); - const prices = missingFrom(raw.prices, rendered.prices); - if (prices.length) add("warning", "prices", "Prices only visible after JavaScript runs.", prices.slice(0, 15)); + const prices = missingFrom(a.prices, b.prices); + if (prices.length) add("warning", "prices", voice.only("Pricing"), prices.slice(0, 15)); - if (rendered.faqItems > raw.faqItems) - add("warning", "faq", `${rendered.faqItems - raw.faqItems} FAQ items (details/summary) only appear after JavaScript runs.`); + if (b.faqItems > a.faqItems) { + const n = b.faqItems - a.faqItems; + add("warning", "faq", voice.only(n === 1 ? "1 FAQ item" : `${n} FAQ items`, n !== 1)); + } - if (rendered.forms > raw.forms) - add("info", "forms", `${rendered.forms - raw.forms} form(s) only appear after JavaScript runs.`); + if (b.forms > a.forms) { + const n = b.forms - a.forms; + add("info", "forms", voice.only(n === 1 ? "1 form" : `${n} forms`, n !== 1)); + } - const ratio = rendered.wordCount ? raw.wordCount / rendered.wordCount : 1; - if (rendered.wordCount >= 50 && ratio < 0.5) - add("critical", "content", `Raw HTML carries ${Math.round(ratio * 100)}% of the rendered text (${raw.wordCount} of ${rendered.wordCount} words).`); - else if (rendered.wordCount >= 50 && ratio < 0.8) - add("warning", "content", `Raw HTML carries ${Math.round(ratio * 100)}% of the rendered text (${raw.wordCount} of ${rendered.wordCount} words).`); + const ratio = b.wordCount ? a.wordCount / b.wordCount : 1; + const pct = Math.round(ratio * 100); + if (b.wordCount >= 50 && ratio < 0.5) add("critical", "content", voice.ratio(pct, a.wordCount, b.wordCount)); + else if (b.wordCount >= 50 && ratio < 0.8) add("warning", "content", voice.ratio(pct, a.wordCount, b.wordCount)); - const rawBlocks = new Set(raw.blocks); - const lostBlocks = rendered.blocks.filter((b) => !rawBlocks.has(b)); - if (lostBlocks.length && rendered.blocks.length) - add(lostBlocks.length / rendered.blocks.length > 0.3 ? "warning" : "info", "blocks", - `${lostBlocks.length} of ${rendered.blocks.length} text blocks are not in the raw HTML.`, - lostBlocks.slice(0, 5).map((b) => (b.length > 140 ? b.slice(0, 137) + "..." : b))); + const earlierBlocks = new Set(a.blocks); + const lostBlocks = b.blocks.filter((x) => !earlierBlocks.has(x)); + if (lostBlocks.length && b.blocks.length) + add(lostBlocks.length / b.blocks.length > 0.3 ? "warning" : "info", "blocks", + voice.missing(`${lostBlocks.length} of ${b.blocks.length} text blocks`), + lostBlocks.slice(0, 5).map((x) => (x.length > 140 ? x.slice(0, 137) + "..." : x))); - if (rendered.unnamedLinks > 0) - add("info", "unnamedLinks", `${rendered.unnamedLinks} link(s) on the rendered page have no accessible name.`); + if (voice.layer === "render" && b.unnamedLinks > 0) + add("info", "unnamedLinks", `${b.unnamedLinks} link(s) on the rendered page have no accessible name.`); - findings.sort((a, b) => SEVERITY[b.severity] - SEVERITY[a.severity]); + findings.sort((x, y) => SEVERITY[y.severity] - SEVERITY[x.severity]); return findings; } diff --git a/src/fetch.js b/src/fetch.js index 7f6ad97..fc9a008 100644 --- a/src/fetch.js +++ b/src/fetch.js @@ -72,29 +72,47 @@ async function readCapped(res) { return { bytes: out, truncated }; } +export const MAX_REDIRECTS = 10; + +// Follows redirects by hand so custom headers (bypass tokens, Authorization) are sent +// only to the origin they were meant for. They are dropped the moment a redirect +// leaves that origin, and never restored. export async function fetchRaw(url, { userAgent, timeoutMs = 20000, headers = {} } = {}) { const started = Date.now(); const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); + const scopedOrigin = new URL(url).origin; try { - const res = await fetch(url, { - redirect: "follow", - signal: controller.signal, - headers: { - accept: "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8", - "accept-language": "en-US,en;q=0.9", - ...headers, - "user-agent": userAgent, - }, - }); + let current = url; + let res; + let redirected = false; + for (let hop = 0; ; hop++) { + const sameOrigin = new URL(current).origin === scopedOrigin; + res = await fetch(current, { + redirect: "manual", + signal: controller.signal, + headers: { + accept: "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8", + "accept-language": "en-US,en;q=0.9", + ...(sameOrigin ? headers : {}), + "user-agent": userAgent, + }, + }); + const location = res.headers.get("location"); + if (res.status < 300 || res.status >= 400 || res.status === 304 || !location) break; + if (hop >= MAX_REDIRECTS) throw new Error(`redirect count exceeded (${MAX_REDIRECTS})`); + await res.body?.cancel().catch(() => {}); + current = new URL(location, current).href; + redirected = true; + } const { bytes, truncated } = await readCapped(res); const contentType = res.headers.get("content-type") || ""; const body = decode(bytes, contentType); return { ok: true, status: res.status, - finalUrl: res.url || url, - redirected: res.redirected, + finalUrl: current, + redirected, contentType, isHtml: !contentType || /html|xml/i.test(contentType), xRobotsTag: res.headers.get("x-robots-tag") || "", diff --git a/src/render.js b/src/render.js index 9db213a..dfb8cfc 100644 --- a/src/render.js +++ b/src/render.js @@ -5,10 +5,11 @@ // blocked; the rendered side is a normal page load. import puppeteer from "puppeteer"; -import { BROWSER_UA, MOBILE_UA } from "./agents.js"; import { fetchRaw, xRobotsFor, splitCredentials } from "./fetch.js"; import { extractSignals } from "./extract.js"; -import { compare } from "./diff.js"; +import { compare, renderVoice, serverVoice } from "./diff.js"; + +const RANK = { critical: 3, warning: 2, info: 1 }; // Every in-page step gets a deadline: a page script stuck in a loop blocks the page's // thread, and an await on it would otherwise never return. @@ -63,6 +64,56 @@ async function closeQuietly(page) { await withTimeout(page.close(), 5000, "closing the page").catch(() => {}); } +// Custom headers often carry secrets (preview bypass tokens, basic auth). Add them per +// request, and only for the origin being checked: never for analytics, fonts, CDNs or a +// redirect that leaves the site. setExtraHTTPHeaders would send them everywhere. +async function scopeHeaders(page, url, headers) { + const origin = new URL(url).origin; + await page.setRequestInterception(true); + page.on("request", (req) => { + if (req.isInterceptResolutionHandled()) return; + let sameOrigin = false; + try { + sameOrigin = new URL(req.url()).origin === origin; + } catch {} + // A redirect hop inherits nothing: decide per request URL. + const base = { ...req.headers() }; + for (const k of Object.keys(headers)) delete base[k.toLowerCase()]; + req.continue({ headers: sameOrigin ? { ...base, ...headers } : base }).catch(() => {}); + }); +} + +// A user agent that matches the Chrome actually running, with matching client hints. +// Overriding the UA without metadata makes Chrome drop its sec-ch-ua headers, and bot +// protection that checks them would then block the rendered side. +export async function browserIdentity(browser, viewport) { + const version = (await browser.version()).replace(/^.*?\//, "") || "138.0.0.0"; + const major = version.split(".")[0]; + const mobile = viewport === "mobile"; + const ua = mobile + ? `Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${major}.0.0.0 Mobile Safari/537.36` + : `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${major}.0.0.0 Safari/537.36`; + const brands = [ + { brand: "Chromium", version: major }, + { brand: "Google Chrome", version: major }, + { brand: "Not.A/Brand", version: "99" }, + ]; + const metadata = { + brands, + fullVersionList: brands.map((b) => ({ ...b, version: b.brand === "Not.A/Brand" ? "99.0.0.0" : version })), + fullVersion: version, + platform: mobile ? "Android" : "Linux", + platformVersion: mobile ? "14.0.0" : "", + architecture: mobile ? "" : "x86", + model: mobile ? "Pixel 8" : "", + mobile, + bitness: mobile ? "" : "64", + wow64: false, + }; + const chHeader = brands.map((b) => `"${b.brand}";v="${b.version}"`).join(", "); + return { ua, metadata, plainHeaders: { "sec-ch-ua": chHeader, "sec-ch-ua-mobile": mobile ? "?1" : "?0", "sec-ch-ua-platform": `"${metadata.platform}"` } }; +} + const VIEWPORTS = { desktop: { width: 1366, height: 900, deviceScaleFactor: 1, isMobile: false, hasTouch: false }, mobile: { width: 412, height: 915, deviceScaleFactor: 2.6, isMobile: true, hasTouch: true }, @@ -168,7 +219,7 @@ async function autoScroll(page, maxMs = 4000) { }, maxMs); } -async function readRendered(browser, url, { timeoutMs, settleMs, viewport, headers }) { +async function readRendered(browser, url, { timeoutMs, settleMs, viewport, headers, identity }) { const page = await browser.newPage(); const notes = []; const guard = await attachGuard(page); @@ -178,9 +229,9 @@ async function readRendered(browser, url, { timeoutMs, settleMs, viewport, heade d.dismiss().catch(() => {}); }); try { - await page.setUserAgent(viewport === "mobile" ? MOBILE_UA : BROWSER_UA); + await page.setUserAgent(identity.ua, identity.metadata); await page.setViewport(VIEWPORTS[viewport] || VIEWPORTS.desktop); - if (Object.keys(headers).length) await page.setExtraHTTPHeaders(headers); + if (Object.keys(headers).length) await scopeHeaders(page, url, headers); let res = null; try { res = await page.goto(url, { waitUntil: "networkidle2", timeout: timeoutMs }); @@ -222,14 +273,14 @@ export async function renderGap(url, { agent, timeoutMs = 30000, settleMs = 1000 // findings. A name that does not resolve is the caller's mistake, so it throws. if (/ENOTFOUND|EAI_AGAIN|Invalid URL/i.test(raw.error)) throw new Error(`cannot reach ${url}: ${raw.error}`); return { - version: 1, url, agent: agent.token, viewport, + version: 2, url, agent: agent.token, viewport, raw: { error: raw.error }, rendered: null, notes: [], - findings: [{ severity: "critical", signal: "access", message: `${agent.token} could not load the page: ${raw.error}.` }], + findings: [{ severity: "critical", layer: "access", signal: "access", message: `${agent.token} could not load the page: ${raw.error}.` }], }; } const result = { - version: 1, + version: 2, url, agent: agent.token, viewport, @@ -245,24 +296,66 @@ export async function renderGap(url, { agent, timeoutMs = 30000, settleMs = 1000 if (raw.challenge) blockers.push(`${agent.token} was served a bot challenge page instead of the content.`); if (!raw.isHtml) blockers.push(`${agent.token} got ${raw.contentType || "a non-HTML response"}, not HTML.`); if (blockers.length) { - result.findings = blockers.map((message) => ({ severity: "critical", signal: "access", message })); + result.findings = blockers.map((message) => ({ severity: "critical", layer: "access", signal: "access", message })); return result; } if (raw.truncated) result.notes.push("raw response was larger than 15 MB and was cut off"); + // Reading B: the same URL as a normal browser, still without JavaScript. Comparing + // the crawler's HTML (A) with B shows how the server treats the crawler; comparing B + // with the rendered page (C) shows what JavaScript adds. Without B those two causes + // would be blended and a server-side difference would be blamed on JavaScript. const own = !browser; browser = browser || (await launchBrowser()); try { + // B uses the same identity as the rendered page, so A vs B differs only in user agent + // and B vs C differs only in JavaScript. + const identity = await browserIdentity(browser, viewport); + const plain = await fetchRaw(url, { userAgent: identity.ua, timeoutMs, headers: { ...identity.plainHeaders, ...headers } }); + const plainUsable = plain.ok && plain.status < 400 && !plain.challenge && plain.isHtml; const rawSignals = await readRaw(browser, raw.body, raw.finalUrl); - const rendered = await readRendered(browser, url, { timeoutMs, settleMs, viewport, headers }); + const plainSignals = plainUsable ? await readRaw(browser, plain.body, plain.finalUrl) : null; + const rendered = await readRendered(browser, url, { timeoutMs, settleMs, viewport, headers, identity }); result.notes.push(...rendered.notes); result.raw.signals = strip(rawSignals); + result.browserHtml = plainUsable + ? { status: plain.status, finalUrl: plain.finalUrl, bytes: plain.bytes, signals: strip(plainSignals) } + : { status: plain.ok ? plain.status : null, error: plain.ok ? undefined : plain.error, usable: false }; result.rendered = { status: rendered.status, finalUrl: rendered.finalUrl, signals: strip(rendered.signals) }; - result.findings = compare(rawSignals, rendered.signals); + + if (plainUsable) { + result.findings = [ + ...compare(rawSignals, plainSignals, serverVoice(agent.token)), + ...compare(plainSignals, rendered.signals, renderVoice), + ]; + // The reverse direction: the crawler gets noticeably more than browsers do. + if (plainSignals.wordCount >= 50 && rawSignals.wordCount > plainSignals.wordCount * 1.25) { + result.findings.push({ + severity: "info", + layer: "server", + signal: "extraForCrawler", + message: `${agent.token} is sent more text than browsers (${rawSignals.wordCount} vs ${plainSignals.wordCount} words). Check that both get the same content.`, + }); + } + } else { + result.notes.push( + "the plain browser request was blocked or failed, so server differences and JavaScript differences could not be separated; findings below compare the crawler's HTML with the rendered page directly", + ); + result.findings = compare(rawSignals, rendered.signals, renderVoice).map((f) => ({ ...f, layer: "combined" })); + } + result.findings.sort((x, y) => RANK[y.severity] - RANK[x.severity]); + + const total = rendered.signals.wordCount; + result.summary = { + crawlerWords: rawSignals.wordCount, + browserHtmlWords: plainUsable ? plainSignals.wordCount : null, + renderedWords: total, + crawlerShareOfRendered: total ? Math.min(100, Math.round((rawSignals.wordCount / total) * 100)) : null, + }; const directives = xRobotsFor(raw.xRobotsTag, agent.token); if (directives.includes("noindex") || directives.includes("none")) { - result.findings.unshift({ severity: "critical", signal: "xRobotsTag", message: `The response tells ${agent.token} not to index this page (X-Robots-Tag: ${raw.xRobotsTag}).` }); + result.findings.unshift({ severity: "critical", layer: "access", signal: "xRobotsTag", message: `The response tells ${agent.token} not to index this page (X-Robots-Tag: ${raw.xRobotsTag}).` }); } if (rendered.status && rendered.status >= 400) { result.notes.push(`the browser got HTTP ${rendered.status}; check the URL or pass --header for protected previews`); @@ -270,9 +363,10 @@ export async function renderGap(url, { agent, timeoutMs = 30000, settleMs = 1000 if (!samePage(raw.finalUrl, rendered.finalUrl)) { result.findings.unshift({ severity: "warning", + layer: "access", signal: "redirect", message: "The crawler and the browser ended up on different URLs, so they are not reading the same page.", - items: [`raw: ${raw.finalUrl}`, `rendered: ${rendered.finalUrl}`], + items: [`${agent.token}: ${raw.finalUrl}`, `browser: ${rendered.finalUrl}`], }); } return result; diff --git a/src/report.js b/src/report.js index b8a6b66..bd249a9 100644 --- a/src/report.js +++ b/src/report.js @@ -10,24 +10,45 @@ const bold = (s) => c(1, s); const MARK = { critical: red("CRITICAL"), warning: yellow("WARNING "), info: dim("INFO ") }; +const LAYERS = [ + ["access", (r) => "Access"], + ["server", (r) => `How the server treats ${r.agent} (its HTML vs a browser's HTML, both before JavaScript)`], + ["render", () => "What JavaScript adds (browser HTML vs the rendered page)"], + ["combined", (r) => `${r.agent}'s HTML vs the rendered page (causes not separated)`], +]; + export function formatGap(r) { const out = []; out.push(bold(`rendergap: ${r.url}`)); - const words = r.raw.signals ? `, ${r.raw.signals.wordCount} words` : ""; - out.push(dim(r.raw.error ? `raw fetch as ${r.agent}: failed` : `raw fetch as ${r.agent}: HTTP ${r.raw.status}, ${r.raw.bytes} bytes${words}`)); - if (r.rendered) out.push(dim(`rendered in Chrome (${r.viewport}): HTTP ${r.rendered.status}, ${r.rendered.signals.wordCount} words`)); + if (r.raw.error) out.push(dim(`${r.agent} HTML: request failed`)); + else out.push(dim(`${r.agent} HTML: HTTP ${r.raw.status}, ${r.raw.bytes} bytes${r.raw.signals ? `, ${r.raw.signals.wordCount} words` : ""}`)); + if (r.browserHtml?.signals) out.push(dim(`browser HTML: HTTP ${r.browserHtml.status}, ${r.browserHtml.bytes} bytes, ${r.browserHtml.signals.wordCount} words`)); + if (r.rendered) out.push(dim(`rendered page (${r.viewport}): HTTP ${r.rendered.status}, ${r.rendered.signals.wordCount} words`)); + const sm = r.summary; + if (sm?.renderedWords) { + out.push( + sm.crawlerWords >= sm.renderedWords + ? `${r.agent}'s HTML already holds as much text as a visitor sees (${sm.crawlerWords} words before JavaScript, ${sm.renderedWords} after).` + : `${r.agent}'s HTML carries ${sm.crawlerShareOfRendered}% of the text a visitor sees (${sm.crawlerWords} of ${sm.renderedWords} words).`, + ); + } for (const n of r.notes || []) out.push(dim(`note: ${n}`)); out.push(""); if (!r.findings.length) { - out.push(green("No gap found. What a browser shows is also in the raw HTML.")); + out.push(green(`No gap found. ${r.agent} gets the same HTML as a browser, and what a browser shows is already in that HTML.`)); return out.join("\n"); } - for (const f of r.findings) { - out.push(`${MARK[f.severity]} ${f.message}`); - for (const item of f.items || []) out.push(dim(` - ${item}`)); + for (const [layer, title] of LAYERS) { + const group = r.findings.filter((f) => (f.layer || "render") === layer); + if (!group.length) continue; + out.push(bold(title(r))); + for (const f of group) { + out.push(`${MARK[f.severity]} ${f.message}`); + for (const item of f.items || []) out.push(dim(` - ${item}`)); + } + out.push(""); } const n = (s) => r.findings.filter((f) => f.severity === s).length; - out.push(""); out.push(`${n("critical")} critical, ${n("warning")} warning, ${n("info")} info`); return out.join("\n"); } diff --git a/src/robots.js b/src/robots.js index da3e379..7cd88f1 100644 --- a/src/robots.js +++ b/src/robots.js @@ -3,45 +3,67 @@ // - within a group, the longest matching rule wins; on a tie, allow wins // - "*" matches any run of characters, a trailing "$" anchors the end +// Keys Google's reference parser accepts, including its documented misspellings. +const KEYS = { + "user-agent": "user-agent", useragent: "user-agent", "user agent": "user-agent", + allow: "allow", + disallow: "disallow", dissallow: "disallow", dissalow: "disallow", disalow: "disallow", diasllow: "disallow", disallaw: "disallow", +}; + +function splitLine(line) { + const idx = line.indexOf(":"); + if (idx !== -1) return [line.slice(0, idx).trim().toLowerCase(), line.slice(idx + 1).trim()]; + // "user-agent FooBot" with no colon is accepted, as in Google's parser. + const m = /^(user[- ]?agent|allow|disallow|dissallow|dissalow|disalow|diasllow|disallaw)\s+(.+)$/i.exec(line); + return m ? [m[1].toLowerCase(), m[2].trim()] : [null, null]; +} + +// The product token is the leading run of letters, "_" and "-": "GPTBot/1.1" and +// "Foo Bar" name "gptbot" and "foo". +function productToken(value) { + if (value.startsWith("*")) return "*"; + const m = /^[A-Za-z_-]+/.exec(value); + return m ? m[0].toLowerCase() : ""; +} + export function parseRobots(text) { const groups = []; let current = null; - let lastWasAgent = false; + let inAgentRun = false; for (const rawLine of String(text || "").replace(/^\uFEFF/, "").split(/\r\n|\r|\n/)) { const line = rawLine.replace(/#.*$/, "").trim(); if (!line) continue; - const idx = line.indexOf(":"); - if (idx === -1) continue; - const key = line.slice(0, idx).trim().toLowerCase(); - const value = line.slice(idx + 1).trim(); + const [rawKey, value] = splitLine(line); + const key = KEYS[rawKey]; if (key === "user-agent") { - if (!current || !lastWasAgent) { + if (!current || !inAgentRun) { current = { agents: [], rules: [] }; groups.push(current); } - // "GPTBot/1.1" and "GPTBot" name the same product token. - current.agents.push(value.split("/")[0].trim().toLowerCase()); - lastWasAgent = true; + const token = productToken(value); + if (token) current.agents.push(token); + inAgentRun = true; } else if (key === "allow" || key === "disallow") { - lastWasAgent = false; + // Only rules end a run of user-agent lines. Sitemap, crawl-delay and unknown + // lines are ignored and do not split a group. + inAgentRun = false; if (!current) continue; current.rules.push({ allow: key === "allow", path: normalizePath(value) }); - } else { - lastWasAgent = false; } } return groups; } -// RFC 9309 compares percent-encoded forms: decode what is safe to decode, then encode -// everything outside the unreserved set the same way on both sides. +// RFC 9309 compares percent-encoded forms: encode everything outside the allowed set +// the same way on both sides. export function normalizePath(p) { let out = ""; for (let i = 0; i < p.length; i++) { const ch = p[i]; if (ch === "%" && /^[0-9a-f]{2}$/i.test(p.slice(i + 1, i + 3))) { - const decoded = String.fromCharCode(parseInt(p.slice(i + 1, i + 3), 16)); - out += /[A-Za-z0-9\-._~]/.test(decoded) ? decoded : "%" + p.slice(i + 1, i + 3).toUpperCase(); + // Escapes are kept, only their hex is uppercased. Like Google's parser, an + // encoded letter (%62) does not match the plain letter (b). + out += "%" + p.slice(i + 1, i + 3).toUpperCase(); i += 2; } else if (/[A-Za-z0-9\-._~!$&'()*+,;=:@\/?#[\]]/.test(ch)) { out += ch; diff --git a/test/conformance.test.js b/test/conformance.test.js new file mode 100644 index 0000000..dc6aec8 --- /dev/null +++ b/test/conformance.test.js @@ -0,0 +1,180 @@ +// robots.txt conformance, ported from Google's reference parser test suite: +// https://github.com/google/robotstxt/blob/master/robots_test.cc (Apache License 2.0). +// Cases are copied as written. Two deliberate differences, noted inline: +// - rendergap matches only paths that came through the WHATWG URL parser, which +// percent-encodes non-ASCII. Google's cases that pass a raw "ツ" in the URL test a +// path rendergap can never receive, so they are run through the URL parser instead. +// - The product-token validity helper (IsValidUserAgentToObey) is not ported: our +// crawler tokens are a fixed, known list. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { parseRobots, isAllowed } from "../src/robots.js"; + +function allowed(robotstxt, agent, url) { + const u = new URL(url); + return isAllowed(parseRobots(robotstxt), agent, u.pathname + u.search).allowed; +} +const T = (txt, agent, url) => assert.equal(allowed(txt, agent, url), true, `${agent} ${url} should be allowed`); +const F = (txt, agent, url) => assert.equal(allowed(txt, agent, url), false, `${agent} ${url} should be disallowed`); + +test("ID_LineSyntax_Line", () => { + const url = "http://foo.bar/x/y"; + F("user-agent: FooBot\ndisallow: /\n", "FooBot", url); + T("foo: FooBot\nbar: /\n", "FooBot", url); + F("user-agent FooBot\ndisallow /\n", "FooBot", url); +}); + +test("ID_LineSyntax_Groups", () => { + const r = + "allow: /foo/bar/\n\nuser-agent: FooBot\ndisallow: /\nallow: /x/\nuser-agent: BarBot\ndisallow: /\nallow: /y/\n\n\nallow: /w/\nuser-agent: BazBot\n\nuser-agent: FooBot\nallow: /z/\ndisallow: /\n"; + T(r, "FooBot", "http://foo.bar/x/b"); + T(r, "FooBot", "http://foo.bar/z/d"); + F(r, "FooBot", "http://foo.bar/y/c"); + T(r, "BarBot", "http://foo.bar/y/c"); + T(r, "BarBot", "http://foo.bar/w/a"); + F(r, "BarBot", "http://foo.bar/z/d"); + T(r, "BazBot", "http://foo.bar/z/d"); + F(r, "FooBot", "http://foo.bar/foo/bar/"); + F(r, "BarBot", "http://foo.bar/foo/bar/"); + F(r, "BazBot", "http://foo.bar/foo/bar/"); +}); + +test("ID_LineSyntax_Groups_OtherRules", () => { + const a = "User-agent: BarBot\nSitemap: https://foo.bar/sitemap\nUser-agent: *\nDisallow: /\n"; + F(a, "FooBot", "http://foo.bar/"); + F(a, "BarBot", "http://foo.bar/"); + const b = "User-agent: FooBot\nInvalid-Unknown-Line: unknown\nUser-agent: *\nDisallow: /\n"; + F(b, "FooBot", "http://foo.bar/"); + F(b, "BarBot", "http://foo.bar/"); +}); + +test("ID_REPLineNamesCaseInsensitive", () => { + for (const r of ["USER-AGENT: FooBot\nALLOW: /x/\nDISALLOW: /\n", "user-agent: FooBot\nallow: /x/\ndisallow: /\n", "uSeR-aGeNt: FooBot\nAlLoW: /x/\ndIsAlLoW: /\n"]) { + T(r, "FooBot", "http://foo.bar/x/y"); + F(r, "FooBot", "http://foo.bar/a/b"); + } +}); + +test("ID_UserAgentValueCaseInsensitive", () => { + for (const r of ["User-Agent: FOO BAR\nAllow: /x/\nDisallow: /\n", "User-Agent: foo bar\nAllow: /x/\nDisallow: /\n", "User-Agent: FoO bAr\nAllow: /x/\nDisallow: /\n"]) { + for (const agent of ["Foo", "foo"]) { + T(r, agent, "http://foo.bar/x/y"); + F(r, agent, "http://foo.bar/a/b"); + } + } +}); + +test("GoogleOnly_AcceptUserAgentUpToFirstSpace", () => { + const r = "User-Agent: *\nDisallow: /\nUser-Agent: Foo Bar\nAllow: /x/\nDisallow: /\n"; + T(r, "Foo", "http://foo.bar/x/y"); +}); + +test("ID_GlobalGroups_Secondary", () => { + const url = "http://foo.bar/x/y"; + T("", "FooBot", url); + F("user-agent: *\nallow: /\nuser-agent: FooBot\ndisallow: /\n", "FooBot", url); + T("user-agent: *\nallow: /\nuser-agent: FooBot\ndisallow: /\n", "BarBot", url); + T("user-agent: FooBot\nallow: /\nuser-agent: BarBot\ndisallow: /\nuser-agent: BazBot\ndisallow: /\n", "QuxBot", url); +}); + +test("ID_AllowDisallow_Value_CaseSensitive", () => { + F("user-agent: FooBot\ndisallow: /x/\n", "FooBot", "http://foo.bar/x/y"); + T("user-agent: FooBot\ndisallow: /X/\n", "FooBot", "http://foo.bar/x/y"); +}); + +test("ID_LongestMatch", () => { + const url = "http://foo.bar/x/page.html"; + F("user-agent: FooBot\ndisallow: /x/page.html\nallow: /x/\n", "FooBot", url); + T("user-agent: FooBot\nallow: /x/page.html\ndisallow: /x/\n", "FooBot", url); + F("user-agent: FooBot\nallow: /x/page.html\ndisallow: /x/\n", "FooBot", "http://foo.bar/x/"); + T("user-agent: FooBot\ndisallow: \nallow: \n", "FooBot", url); + T("user-agent: FooBot\ndisallow: /\nallow: /\n", "FooBot", url); + F("user-agent: FooBot\ndisallow: /x\nallow: /x/\n", "FooBot", "http://foo.bar/x"); + T("user-agent: FooBot\ndisallow: /x\nallow: /x/\n", "FooBot", "http://foo.bar/x/"); + T("user-agent: FooBot\ndisallow: /x/page.html\nallow: /x/page.html\n", "FooBot", url); + F("user-agent: FooBot\nallow: /page\ndisallow: /*.html\n", "FooBot", "http://foo.bar/page.html"); + T("user-agent: FooBot\nallow: /page\ndisallow: /*.html\n", "FooBot", "http://foo.bar/page"); + T("user-agent: FooBot\nallow: /x/page.\ndisallow: /*.html\n", "FooBot", url); + F("user-agent: FooBot\nallow: /x/page.\ndisallow: /*.html\n", "FooBot", "http://foo.bar/x/y.html"); + T("User-agent: *\nDisallow: /x/\nUser-agent: FooBot\nDisallow: /y/\n", "FooBot", "http://foo.bar/x/page"); + F("User-agent: *\nDisallow: /x/\nUser-agent: FooBot\nDisallow: /y/\n", "FooBot", "http://foo.bar/y/page"); +}); + +test("ID_Encoding", () => { + T("User-agent: FooBot\nDisallow: /\nAllow: /foo/bar?qux=taz&baz=http://foo.bar?tar&par\n", "FooBot", + "http://foo.bar/foo/bar?qux=taz&baz=http://foo.bar?tar&par"); + T("User-agent: FooBot\nDisallow: /\nAllow: /foo/bar/ツ\n", "FooBot", "http://foo.bar/foo/bar/%E3%83%84"); + // Google expects the raw "ツ" URL to be disallowed because its matcher receives the + // path unencoded. rendergap's paths always come through the URL parser, which encodes + // it, so the same URL is allowed here. See the header comment. + T("User-agent: FooBot\nDisallow: /\nAllow: /foo/bar/ツ\n", "FooBot", "http://foo.bar/foo/bar/ツ"); + T("User-agent: FooBot\nDisallow: /\nAllow: /foo/bar/%E3%83%84\n", "FooBot", "http://foo.bar/foo/bar/%E3%83%84"); + F("User-agent: FooBot\nDisallow: /\nAllow: /foo/bar/%62%61%7A\n", "FooBot", "http://foo.bar/foo/bar/baz"); + T("User-agent: FooBot\nDisallow: /\nAllow: /foo/bar/%62%61%7A\n", "FooBot", "http://foo.bar/foo/bar/%62%61%7A"); +}); + +test("ID_SpecialCharacters", () => { + const a = "User-agent: FooBot\nDisallow: /foo/bar/quz\nAllow: /foo/*/qux\n"; + F(a, "FooBot", "http://foo.bar/foo/bar/quz"); + T(a, "FooBot", "http://foo.bar/foo/quz"); + T(a, "FooBot", "http://foo.bar/foo//quz"); + T(a, "FooBot", "http://foo.bar/foo/bax/quz"); + const b = "User-agent: FooBot\nDisallow: /foo/bar$\nAllow: /foo/bar/qux\n"; + F(b, "FooBot", "http://foo.bar/foo/bar"); + T(b, "FooBot", "http://foo.bar/foo/bar/qux"); + T(b, "FooBot", "http://foo.bar/foo/bar/"); + T(b, "FooBot", "http://foo.bar/foo/bar/baz"); + const c = "User-agent: FooBot\n# Disallow: /\nDisallow: /foo/quz#qux\nAllow: /\n"; + T(c, "FooBot", "http://foo.bar/foo/bar"); + F(c, "FooBot", "http://foo.bar/foo/quz"); +}); + +test("ID_UTF8ByteOrderMarkIsSkipped", () => { + F("User-Agent: foo\nAllow: /AnyValue\nDisallow: /\n", "foo", "http://foo.bar/x"); + T("User-Agent: foo\nAllow: /AnyValue\nDisallow: /\n", "foo", "http://foo.bar/AnyValue"); +}); + +test("GoogleOnly_DocumentationChecks", () => { + const fish = "user-agent: FooBot\ndisallow: /\nallow: /fish\n"; + F(fish, "FooBot", "http://foo.bar/bar"); + for (const p of ["/fish", "/fish.html", "/fish/salmon.html", "/fishheads", "/fishheads/yummy.html", "/fish.html?id=anything"]) T(fish, "FooBot", "http://foo.bar" + p); + for (const p of ["/Fish.asp", "/catfish", "/?id=fish"]) F(fish, "FooBot", "http://foo.bar" + p); + + const fishStar = "user-agent: FooBot\ndisallow: /\nallow: /fish*\n"; + F(fishStar, "FooBot", "http://foo.bar/bar"); + for (const p of ["/fish", "/fish.html", "/fish/salmon.html", "/fishheads", "/fishheads/yummy.html", "/fish.html?id=anything"]) T(fishStar, "FooBot", "http://foo.bar" + p); + for (const p of ["/Fish.bar", "/catfish", "/?id=fish"]) F(fishStar, "FooBot", "http://foo.bar" + p); + + const fishDir = "user-agent: FooBot\ndisallow: /\nallow: /fish/\n"; + F(fishDir, "FooBot", "http://foo.bar/bar"); + for (const p of ["/fish/", "/fish/salmon", "/fish/?salmon", "/fish/salmon.html", "/fish/?id=anything"]) T(fishDir, "FooBot", "http://foo.bar" + p); + for (const p of ["/fish", "/fish.html", "/Fish/Salmon.html"]) F(fishDir, "FooBot", "http://foo.bar" + p); + + const php = "user-agent: FooBot\ndisallow: /\nallow: /*.php\n"; + F(php, "FooBot", "http://foo.bar/bar"); + for (const p of ["/filename.php", "/folder/filename.php", "/folder/filename.php?parameters", "//folder/any.php.file.html", "/filename.php/", "/index?f=filename.php/"]) T(php, "FooBot", "http://foo.bar" + p); + for (const p of ["/php/", "/index?php", "/windows.PHP"]) F(php, "FooBot", "http://foo.bar" + p); + + const phpEnd = "user-agent: FooBot\ndisallow: /\nallow: /*.php$\n"; + F(phpEnd, "FooBot", "http://foo.bar/bar"); + for (const p of ["/filename.php", "/folder/filename.php"]) T(phpEnd, "FooBot", "http://foo.bar" + p); + for (const p of ["/filename.php?parameters", "/filename.php/", "/filename.php5", "/php/", "/filename?php", "/aaaphpaaa", "//windows.PHP"]) F(phpEnd, "FooBot", "http://foo.bar" + p); + + const fishPhp = "user-agent: FooBot\ndisallow: /\nallow: /fish*.php\n"; + F(fishPhp, "FooBot", "http://foo.bar/bar"); + T(fishPhp, "FooBot", "http://foo.bar/fish.php"); + T(fishPhp, "FooBot", "http://foo.bar/fishheads/catfish.php?parameters"); + F(fishPhp, "FooBot", "http://foo.bar/Fish.PHP"); + + T("user-agent: FooBot\nallow: /p\ndisallow: /\n", "FooBot", "http://example.com/page"); + T("user-agent: FooBot\nallow: /folder\ndisallow: /folder\n", "FooBot", "http://example.com/folder/page"); + F("user-agent: FooBot\nallow: /page\ndisallow: /*.htm\n", "FooBot", "http://example.com/page.htm"); + T("user-agent: FooBot\nallow: /$\ndisallow: /\n", "FooBot", "http://example.com/"); + F("user-agent: FooBot\nallow: /$\ndisallow: /\n", "FooBot", "http://example.com/page.html"); +}); + +test("Google's documented misspellings are accepted", () => { + F("useragent: FooBot\ndisalow: /\n", "FooBot", "http://foo.bar/x"); + F("user agent: FooBot\ndissallow: /\n", "FooBot", "http://foo.bar/x"); +}); diff --git a/test/e2e.test.js b/test/e2e.test.js index 1e404ac..39189ae 100644 --- a/test/e2e.test.js +++ b/test/e2e.test.js @@ -28,10 +28,11 @@ before(async () => { res.end(fixture(page)); }); await new Promise((r) => server.listen(0, "127.0.0.1", r)); + server.unref(); origin = `http://127.0.0.1:${server.address().port}`; }); -after(() => server.close()); +after(() => server?.close()); test("crawler check reads robots.txt and server blocks", async () => { const r = await checkCrawlers(`${origin}/`, { delayMs: 0 }); diff --git a/test/hostile.test.js b/test/hostile.test.js index fa1e912..6e9f5c3 100644 --- a/test/hostile.test.js +++ b/test/hostile.test.js @@ -125,6 +125,44 @@ setTimeout(() => { for (;;) {} }, 300); document.getElementById("x").innerHTML = "

Service area

Las Cruces, Mesilla, Organ and the rest of Dona Ana County, same day.

"; `)), + // No JavaScript anywhere: the server simply sends GPTBot a thinner page. + "/uacloak": (req, res) => { + if (/GPTBot/.test(req.headers["user-agent"])) return html(res, page(`

${PARA}

`)); + html(res, page(STATIC_BODY + `

${PARA} Again.

`.repeat(6))); + }, + + // Both problems at once: thinner HTML for GPTBot, and pricing added by JavaScript. + "/both": (req, res) => { + const js = `
`; + if (/GPTBot/.test(req.headers["user-agent"])) return html(res, page(`

${PARA}

${js}`)); + html(res, page(STATIC_BODY + js)); + }, + + // A JavaScript challenge, the way Cloudflare-style walls work: known crawlers pass, + // anything else must run a script that sets a cookie and reloads. A plain fetch + // never passes; a real browser does. + "/fingerprint": (req, res) => { + const ua = req.headers["user-agent"] || ""; + if (/bot/i.test(ua) || /(^|;\s*)cf=ok/.test(req.headers.cookie || "")) return html(res, page(STATIC_BODY)); + html(res, `Just a moment...`); + }, + + // Anything claiming to be Chrome without Chrome's client hints is refused. + "/hints": (req, res) => { + const ua = req.headers["user-agent"] || ""; + if (/Chrome\//.test(ua) && !/bot/i.test(ua) && !/Chrom/.test(req.headers["sec-ch-ua"] || "")) { + res.writeHead(403, { "content-type": "text/html" }); + return res.end("Denied"); + } + html(res, page(STATIC_BODY)); + }, + + // The crawler gets more than browsers do. + "/extraforbots": (req, res) => { + if (/GPTBot/.test(req.headers["user-agent"])) return html(res, page(STATIC_BODY + `

${PARA} Bonus.

`.repeat(10))); + html(res, page(STATIC_BODY + `

${PARA} Shared.

`)); + }, + // A bakery's real "cookies" section, added by JS: this IS content. "/bakery": (req, res) => html(res, page(`

${PARA}

+

Home

We repair water heaters, leaks and drains for homes across the valley, same day.

+ +`); + }); + await new Promise((r) => home.listen(0, "127.0.0.1", r)); + home.unref(); + homeOrigin = `http://127.0.0.1:${home.address().port}`; +}); + +// Hooks run even when a name filter skips every test, before setup ever ran. +after(() => { + for (const s of [home, third]) { + s?.closeAllConnections?.(); + s?.close(); + } +}); + +function leaked() { + return seenByThird.filter((r) => JSON.stringify(r.headers).includes(SECRET) || r.headers.authorization); +} + +async function hasChrome() { + if (process.env.CHROME_PATH) return existsSync(process.env.CHROME_PATH); + try { + const { default: p } = await import("puppeteer"); + return existsSync(p.executablePath()); + } catch { + return false; + } +} + +test("raw fetch drops custom headers on a cross-origin redirect", async () => { + seenByThird.length = 0; + const r = await fetchRaw(`${homeOrigin}/redirect-out`, { userAgent: "t", headers: { "x-preview-token": SECRET, authorization: "Basic abc" } }); + assert.equal(r.status, 200); + assert.ok(seenByThird.length > 0, "redirect target was not reached"); + assert.deepEqual(leaked(), []); +}); + +test("crawler check never sends secrets off-origin", async () => { + seenByThird.length = 0; + const port = new URL(homeOrigin).port; + await checkCrawlers(`http://u:${SECRET}@127.0.0.1:${port}/redirect-out`, { + delayMs: 0, + agents: [findAgent("gptbot")], + headers: { "x-preview-token": SECRET }, + }); + assert.ok(seenByThird.length > 0); + assert.deepEqual(leaked(), []); +}); + +test("render gap never sends secrets to third-party subresources", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + seenByThird.length = 0; + const { renderGap } = await import("../src/render.js"); + const port = new URL(homeOrigin).port; + await renderGap(`http://u:${SECRET}@127.0.0.1:${port}/`, { + agent: findAgent("gptbot"), + settleMs: 300, + headers: { "x-preview-token": SECRET }, + }); + assert.ok(seenByThird.some((r) => r.url.includes("pixel") || r.url.includes("tag.js")), "subresources were not requested"); + assert.deepEqual(leaked().map((r) => r.url), []); +}); + +test("render gap never sends secrets across a redirect", async (t) => { + if (!(await hasChrome())) return t.skip("no Chrome"); + seenByThird.length = 0; + const { renderGap } = await import("../src/render.js"); + await renderGap(`${homeOrigin}/redirect-out`, { + agent: findAgent("gptbot"), + settleMs: 100, + headers: { "x-preview-token": SECRET }, + }); + assert.ok(seenByThird.some((r) => r.url.startsWith("/landing"))); + assert.deepEqual(leaked().map((r) => r.url), []); +}); diff --git a/test/units.test.js b/test/units.test.js index 05e01a5..a155318 100644 --- a/test/units.test.js +++ b/test/units.test.js @@ -48,7 +48,8 @@ test("percent-encoding is normalized on both sides", () => { assert.equal(isAllowed(g, "GPTBot", "/caf%C3%A9/menu").allowed, false); assert.equal(isAllowed(g, "GPTBot", "/café/menu").allowed, false); assert.equal(isAllowed(g, "GPTBot", "/a%2Fb").allowed, false); - assert.equal(normalizePath("/%7Euser"), "/~user"); + // Escapes keep their form (hex uppercased); an encoded letter is not the letter. + assert.equal(normalizePath("/%7euser"), "/%7Euser"); }); test("robots.txt itself is always allowed", () => {