diff --git a/package.json b/package.json index 543053148..4a232634b 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "docs:webhooks:plan": "tsx scripts/sync-docs-webhooks.ts --dry-run", "docs:webhooks:sync": "tsx scripts/sync-docs-webhooks.ts", "docs:links:check": "tsx scripts/check-docs-menu-links.ts", + "sitemap:check": "tsx scripts/check-sitemap.ts", "husky": "pnpm run format && pnpm run test", "pretest": "pnpm run content:build", "test": "run-s test:tsc test:lint test:unit test:chat test:desktop", diff --git a/scripts/check-sitemap.ts b/scripts/check-sitemap.ts new file mode 100644 index 000000000..1f222b38c --- /dev/null +++ b/scripts/check-sitemap.ts @@ -0,0 +1,161 @@ +import { load } from 'cheerio' +import { parseArgs } from 'node:util' +import { pathToFileURL } from 'node:url' + +export function getSitemapUrls(xml: string) { + const $ = load(xml, { xmlMode: true }) + return $('urlset > url > loc') + .toArray() + .map((node) => $(node).text().trim()) +} + +export function checkSitemapPage( + expectedUrl: string, + response: { status: number; headers: Headers; body: string }, +) { + const failures: Array = [] + const expected = new URL(expectedUrl) + if ( + /\/(?:drafts|readme(?:\.[^/]+)?)(?:\/|$)|\.draft(?:\/|$)/i.test( + expected.pathname, + ) || + /^\/(?:account|admin|login|partners-embed|sponsors-embed)(?:\/|$)/.test( + expected.pathname, + ) + ) { + failures.push('disallowed sitemap path') + } + if (response.status !== 200) failures.push(`HTTP ${response.status}`) + const $ = load(response.body) + const canonical = $('head link[rel]') + .toArray() + .filter((node) => + ($(node).attr('rel') ?? '') + .toLowerCase() + .split(/\s+/) + .includes('canonical'), + ) + if (canonical.length !== 1) { + failures.push(`expected one canonical link, found ${canonical.length}`) + } else { + const href = $(canonical[0]).attr('href') + try { + if (!href || new URL(href, expected).href !== expected.href) { + failures.push('canonical does not match sitemap URL') + } + } catch { + failures.push('invalid canonical URL') + } + } + const directives = [ + response.headers.get('x-robots-tag') ?? '', + ...$('meta[name]') + .toArray() + .filter((node) => + ['robots', 'googlebot', 'bingbot'].includes( + ($(node).attr('name') ?? '').toLowerCase(), + ), + ) + .map((node) => $(node).attr('content') ?? ''), + ] + if ( + directives.some((value) => + /\b(?:noindex|none)\b/i.test( + value.replace(/\bmax-image-preview\s*:\s*none\b/gi, ''), + ), + ) + ) { + failures.push('page is noindex') + } + if ( + $('h1') + .toArray() + .some((node) => + /^(?:content temporarily unavailable|page not found|not found|something went wrong)$/i.test( + $(node).text().trim(), + ), + ) + ) + failures.push('error or temporary-unavailable heading') + return failures +} + +async function main() { + const { values } = parseArgs({ + options: { + sitemap: { type: 'string' }, + 'base-url': { type: 'string' }, + limit: { type: 'string' }, + }, + }) + if (!values.sitemap) + throw new Error( + 'Pass --sitemap ; optionally --base-url and --limit .', + ) + const sitemap = new URL(values.sitemap) + const base = + values['base-url'] !== undefined ? new URL(values['base-url']) : undefined + const limit = values.limit === undefined ? Infinity : Number(values.limit) + if ( + values.limit !== undefined && + (!Number.isSafeInteger(limit) || limit < 1) + ) { + throw new Error('--limit must be a positive integer') + } + const response = await fetch(sitemap, { + redirect: 'manual', + signal: AbortSignal.timeout(30_000), + }) + if (response.status !== 200) + throw new Error(`Sitemap returned HTTP ${response.status}`) + const urls = getSitemapUrls(await response.text()) + if (!urls.length) throw new Error('Sitemap contains no URL entries') + const selected = urls.slice(0, limit) + let failed = 0 + for (let offset = 0; offset < selected.length; offset += 4) { + const results = await Promise.all( + selected.slice(offset, offset + 4).map(async (url) => { + try { + const canonical = new URL(url) + const target = base + ? new URL(canonical.pathname + canonical.search, base.origin) + : canonical + const page = await fetch(target, { + redirect: 'manual', + signal: AbortSignal.timeout(20_000), + }) + return { + url, + failures: checkSitemapPage(url, { + status: page.status, + headers: page.headers, + body: await page.text(), + }), + } + } catch (error) { + return { + url, + failures: [error instanceof Error ? error.message : String(error)], + } + } + }), + ) + for (const result of results) { + if (result.failures.length) { + failed++ + console.error(`${result.url}: ${result.failures.join('; ')}`) + } + } + } + console.log( + `Checked ${selected.length}/${urls.length} sitemap URLs: ${failed} failed.${selected.length < urls.length ? ' Partial check only.' : ''}`, + ) + if (failed) process.exitCode = 1 +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + await main() +} diff --git a/sitemap-checking.md b/sitemap-checking.md new file mode 100644 index 000000000..73d661a4d --- /dev/null +++ b/sitemap-checking.md @@ -0,0 +1,19 @@ +# Check the sitemap contract + +Run the full read-only check against production: + +```sh +pnpm sitemap:check --sitemap https://tanstack.com/sitemap.xml +``` + +For a commit preview, use its sitemap URL and pass its origin as `--base-url`. Entries retain their production canonical URLs, while HTTP requests go to the preview: + +```sh +pnpm sitemap:check --sitemap https://YOUR-PREVIEW.workers.dev/sitemap.xml --base-url https://YOUR-PREVIEW.workers.dev +``` + +The checker makes four concurrent requests, does not follow redirects, and exits nonzero for failed pages. It checks status 200, exactly one matching canonical link, robots meta/header directives, error headings, and disallowed repository-artifact or private paths. It does not run page JavaScript or prove content quality. + +Use `--limit 20` for a quick check of the first 20 entries. The output labels this as partial; passing a sample does not verify the whole sitemap. Without a limit every entry is checked, which can take time on a large sitemap. Neither command changes deployed content or clears caches. + +Contract regression tests run in the normal unit suite. The deployed check is available for release verification; it is not yet a mandatory deployment gate while the remaining failures in issue #1153 are being fixed. diff --git a/src/utils/sitemap-docs.ts b/src/utils/sitemap-docs.ts index 3b9231bda..04397cd57 100644 --- a/src/utils/sitemap-docs.ts +++ b/src/utils/sitemap-docs.ts @@ -17,7 +17,7 @@ export function isHighValueDocsSlug(slug: string) { return ( segments.length > 0 && !segments.some((segment) => - /^(?:drafts|readme)$|\.draft$/i.test(segment), + /^(?:drafts|readme(?:\.[^/]+)?)$|\.draft$/i.test(segment), ) && !segments.some((segment) => LOW_VALUE_DOCS_SITEMAP_SEGMENTS.has(segment)) && !LOW_VALUE_DOCS_SITEMAP_SLUGS.has(slug) diff --git a/tests/sitemap-contract.test.ts b/tests/sitemap-contract.test.ts new file mode 100644 index 000000000..c95d5f2bb --- /dev/null +++ b/tests/sitemap-contract.test.ts @@ -0,0 +1,133 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' +import { checkSitemapPage, getSitemapUrls } from '../scripts/check-sitemap' + +const url = 'https://tanstack.com/charts/catalog' +const canonical = `` +function check(body = canonical, status = 200, headers = new Headers()) { + return checkSitemapPage(url, { body, status, headers }) +} + +test('sitemap parser decodes XML entities and ignores unrelated links', () => { + assert.deepEqual( + getSitemapUrls( + 'https://tanstack.com/partners?a=1&b=2', + ), + ['https://tanstack.com/partners?a=1&b=2'], + ) + assert.deepEqual(getSitemapUrls('Home'), []) +}) + +test('contract accepts a healthy self-canonical page and rejects status/canonical failures', () => { + assert.deepEqual(check(), []) + assert.ok( + check(`${canonical}`).includes( + 'expected one canonical link, found 0', + ), + ) + assert.ok(check(canonical, 307).includes('HTTP 307')) + assert.ok(check(canonical, 404).includes('HTTP 404')) + assert.ok(check('').includes('expected one canonical link, found 0')) + assert.ok( + check(canonical + canonical).includes( + 'expected one canonical link, found 2', + ), + ) + assert.ok( + check('').includes( + 'canonical does not match sitemap URL', + ), + ) + assert.ok( + check('').includes( + 'invalid canonical URL', + ), + ) +}) + +test('contract rejects robots directives, placeholders and forbidden paths', () => { + assert.ok( + check( + canonical + '', + ).includes('page is noindex'), + ) + assert.ok( + check( + canonical, + 200, + new Headers({ 'X-Robots-Tag': 'googlebot: none' }), + ).includes('page is noindex'), + ) + assert.ok( + check(canonical + '

Content temporarily unavailable

').includes( + 'error or temporary-unavailable heading', + ), + ) + for (const path of [ + '/router/latest/docs/how-to/drafts/test', + '/db/latest/docs/README', + '/db/latest/docs/README.md', + '/router/latest/docs/test.draft', + '/admin/users', + ]) { + const expected = `https://tanstack.com${path}` + assert.ok( + checkSitemapPage(expected, { + body: ``, + status: 200, + headers: new Headers(), + }).includes('disallowed sitemap path'), + ) + } +}) + +test('image-preview restrictions do not prevent page indexing', () => { + for (const directive of [ + 'max-image-preview:none', + 'index, MAX-IMAGE-PREVIEW: none', + 'googlebot: max-image-preview: none', + ]) { + assert.deepEqual( + check(canonical + ``), + [], + ) + assert.deepEqual( + check(canonical, 200, new Headers({ 'X-Robots-Tag': directive })), + [], + ) + assert.ok( + check( + canonical, + 200, + new Headers({ 'X-Robots-Tag': `${directive}, noindex` }), + ).includes('page is noindex'), + ) + assert.ok( + check( + canonical, + 200, + new Headers({ 'X-Robots-Tag': `${directive}, none` }), + ).includes('page is noindex'), + ) + } +}) + +test('CLI rejects an explicitly empty preview origin before fetching', () => { + const result = spawnSync( + process.execPath, + [ + '--import', + 'tsx', + fileURLToPath(new URL('../scripts/check-sitemap.ts', import.meta.url)), + '--sitemap', + 'http://127.0.0.1:1/sitemap.xml', + '--base-url', + '', + ], + { encoding: 'utf8' }, + ) + assert.notEqual(result.status, 0) + assert.match(result.stderr, /ERR_INVALID_URL/) +}) diff --git a/tests/sitemap-docs.test.ts b/tests/sitemap-docs.test.ts index e4334487e..a70908f9d 100644 --- a/tests/sitemap-docs.test.ts +++ b/tests/sitemap-docs.test.ts @@ -7,6 +7,8 @@ test('sitemap excludes repository artifacts while retaining public guides', () = 'how-to/drafts/README', 'how-to/drafts/search-params-in-forms.draft', 'how-to/README', + 'how-to/README.md', + 'README.mdx', 'superpowers/specs/2026-07-31-tanstack-devtools-branding-design', 'superpowers/plans/internal-implementation', 'contributing',