test(seo): add a deployed sitemap contract checker

package.json

@@ -38,6 +38,7 @@
     "docs:webhooks:plan": "tsx scripts/sync-docs-webhooks.ts --dry-run",
     "docs:webhooks:sync": "tsx scripts/sync-docs-webhooks.ts",
     "docs:links:check": "tsx scripts/check-docs-menu-links.ts",
+    "sitemap:check": "tsx scripts/check-sitemap.ts",
     "husky": "pnpm run format && pnpm run test",
     "pretest": "pnpm run content:build",
     "test": "run-s test:tsc test:lint test:unit test:chat test:desktop",

scripts/check-sitemap.ts

@@ -0,0 +1,161 @@
+import { load } from 'cheerio'
+import { parseArgs } from 'node:util'
+import { pathToFileURL } from 'node:url'
+
+export function getSitemapUrls(xml: string) {
+  const $ = load(xml, { xmlMode: true })
+  return $('urlset > url > loc')
+    .toArray()
+    .map((node) => $(node).text().trim())
+}
+
+export function checkSitemapPage(
+  expectedUrl: string,
+  response: { status: number; headers: Headers; body: string },
+) {
+  const failures: Array<string> = []
+  const expected = new URL(expectedUrl)
+  if (
+    /\/(?:drafts|readme(?:\.[^/]+)?)(?:\/|$)|\.draft(?:\/|$)/i.test(
+      expected.pathname,
+    ) ||
+    /^\/(?:account|admin|login|partners-embed|sponsors-embed)(?:\/|$)/.test(
+      expected.pathname,
+    )
+  ) {
+    failures.push('disallowed sitemap path')
+  }
+  if (response.status !== 200) failures.push(`HTTP ${response.status}`)
+  const $ = load(response.body)
+  const canonical = $('head link[rel]')
+    .toArray()
+    .filter((node) =>
+      ($(node).attr('rel') ?? '')
+        .toLowerCase()
+        .split(/\s+/)
+        .includes('canonical'),
+    )
+  if (canonical.length !== 1) {
+    failures.push(`expected one canonical link, found ${canonical.length}`)
+  } else {
+    const href = $(canonical[0]).attr('href')
+    try {
+      if (!href || new URL(href, expected).href !== expected.href) {
+        failures.push('canonical does not match sitemap URL')
+      }
+    } catch {
+      failures.push('invalid canonical URL')
+    }
+  }
+  const directives = [
+    response.headers.get('x-robots-tag') ?? '',
+    ...$('meta[name]')
+      .toArray()
+      .filter((node) =>
+        ['robots', 'googlebot', 'bingbot'].includes(
+          ($(node).attr('name') ?? '').toLowerCase(),
+        ),
+      )
+      .map((node) => $(node).attr('content') ?? ''),
+  ]
+  if (
+    directives.some((value) =>
+      /\b(?:noindex|none)\b/i.test(
+        value.replace(/\bmax-image-preview\s*:\s*none\b/gi, ''),
+      ),
+    )
+  ) {
+    failures.push('page is noindex')
+  }
+  if (
+    $('h1')
+      .toArray()
+      .some((node) =>
+        /^(?:content temporarily unavailable|page not found|not found|something went wrong)$/i.test(
+          $(node).text().trim(),
+        ),
+      )
+  )
+    failures.push('error or temporary-unavailable heading')
+  return failures
+}
+
+async function main() {
+  const { values } = parseArgs({
+    options: {
+      sitemap: { type: 'string' },
+      'base-url': { type: 'string' },
+      limit: { type: 'string' },
+    },
+  })
+  if (!values.sitemap)
+    throw new Error(
+      'Pass --sitemap <URL>; optionally --base-url <preview origin> and --limit <count>.',
+    )
+  const sitemap = new URL(values.sitemap)
+  const base =
+    values['base-url'] !== undefined ? new URL(values['base-url']) : undefined
+  const limit = values.limit === undefined ? Infinity : Number(values.limit)
+  if (
+    values.limit !== undefined &&
+    (!Number.isSafeInteger(limit) || limit < 1)
+  ) {
+    throw new Error('--limit must be a positive integer')
+  }
+  const response = await fetch(sitemap, {
+    redirect: 'manual',
+    signal: AbortSignal.timeout(30_000),
+  })
+  if (response.status !== 200)
+    throw new Error(`Sitemap returned HTTP ${response.status}`)
+  const urls = getSitemapUrls(await response.text())
+  if (!urls.length) throw new Error('Sitemap contains no URL entries')
+  const selected = urls.slice(0, limit)
+  let failed = 0
+  for (let offset = 0; offset < selected.length; offset += 4) {
+    const results = await Promise.all(
+      selected.slice(offset, offset + 4).map(async (url) => {
+        try {
+          const canonical = new URL(url)
+          const target = base
+            ? new URL(canonical.pathname + canonical.search, base.origin)
+            : canonical
+          const page = await fetch(target, {
+            redirect: 'manual',
+            signal: AbortSignal.timeout(20_000),
+          })
+          return {
+            url,
+            failures: checkSitemapPage(url, {
+              status: page.status,
+              headers: page.headers,
+              body: await page.text(),
+            }),
+          }
+        } catch (error) {
+          return {
+            url,
+            failures: [error instanceof Error ? error.message : String(error)],
+          }
+        }
+      }),
+    )
+    for (const result of results) {
+      if (result.failures.length) {
+        failed++
+        console.error(`${result.url}: ${result.failures.join('; ')}`)
+      }
+    }
+  }
+  console.log(
+    `Checked ${selected.length}/${urls.length} sitemap URLs: ${failed} failed.${selected.length < urls.length ? ' Partial check only.' : ''}`,
+  )
+  if (failed) process.exitCode = 1
+}
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  await main()
+}

sitemap-checking.md

@@ -0,0 +1,19 @@
+# Check the sitemap contract
+
+Run the full read-only check against production:
+
+```sh
+pnpm sitemap:check --sitemap https://tanstack.com/sitemap.xml
+```
+
+For a commit preview, use its sitemap URL and pass its origin as `--base-url`. Entries retain their production canonical URLs, while HTTP requests go to the preview:
+
+```sh
+pnpm sitemap:check --sitemap https://YOUR-PREVIEW.workers.dev/sitemap.xml --base-url https://YOUR-PREVIEW.workers.dev
+```
+
+The checker makes four concurrent requests, does not follow redirects, and exits nonzero for failed pages. It checks status 200, exactly one matching canonical link, robots meta/header directives, error headings, and disallowed repository-artifact or private paths. It does not run page JavaScript or prove content quality.
+
+Use `--limit 20` for a quick check of the first 20 entries. The output labels this as partial; passing a sample does not verify the whole sitemap. Without a limit every entry is checked, which can take time on a large sitemap. Neither command changes deployed content or clears caches.
+
+Contract regression tests run in the normal unit suite. The deployed check is available for release verification; it is not yet a mandatory deployment gate while the remaining failures in issue #1153 are being fixed.

src/utils/sitemap-docs.ts

@@ -17,7 +17,7 @@ export function isHighValueDocsSlug(slug: string) {
   return (
     segments.length > 0 &&
     !segments.some((segment) =>
-      /^(?:drafts|readme)$|\.draft$/i.test(segment),
+      /^(?:drafts|readme(?:\.[^/]+)?)$|\.draft$/i.test(segment),
     ) &&
     !segments.some((segment) => LOW_VALUE_DOCS_SITEMAP_SEGMENTS.has(segment)) &&
     !LOW_VALUE_DOCS_SITEMAP_SLUGS.has(slug)

tests/sitemap-contract.test.ts

@@ -0,0 +1,133 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { spawnSync } from 'node:child_process'
+import { fileURLToPath } from 'node:url'
+import { checkSitemapPage, getSitemapUrls } from '../scripts/check-sitemap'
+
+const url = 'https://tanstack.com/charts/catalog'
+const canonical = `<link rel="canonical" href="${url}">`
+function check(body = canonical, status = 200, headers = new Headers()) {
+  return checkSitemapPage(url, { body, status, headers })
+}
+
+test('sitemap parser decodes XML entities and ignores unrelated links', () => {
+  assert.deepEqual(
+    getSitemapUrls(
+      '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>https://tanstack.com/partners?a=1&amp;b=2</loc></url></urlset>',
+    ),
+    ['https://tanstack.com/partners?a=1&b=2'],
+  )
+  assert.deepEqual(getSitemapUrls('<html><a href="/">Home</a></html>'), [])
+})
+
+test('contract accepts a healthy self-canonical page and rejects status/canonical failures', () => {
+  assert.deepEqual(check(), [])
+  assert.ok(
+    check(`<html><head></head><body>${canonical}</body></html>`).includes(
+      'expected one canonical link, found 0',
+    ),
+  )
+  assert.ok(check(canonical, 307).includes('HTTP 307'))
+  assert.ok(check(canonical, 404).includes('HTTP 404'))
+  assert.ok(check('').includes('expected one canonical link, found 0'))
+  assert.ok(
+    check(canonical + canonical).includes(
+      'expected one canonical link, found 2',
+    ),
+  )
+  assert.ok(
+    check('<link rel="canonical" href="/elsewhere">').includes(
+      'canonical does not match sitemap URL',
+    ),
+  )
+  assert.ok(
+    check('<link rel="canonical" href="http://[">').includes(
+      'invalid canonical URL',
+    ),
+  )
+})
+
+test('contract rejects robots directives, placeholders and forbidden paths', () => {
+  assert.ok(
+    check(
+      canonical + '<meta name="ROBOTS" content="noindex, follow">',
+    ).includes('page is noindex'),
+  )
+  assert.ok(
+    check(
+      canonical,
+      200,
+      new Headers({ 'X-Robots-Tag': 'googlebot: none' }),
+    ).includes('page is noindex'),
+  )
+  assert.ok(
+    check(canonical + '<h1>Content temporarily unavailable</h1>').includes(
+      'error or temporary-unavailable heading',
+    ),
+  )
+  for (const path of [
+    '/router/latest/docs/how-to/drafts/test',
+    '/db/latest/docs/README',
+    '/db/latest/docs/README.md',
+    '/router/latest/docs/test.draft',
+    '/admin/users',
+  ]) {
+    const expected = `https://tanstack.com${path}`
+    assert.ok(
+      checkSitemapPage(expected, {
+        body: `<link rel="canonical" href="${expected}">`,
+        status: 200,
+        headers: new Headers(),
+      }).includes('disallowed sitemap path'),
+    )
+  }
+})
+
+test('image-preview restrictions do not prevent page indexing', () => {
+  for (const directive of [
+    'max-image-preview:none',
+    'index, MAX-IMAGE-PREVIEW: none',
+    'googlebot: max-image-preview: none',
+  ]) {
+    assert.deepEqual(
+      check(canonical + `<meta name="robots" content="${directive}">`),
+      [],
+    )
+    assert.deepEqual(
+      check(canonical, 200, new Headers({ 'X-Robots-Tag': directive })),
+      [],
+    )
+    assert.ok(
+      check(
+        canonical,
+        200,
+        new Headers({ 'X-Robots-Tag': `${directive}, noindex` }),
+      ).includes('page is noindex'),
+    )
+    assert.ok(
+      check(
+        canonical,
+        200,
+        new Headers({ 'X-Robots-Tag': `${directive}, none` }),
+      ).includes('page is noindex'),
+    )
+  }
+})
+
+test('CLI rejects an explicitly empty preview origin before fetching', () => {
+  const result = spawnSync(
+    process.execPath,
+    [
+      '--import',
+      'tsx',
+      fileURLToPath(new URL('../scripts/check-sitemap.ts', import.meta.url)),
+      '--sitemap',
+      'http://127.0.0.1:1/sitemap.xml',
+      '--base-url',
+      '',
+    ],
+    { encoding: 'utf8' },
+  )
+  assert.notEqual(result.status, 0)
+  assert.match(result.stderr, /ERR_INVALID_URL/)
+})

tests/sitemap-docs.test.ts

@@ -7,6 +7,8 @@ test('sitemap excludes repository artifacts while retaining public guides', () =
     'how-to/drafts/README',
     'how-to/drafts/search-params-in-forms.draft',
     'how-to/README',
+    'how-to/README.md',
+    'README.mdx',
     'superpowers/specs/2026-07-31-tanstack-devtools-branding-design',
     'superpowers/plans/internal-implementation',
     'contributing',