From f6bff2b75b093435e409d6f759b1b149e74a2e30 Mon Sep 17 00:00:00 2001 From: Harlan GitHub Agent Date: Thu, 12 Nov 2026 12:32:26 +1100 Subject: [PATCH] chore(seo): remove the experiment E retired sitemap The experiment ended on 2026-11-11 as planned, and its handler already serves an empty list. Search Console keeps its own record of the submitted child. Co-Authored-By: Claude Opus 5.5 --- .../server/api/__sitemap__/retired.ts | 41 ------------ .../registry/server/utils/retired-sitemap.ts | 67 ------------------- nuxt.config.ts | 8 --- scripts/check-retired-sitemap.ts | 48 ------------- test/unit/retired-sitemap.test.ts | 33 --------- 5 files changed, 197 deletions(-) delete mode 100644 layers/registry/server/api/__sitemap__/retired.ts delete mode 100644 layers/registry/server/utils/retired-sitemap.ts delete mode 100644 scripts/check-retired-sitemap.ts delete mode 100644 test/unit/retired-sitemap.test.ts diff --git a/layers/registry/server/api/__sitemap__/retired.ts b/layers/registry/server/api/__sitemap__/retired.ts deleted file mode 100644 index cd8de1ee..00000000 --- a/layers/registry/server/api/__sitemap__/retired.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { SitemapUrlInput } from '#sitemap/types' -import { GONE_SKILLS_SQL } from '~~/server/utils/source-gone-skills' -import { getDB } from '#server/utils/db' -import retiredUrls from '../../data/retired-urls.json' -import { buildRetiredSitemapEntries, isRetiredSitemapActive } from '../../utils/retired-sitemap' -import { MARKETING_REDIRECTS } from '../../utils/tag-quality' - -// Temporary sitemap of retired URLs. Experiment E, remove 2026-11-11. The -// reasons, the status guarantee and the removal steps are in -// `../../utils/retired-sitemap.ts`. -// h3 `defineEventHandler`: the helpers below read the Nitro event, and -// `defineSitemapEventHandler` types its argument as the narrower `RequestEvent`. -export default defineEventHandler(async (event): Promise => { - // Past the removal date, skip the D1 queries too. - const now = new Date() - if (!isRetiredSitemapActive(now)) - return [] - - const db = getDB(event) - const [gone, collections] = await Promise.all([ - db.prepare(GONE_SKILLS_SQL).all<{ owner: string, repo: string, name: string }>(), - db.prepare(` - SELECT u.login, c.slug - FROM collections_v2 c - JOIN users u ON u.id = c.author_user_id - WHERE c.deleted_at IS NOT NULL - AND NOT EXISTS ( - SELECT 1 FROM collections_v2 live - WHERE live.author_user_id = c.author_user_id AND live.slug = c.slug AND live.deleted_at IS NULL - ) - `).all<{ login: string, slug: string }>(), - ]) - - return buildRetiredSitemapEntries({ - now, - goneSkillPaths: (gone.results ?? []).map(row => `/gh/${row.owner}/${row.repo}/${row.name}`), - deletedCollectionPaths: (collections.results ?? []).map(row => `/@${row.login}/${row.slug}`), - // Tag pages that hand off to a category: a static 301 each. - probedPaths: [...retiredUrls.paths, ...Object.keys(MARKETING_REDIRECTS).map(slug => `/skills/tag/${slug}`)], - }) -}) diff --git a/layers/registry/server/utils/retired-sitemap.ts b/layers/registry/server/utils/retired-sitemap.ts deleted file mode 100644 index 494bb931..00000000 --- a/layers/registry/server/utils/retired-sitemap.ts +++ /dev/null @@ -1,67 +0,0 @@ -/** - * SEO EXPERIMENT E, started 2026-09-30. REMOVE ON 2026-11-11. - * - * A temporary `retired` sitemap. It lists URLs that now answer 301, 404 or 410, - * each with one stable `lastmod`, so Google recrawls them and drops them sooner. - * Google holds about 52k old URLs as "Crawled, currently not indexed", and it - * crawls about 29 HTML pages a day. Practitioner advice, not documented - * policy: a temporary sitemap of changed URLs speeds up the purge. - * Scale rule: "Crawled, currently not indexed" falls by 10k in 4 weeks. - * - * Every URL here must really answer 301, 404 or 410. A URL that answers 200 - * would advertise a live page as retired. The sources are Skills the gone - * middleware answers 410 for, collections with `deleted_at` set, and paths - * probed on the generation date (`data/retired-urls.json`). - * `scripts/check-retired-sitemap.ts` re-probes the served sitemap. - * - * Do not submit it to Search Console until the owner decides to. - * - * REMOVAL. On 2026-11-11 delete `__sitemap__/retired.ts`, this file, - * `data/retired-urls.json`, and the `retired` entry in `nuxt.config.ts`. The - * handler already returns an empty list after that date, so a late removal - * does no harm. - */ - -export const RETIRED_SITEMAP_REMOVAL_DATE = '2026-11-11' - -/** - * One fixed `lastmod` for every URL: the experiment start. Google ignores a - * `lastmod` that changes on every regeneration without a real change, so a - * `new Date()` here would teach it to ignore the whole sitemap. - */ -export const RETIRED_SITEMAP_LASTMOD = '2026-09-30' - -export interface RetiredSitemapEntry { - loc: string - lastmod: string -} - -export interface RetiredSitemapInput { - now: Date - /** `/gh///` paths of Skills whose SKILL.md is gone: 410. */ - goneSkillPaths: readonly string[] - /** `/@/` paths of deleted collections: 301 or 404. */ - deletedCollectionPaths: readonly string[] - /** Paths probed on the generation date, each 301, 404 or 410. */ - probedPaths: readonly string[] -} - -export function isRetiredSitemapActive(now: Date): boolean { - return now.toISOString().slice(0, 10) < RETIRED_SITEMAP_REMOVAL_DATE -} - -/** - * The sitemap rows: deduplicated, sorted, all with the fixed experiment `lastmod`. - * After the removal date it returns nothing. - */ -export function buildRetiredSitemapEntries(input: RetiredSitemapInput): RetiredSitemapEntry[] { - if (!isRetiredSitemapActive(input.now)) - return [] - const lastmod = RETIRED_SITEMAP_LASTMOD - const paths = new Set() - for (const path of [...input.goneSkillPaths, ...input.deletedCollectionPaths, ...input.probedPaths]) { - if (path.startsWith('/')) - paths.add(path) - } - return [...paths].sort().map(loc => ({ loc, lastmod })) -} diff --git a/nuxt.config.ts b/nuxt.config.ts index 22cdda55..4ac46f1c 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -919,14 +919,6 @@ export default defineNuxtConfig({ includeAppSources: false, chunks: 10000, }, - // Experiment E, remove 2026-11-11: retired URLs (301, 404, 410) with a - // fresh lastmod, so Google recrawls and drops them sooner. Not submitted. - // Steps: `layers/registry/server/utils/retired-sitemap.ts`. - retired: { - sources: ['/api/__sitemap__/retired'], - includeAppSources: false, - chunks: 10000, - }, // `/skills/demos` and one page per demo, which the pages sitemap's // `/skills/**` exclude drops. Listed while page-admissions admits // `/skills/demos`, the entry every demo page reads. diff --git a/scripts/check-retired-sitemap.ts b/scripts/check-retired-sitemap.ts deleted file mode 100644 index 385695c4..00000000 --- a/scripts/check-retired-sitemap.ts +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Probe every URL in the served `retired` sitemap and fail if the sitemap does - * not load, is empty, or lists a URL that answers 200. Experiment E, remove - * 2026-11-11. - * - * Usage: pnpm tsx scripts/check-retired-sitemap.ts [origin] - * The origin defaults to https://skilld.dev. - * - * Exit codes: 0 every URL answers 301, 404 or 410; 1 the sitemap failed to - * load or is empty, or a URL answered something else. - */ -const origin = process.argv[2] ?? 'https://skilld.dev' -const CONCURRENCY = 10 -const ALLOWED = new Set([301, 404, 410]) -const sitemapUrl = `${origin}/__sitemap__/retired.xml` - -function fail(message: string): never { - console.error(message) - process.exit(1) -} - -const sitemap = await fetch(sitemapUrl).catch((error: unknown) => fail(`Could not fetch ${sitemapUrl}: ${String(error)}`)) -if (!sitemap.ok) - fail(`${sitemapUrl} answered ${sitemap.status}`) -const xml = await sitemap.text() -const locs = [...xml.matchAll(/([^<]+)<\/loc>/g)].map((match) => { - const url = new URL(match[1]!) - return url.pathname + url.search -}) -console.log(`${locs.length} URLs in the retired sitemap`) -if (!locs.length) - fail(`${sitemapUrl} lists no URLs`) - -const bad: string[] = [] -let next = 0 -async function worker() { - while (next < locs.length) { - const path = locs[next++]! - const response = await fetch(`${origin}${path}`, { method: 'HEAD', redirect: 'manual' }) - if (!ALLOWED.has(response.status)) - bad.push(`${response.status} ${path}`) - } -} -await Promise.all(Array.from({ length: CONCURRENCY }, worker)) - -if (bad.length) - fail(`${bad.length} URLs do not answer 301, 404 or 410:\n${bad.slice(0, 50).join('\n')}`) -console.log('every URL answers 301, 404 or 410') diff --git a/test/unit/retired-sitemap.test.ts b/test/unit/retired-sitemap.test.ts deleted file mode 100644 index b469e4f9..00000000 --- a/test/unit/retired-sitemap.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { buildRetiredSitemapEntries, isRetiredSitemapActive } from '../../layers/registry/server/utils/retired-sitemap' - -const base = { - goneSkillPaths: ['/gh/o/r/gone'], - deletedCollectionPaths: ['/@harlan-zw/old'], - probedPaths: ['/orgs/acme', '/gh/o/r/gone'], -} - -describe('buildRetiredSitemapEntries', () => { - it('merges every source once, sorted, each with the same stable lastmod', () => { - const now = new Date('2026-10-02T08:30:00Z') - const entries = buildRetiredSitemapEntries({ now, ...base }) - expect(entries.map(e => e.loc)).toEqual(['/@harlan-zw/old', '/gh/o/r/gone', '/orgs/acme']) - expect(new Set(entries.map(e => e.lastmod))).toEqual(new Set(['2026-09-30'])) - }) - - it('keeps lastmod fixed however late it regenerates', () => { - const early = buildRetiredSitemapEntries({ now: new Date('2026-10-01T00:00:00Z'), ...base }) - const late = buildRetiredSitemapEntries({ now: new Date('2026-11-01T12:00:00Z'), ...base }) - expect(late).toEqual(early) - }) - - it('drops anything that is not a site path', () => { - const entries = buildRetiredSitemapEntries({ now: new Date('2026-10-02T00:00:00Z'), ...base, probedPaths: ['https://evil.example/x', 'relative'] }) - expect(entries.map(e => e.loc)).toEqual(['/@harlan-zw/old', '/gh/o/r/gone']) - }) - - it('goes empty on the removal date', () => { - expect(isRetiredSitemapActive(new Date('2026-11-10T23:59:59Z'))).toBe(true) - expect(buildRetiredSitemapEntries({ now: new Date('2026-11-11T00:00:00Z'), ...base })).toEqual([]) - }) -})