From 40fc2c1279b83264ec7e666e68fe20aeaded52e6 Mon Sep 17 00:00:00 2001 From: m1ngsama Date: Sat, 26 Sep 2026 22:31:01 +0800 Subject: [PATCH 1/2] feat: try an optional mirror before GitHub for the tree and files --- README.md | 8 ++ src/__tests__/client.test.ts | 143 +++++++++++++++++++++++++++++++++++ src/client.ts | 70 ++++++++++++++--- src/types.ts | 1 + 4 files changed, 211 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 610f678..55af5c5 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ const matches = await docs.search('repair', { pathPrefix: 'repair' }); | `cacheTtlMs.dir` | `300000` | Directory and tree cache TTL | | `cacheTtlMs.file` | `600000` | File cache TTL | | `store` | none | Persistent cache, see below | +| `mirror` | none | Mirror base URL, see below | ### `docs.listDir(path?)` @@ -73,6 +74,13 @@ tree under `tree` and file content under `blob-`, where `` is the Git serves stored content only when its hash matches the blob id in the last known tree, so a changed file is always refetched. Store errors and corrupt entries are ignored; eviction is up to the store. +### `mirror` + +A base URL tried before GitHub by `listAll` and `getFile`, such as +`https://docs.nbtca.space/docs-api`. It serves `index.json` in the shape of GitHub's recursive tree +response and each file at `raw/`. Any mirror failure, invalid or truncated index, or wait past +5 seconds falls back to GitHub. The GitHub token is never sent to the mirror. + ### `docs.search(query, options?)` Searches paths, titles, summaries, Markdown text, and semantic component attributes. Results are diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index e2a1801..d1ffafa 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -1047,3 +1047,146 @@ describe('search', () => { expect(fetchMock).not.toHaveBeenCalled(); }); }); + +describe('mirror', () => { + const mirror = 'https://docs.example.org/docs-api/'; + const guideSha = 'b5aaad7d6dda27ea24335cdd4722c8129113f4cd'; + const mirrorTree = { + truncated: false, + tree: [ + { path: 'repair/guide.md', type: 'blob', sha: guideSha }, + { path: 'README.md', type: 'blob', sha: guideSha }, + ], + }; + + function routes(handlers: { + mirror: (url: string) => unknown; + github?: (url: string) => unknown; + }) { + const fetchMock = vi.fn().mockImplementation(async (url: string) => { + if (url.startsWith('https://docs.example.org/')) return handlers.mirror(url); + if (!handlers.github) throw new TypeError('GitHub is unreachable'); + return handlers.github(url); + }); + vi.stubGlobal('fetch', fetchMock); + return fetchMock; + } + + function memoryStore() { + const map = new Map(); + return { + map, + read: (key: string) => map.get(key), + write: (key: string, value: string) => { + map.set(key, value); + }, + }; + } + + it('lists and reads from the mirror first without sending the GitHub token', async () => { + const fetchMock = routes({ + mirror: (url) => + url.endsWith('/index.json') + ? { ok: true, json: async () => mirrorTree } + : { ok: true, text: async () => '# Guide' }, + }); + const client = createDocsClient({ mirror, token: 'secret' }); + + await expect(client.listAll()).resolves.toEqual([ + { name: 'guide.md', path: 'repair/guide.md', type: 'file', sha: guideSha }, + ]); + await expect(client.getFile('repair/guide.md')).resolves.toBe('# Guide'); + expect(fetchMock.mock.calls).toEqual([ + ['https://docs.example.org/docs-api/index.json', expect.objectContaining({ headers: {} })], + [ + 'https://docs.example.org/docs-api/raw/repair/guide.md', + expect.objectContaining({ headers: {} }), + ], + ]); + }); + + it('falls back to GitHub when the mirror fails', async () => { + const fetchMock = routes({ + mirror: () => ({ ok: false, status: 503 }), + github: (url) => + url.includes('/git/trees/') + ? { ok: true, json: async () => mockTree } + : { ok: true, text: async () => '# From GitHub' }, + }); + const client = createDocsClient({ mirror }); + + await expect(client.listAll()).resolves.toHaveLength(3); + await expect(client.getFile('intro.md')).resolves.toBe('# From GitHub'); + expect(fetchMock).toHaveBeenCalledTimes(4); + }); + + it.each([ + ['a malformed index', { message: 'not a tree' }], + ['a truncated index', { ...mirrorTree, truncated: true }], + ])('falls back to GitHub on %s', async (_label, index) => { + routes({ + mirror: () => ({ ok: true, json: async () => index }), + github: () => ({ ok: true, json: async () => mockTree }), + }); + await expect(createDocsClient({ mirror }).listAll()).resolves.toHaveLength(3); + }); + + it('keeps the GitHub error when both sources fail', async () => { + routes({ + mirror: () => { + throw new TypeError('fetch failed'); + }, + github: () => ({ ok: false, status: 404 }), + }); + await expect(createDocsClient({ mirror }).getFile('missing.md')).rejects.toMatchObject({ + name: 'DocsFetchError', + status: 404, + }); + }); + + it('stores mirror content under its verified blob sha', async () => { + const fetchMock = routes({ + mirror: (url) => + url.endsWith('/index.json') + ? { ok: true, json: async () => mirrorTree } + : { ok: true, text: async () => '# Guide' }, + }); + const store = memoryStore(); + await createDocsClient({ mirror, store }).listAll(); + await createDocsClient({ mirror, store }).getFile('repair/guide.md'); + expect(store.map.get(`blob-${guideSha}`)).toBe('# Guide'); + + fetchMock.mockClear(); + await expect(createDocsClient({ mirror, store }).getFile('repair/guide.md')).resolves.toBe( + '# Guide', + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('gives up on a slow mirror after five seconds', async () => { + vi.useFakeTimers(); + try { + const fetchMock = vi.fn().mockImplementation((url: string, init: RequestInit) => { + if (url.includes('/git/trees/')) + return Promise.resolve({ ok: true, json: async () => mockTree }); + return new Promise((_resolve, reject) => { + init.signal?.addEventListener('abort', () => { + reject(new DOMException('aborted', 'AbortError')); + }); + }); + }); + vi.stubGlobal('fetch', fetchMock); + const request = createDocsClient({ mirror }).listAll(); + await vi.advanceTimersByTimeAsync(5_000); + await expect(request).resolves.toHaveLength(3); + } finally { + vi.useRealTimers(); + } + }); + + it('rejects a mirror that is not a plain http(s) URL', () => { + for (const value of ['docs.example.org', 'ftp://docs.example.org', `${mirror}?v=1`]) { + expect(() => createDocsClient({ mirror: value })).toThrow(TypeError); + } + }); +}); diff --git a/src/client.ts b/src/client.ts index 5226b54..8dafb8c 100644 --- a/src/client.ts +++ b/src/client.ts @@ -43,6 +43,7 @@ const SKIP = new Set([ ]); const TREE_KEY = 'tree'; +const MIRROR_TIMEOUT_MS = 5_000; const SHA = /^[0-9a-f]{40}$/; const SEARCH_CONCURRENCY = 6; const SEARCH_RESULT_LIMIT = 20; @@ -240,6 +241,15 @@ function assertBranchRef(value: string): void { } } +function mirrorBase(value: string | undefined): string | undefined { + if (value === undefined) return undefined; + const url = URL.canParse(value) ? new URL(value) : undefined; + if (!url || !['https:', 'http:'].includes(url.protocol) || url.search || url.hash) { + throw new TypeError('mirror must be an http(s) URL without a query or fragment'); + } + return url.href.replace(/\/+$/, ''); +} + function cacheTtl(value: number | undefined, fallback: number, name: string): number { const ttl = value ?? fallback; if (!Number.isFinite(ttl) || ttl < 0) { @@ -345,6 +355,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { const apiRepoUrl = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`; const rawRepoUrl = `https://raw.githubusercontent.com/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`; const encodedBranch = encodeURIComponent(branch); + const mirror = mirrorBase(options.mirror); const dirTtlMs = cacheTtl(options.cacheTtlMs?.dir, DEFAULTS.dirTtlMs, 'cacheTtlMs.dir'); const fileTtlMs = cacheTtl(options.cacheTtlMs?.file, DEFAULTS.fileTtlMs, 'cacheTtlMs.file'); @@ -393,6 +404,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { url: string, timeoutMs: number, consume: (response: Response) => Promise, + requestHeaders = headers(), ): Promise { const ctrl = new AbortController(); const timer = setTimeout(() => { @@ -400,7 +412,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { }, timeoutMs); let response: Response | undefined; try { - response = await fetch(url, { signal: ctrl.signal, headers: headers() }); + response = await fetch(url, { signal: ctrl.signal, headers: requestHeaders }); return await consume(response); } finally { clearTimeout(timer); @@ -409,6 +421,37 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { } } + async function fromMirror( + base: string, + path: string, + read: (response: Response) => Promise, + ): Promise { + try { + return await withResponse( + `${base}/${path}`, + MIRROR_TIMEOUT_MS, + async (response) => (response.ok ? await read(response) : undefined), + {}, + ); + } catch { + return undefined; + } + } + + function keepTree(items: DocItem[], generation: number): DocItem[] { + if (generation === cacheGeneration) { + treeCache.set(TREE_KEY, copyItems(items)); + if (store) storeWrite(TREE_KEY, JSON.stringify(items)); + } + return items; + } + + async function keepFile(path: string, content: string, generation: number): Promise { + if (generation === cacheGeneration) fileCache.set(path, content); + if (store) storeWrite(`blob-${await blobSha(content)}`, content); + return content; + } + function recoverFailure( cache: TtlCache, key: string, @@ -472,6 +515,13 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { async function loadAll(generation: number): Promise { const key = TREE_KEY; + if (mirror !== undefined) { + const mirrored = await fromMirror(mirror, 'index.json', async (response) => { + const data = parseTreeResponse(await response.json()); + return data.truncated ? undefined : filterTree(data.tree); + }); + if (mirrored) return keepTree(mirrored, generation); + } const url = `${apiRepoUrl}/git/trees/${encodedBranch}?recursive=1`; try { return await withResponse(url, 20_000, async (response) => { @@ -490,12 +540,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { 'GitHub truncated the repository tree (too many files) -- results would be incomplete', ); } - const items = filterTree(data.tree); - if (generation === cacheGeneration) { - treeCache.set(key, copyItems(items)); - if (store) storeWrite(key, JSON.stringify(items)); - } - return items; + return keepTree(filterTree(data.tree), generation); }); } catch (error) { if (error instanceof DocsFetchError) throw error; @@ -531,6 +576,12 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { if (generation === cacheGeneration) fileCache.set(path, stored); return stored; } + if (mirror !== undefined) { + const mirrored = await fromMirror(mirror, `raw/${encodePath(path)}`, (response) => + response.text(), + ); + if (mirrored !== undefined) return keepFile(path, mirrored, generation); + } const url = `${rawRepoUrl}/${encodedBranch}/${encodePath(path)}`; try { return await withResponse(url, 15_000, async (response) => { @@ -539,10 +590,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { if (stale !== undefined && (await isTransientResponse(response))) return stale; throw new DocsFetchError(path, response.status, `HTTP ${String(response.status)}`); } - const content = await response.text(); - if (generation === cacheGeneration) fileCache.set(path, content); - if (store) storeWrite(`blob-${await blobSha(content)}`, content); - return content; + return keepFile(path, await response.text(), generation); }); } catch (error) { if (error instanceof DocsFetchError) throw error; diff --git a/src/types.ts b/src/types.ts index a09fa6a..5503ad2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -58,6 +58,7 @@ export interface DocsClientOptions { file?: number; }; store?: DocsStore; + mirror?: string; } export interface DocsClient { From ad8b82266542523080e8e5d97c55e343bcd895f9 Mon Sep 17 00:00:00 2001 From: m1ngsama Date: Sat, 26 Sep 2026 22:39:28 +0800 Subject: [PATCH 2/2] fix: stop asking a failing mirror and keep it for missing files --- src/__tests__/client.test.ts | 27 +++++++++++++++++++++++++-- src/client.ts | 8 +++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index d1ffafa..94a6e98 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -1105,7 +1105,7 @@ describe('mirror', () => { ]); }); - it('falls back to GitHub when the mirror fails', async () => { + it('falls back to GitHub when the mirror fails and stops asking it', async () => { const fetchMock = routes({ mirror: () => ({ ok: false, status: 503 }), github: (url) => @@ -1117,7 +1117,30 @@ describe('mirror', () => { await expect(client.listAll()).resolves.toHaveLength(3); await expect(client.getFile('intro.md')).resolves.toBe('# From GitHub'); - expect(fetchMock).toHaveBeenCalledTimes(4); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); + + it('keeps using the mirror after a single missing file', async () => { + const fetchMock = routes({ + mirror: (url) => + url.endsWith('/index.json') + ? { ok: true, json: async () => mirrorTree } + : url.endsWith('/raw/intro.md') + ? { ok: false, status: 404 } + : { ok: true, text: async () => '# From mirror' }, + github: () => ({ ok: true, text: async () => '# From GitHub' }), + }); + const client = createDocsClient({ mirror }); + + await client.listAll(); + await expect(client.getFile('intro.md')).resolves.toBe('# From GitHub'); + await expect(client.getFile('guide/setup.md')).resolves.toBe('# From mirror'); + expect(fetchMock.mock.calls.map(([url]) => String(url).split('/')[2])).toEqual([ + 'docs.example.org', + 'docs.example.org', + 'raw.githubusercontent.com', + 'docs.example.org', + ]); }); it.each([ diff --git a/src/client.ts b/src/client.ts index 8dafb8c..771b019 100644 --- a/src/client.ts +++ b/src/client.ts @@ -356,6 +356,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { const rawRepoUrl = `https://raw.githubusercontent.com/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`; const encodedBranch = encodeURIComponent(branch); const mirror = mirrorBase(options.mirror); + let mirrorFailed = false; const dirTtlMs = cacheTtl(options.cacheTtlMs?.dir, DEFAULTS.dirTtlMs, 'cacheTtlMs.dir'); const fileTtlMs = cacheTtl(options.cacheTtlMs?.file, DEFAULTS.fileTtlMs, 'cacheTtlMs.file'); @@ -426,14 +427,19 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { path: string, read: (response: Response) => Promise, ): Promise { + if (mirrorFailed) return undefined; try { return await withResponse( `${base}/${path}`, MIRROR_TIMEOUT_MS, - async (response) => (response.ok ? await read(response) : undefined), + async (response) => { + if (response.status >= 500) mirrorFailed = true; + return response.ok ? await read(response) : undefined; + }, {}, ); } catch { + mirrorFailed = true; return undefined; } }