feat: try an optional mirror before GitHub for the tree and files

README.md

@@ -37,6 +37,7 @@ const matches = await docs.search('repair', { pathPrefix: 'repair' });
 | `cacheTtlMs.dir`  | `300000`                     | Directory and tree cache TTL |
 | `cacheTtlMs.file` | `600000`                     | File cache TTL               |
 | `store`           | none                         | Persistent cache, see below  |
+| `mirror`          | none                         | Mirror base URL, see below   |
 
 ### `docs.listDir(path?)`
 
@@ -73,6 +74,13 @@ tree under `tree` and file content under `blob-<sha>`, where `<sha>` is the Git
 serves stored content only when its hash matches the blob id in the last known tree, so a changed
 file is always refetched. Store errors and corrupt entries are ignored; eviction is up to the store.
 
+### `mirror`
+
+A base URL tried before GitHub by `listAll` and `getFile`, such as
+`https://docs.nbtca.space/docs-api`. It serves `index.json` in the shape of GitHub's recursive tree
+response and each file at `raw/<path>`. Any mirror failure, invalid or truncated index, or wait past
+5 seconds falls back to GitHub. The GitHub token is never sent to the mirror.
+
 ### `docs.search(query, options?)`
 
 Searches paths, titles, summaries, Markdown text, and semantic component attributes. Results are

src/__tests__/client.test.ts

@@ -1047,3 +1047,169 @@ describe('search', () => {
     expect(fetchMock).not.toHaveBeenCalled();
   });
 });
+
+describe('mirror', () => {
+  const mirror = 'https://docs.example.org/docs-api/';
+  const guideSha = 'b5aaad7d6dda27ea24335cdd4722c8129113f4cd';
+  const mirrorTree = {
+    truncated: false,
+    tree: [
+      { path: 'repair/guide.md', type: 'blob', sha: guideSha },
+      { path: 'README.md', type: 'blob', sha: guideSha },
+    ],
+  };
+
+  function routes(handlers: {
+    mirror: (url: string) => unknown;
+    github?: (url: string) => unknown;
+  }) {
+    const fetchMock = vi.fn().mockImplementation(async (url: string) => {
+      if (url.startsWith('https://docs.example.org/')) return handlers.mirror(url);
+      if (!handlers.github) throw new TypeError('GitHub is unreachable');
+      return handlers.github(url);
+    });
+    vi.stubGlobal('fetch', fetchMock);
+    return fetchMock;
+  }
+
+  function memoryStore() {
+    const map = new Map<string, string>();
+    return {
+      map,
+      read: (key: string) => map.get(key),
+      write: (key: string, value: string) => {
+        map.set(key, value);
+      },
+    };
+  }
+
+  it('lists and reads from the mirror first without sending the GitHub token', async () => {
+    const fetchMock = routes({
+      mirror: (url) =>
+        url.endsWith('/index.json')
+          ? { ok: true, json: async () => mirrorTree }
+          : { ok: true, text: async () => '# Guide' },
+    });
+    const client = createDocsClient({ mirror, token: 'secret' });
+
+    await expect(client.listAll()).resolves.toEqual([
+      { name: 'guide.md', path: 'repair/guide.md', type: 'file', sha: guideSha },
+    ]);
+    await expect(client.getFile('repair/guide.md')).resolves.toBe('# Guide');
+    expect(fetchMock.mock.calls).toEqual([
+      ['https://docs.example.org/docs-api/index.json', expect.objectContaining({ headers: {} })],
+      [
+        'https://docs.example.org/docs-api/raw/repair/guide.md',
+        expect.objectContaining({ headers: {} }),
+      ],
+    ]);
+  });
+
+  it('falls back to GitHub when the mirror fails and stops asking it', async () => {
+    const fetchMock = routes({
+      mirror: () => ({ ok: false, status: 503 }),
+      github: (url) =>
+        url.includes('/git/trees/')
+          ? { ok: true, json: async () => mockTree }
+          : { ok: true, text: async () => '# From GitHub' },
+    });
+    const client = createDocsClient({ mirror });
+
+    await expect(client.listAll()).resolves.toHaveLength(3);
+    await expect(client.getFile('intro.md')).resolves.toBe('# From GitHub');
+    expect(fetchMock).toHaveBeenCalledTimes(3);
+  });
+
+  it('keeps using the mirror after a single missing file', async () => {
+    const fetchMock = routes({
+      mirror: (url) =>
+        url.endsWith('/index.json')
+          ? { ok: true, json: async () => mirrorTree }
+          : url.endsWith('/raw/intro.md')
+            ? { ok: false, status: 404 }
+            : { ok: true, text: async () => '# From mirror' },
+      github: () => ({ ok: true, text: async () => '# From GitHub' }),
+    });
+    const client = createDocsClient({ mirror });
+
+    await client.listAll();
+    await expect(client.getFile('intro.md')).resolves.toBe('# From GitHub');
+    await expect(client.getFile('guide/setup.md')).resolves.toBe('# From mirror');
+    expect(fetchMock.mock.calls.map(([url]) => String(url).split('/')[2])).toEqual([
+      'docs.example.org',
+      'docs.example.org',
+      'raw.githubusercontent.com',
+      'docs.example.org',
+    ]);
+  });
+
+  it.each([
+    ['a malformed index', { message: 'not a tree' }],
+    ['a truncated index', { ...mirrorTree, truncated: true }],
+  ])('falls back to GitHub on %s', async (_label, index) => {
+    routes({
+      mirror: () => ({ ok: true, json: async () => index }),
+      github: () => ({ ok: true, json: async () => mockTree }),
+    });
+    await expect(createDocsClient({ mirror }).listAll()).resolves.toHaveLength(3);
+  });
+
+  it('keeps the GitHub error when both sources fail', async () => {
+    routes({
+      mirror: () => {
+        throw new TypeError('fetch failed');
+      },
+      github: () => ({ ok: false, status: 404 }),
+    });
+    await expect(createDocsClient({ mirror }).getFile('missing.md')).rejects.toMatchObject({
+      name: 'DocsFetchError',
+      status: 404,
+    });
+  });
+
+  it('stores mirror content under its verified blob sha', async () => {
+    const fetchMock = routes({
+      mirror: (url) =>
+        url.endsWith('/index.json')
+          ? { ok: true, json: async () => mirrorTree }
+          : { ok: true, text: async () => '# Guide' },
+    });
+    const store = memoryStore();
+    await createDocsClient({ mirror, store }).listAll();
+    await createDocsClient({ mirror, store }).getFile('repair/guide.md');
+    expect(store.map.get(`blob-${guideSha}`)).toBe('# Guide');
+
+    fetchMock.mockClear();
+    await expect(createDocsClient({ mirror, store }).getFile('repair/guide.md')).resolves.toBe(
+      '# Guide',
+    );
+    expect(fetchMock).not.toHaveBeenCalled();
+  });
+
+  it('gives up on a slow mirror after five seconds', async () => {
+    vi.useFakeTimers();
+    try {
+      const fetchMock = vi.fn().mockImplementation((url: string, init: RequestInit) => {
+        if (url.includes('/git/trees/'))
+          return Promise.resolve({ ok: true, json: async () => mockTree });
+        return new Promise((_resolve, reject) => {
+          init.signal?.addEventListener('abort', () => {
+            reject(new DOMException('aborted', 'AbortError'));
+          });
+        });
+      });
+      vi.stubGlobal('fetch', fetchMock);
+      const request = createDocsClient({ mirror }).listAll();
+      await vi.advanceTimersByTimeAsync(5_000);
+      await expect(request).resolves.toHaveLength(3);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
+  it('rejects a mirror that is not a plain http(s) URL', () => {
+    for (const value of ['docs.example.org', 'ftp://docs.example.org', `${mirror}?v=1`]) {
+      expect(() => createDocsClient({ mirror: value })).toThrow(TypeError);
+    }
+  });
+});

src/client.ts

@@ -43,6 +43,7 @@ const SKIP = new Set([
 ]);
 
 const TREE_KEY = 'tree';
+const MIRROR_TIMEOUT_MS = 5_000;
 const SHA = /^[0-9a-f]{40}$/;
 const SEARCH_CONCURRENCY = 6;
 const SEARCH_RESULT_LIMIT = 20;
@@ -240,6 +241,15 @@ function assertBranchRef(value: string): void {
   }
 }
 
+function mirrorBase(value: string | undefined): string | undefined {
+  if (value === undefined) return undefined;
+  const url = URL.canParse(value) ? new URL(value) : undefined;
+  if (!url || !['https:', 'http:'].includes(url.protocol) || url.search || url.hash) {
+    throw new TypeError('mirror must be an http(s) URL without a query or fragment');
+  }
+  return url.href.replace(/\/+$/, '');
+}
+
 function cacheTtl(value: number | undefined, fallback: number, name: string): number {
   const ttl = value ?? fallback;
   if (!Number.isFinite(ttl) || ttl < 0) {
@@ -345,6 +355,8 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
   const apiRepoUrl = `https://api.github.com/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`;
   const rawRepoUrl = `https://raw.githubusercontent.com/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`;
   const encodedBranch = encodeURIComponent(branch);
+  const mirror = mirrorBase(options.mirror);
+  let mirrorFailed = false;
   const dirTtlMs = cacheTtl(options.cacheTtlMs?.dir, DEFAULTS.dirTtlMs, 'cacheTtlMs.dir');
   const fileTtlMs = cacheTtl(options.cacheTtlMs?.file, DEFAULTS.fileTtlMs, 'cacheTtlMs.file');
 
@@ -393,14 +405,15 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
     url: string,
     timeoutMs: number,
     consume: (response: Response) => Promise<T>,
+    requestHeaders = headers(),
   ): Promise<T> {
     const ctrl = new AbortController();
     const timer = setTimeout(() => {
       ctrl.abort();
     }, timeoutMs);
     let response: Response | undefined;
     try {
-      response = await fetch(url, { signal: ctrl.signal, headers: headers() });
+      response = await fetch(url, { signal: ctrl.signal, headers: requestHeaders });
       return await consume(response);
     } finally {
       clearTimeout(timer);
@@ -409,6 +422,42 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
     }
   }
 
+  async function fromMirror<T>(
+    base: string,
+    path: string,
+    read: (response: Response) => Promise<T | undefined>,
+  ): Promise<T | undefined> {
+    if (mirrorFailed) return undefined;
+    try {
+      return await withResponse(
+        `${base}/${path}`,
+        MIRROR_TIMEOUT_MS,
+        async (response) => {
+          if (response.status >= 500) mirrorFailed = true;
+          return response.ok ? await read(response) : undefined;
+        },
+        {},
+      );
+    } catch {
+      mirrorFailed = true;
+      return undefined;
+    }
+  }
+
+  function keepTree(items: DocItem[], generation: number): DocItem[] {
+    if (generation === cacheGeneration) {
+      treeCache.set(TREE_KEY, copyItems(items));
+      if (store) storeWrite(TREE_KEY, JSON.stringify(items));
+    }
+    return items;
+  }
+
+  async function keepFile(path: string, content: string, generation: number): Promise<string> {
+    if (generation === cacheGeneration) fileCache.set(path, content);
+    if (store) storeWrite(`blob-${await blobSha(content)}`, content);
+    return content;
+  }
+
   function recoverFailure<T>(
     cache: TtlCache<T>,
     key: string,
@@ -472,6 +521,13 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
 
   async function loadAll(generation: number): Promise<DocItem[]> {
     const key = TREE_KEY;
+    if (mirror !== undefined) {
+      const mirrored = await fromMirror(mirror, 'index.json', async (response) => {
+        const data = parseTreeResponse(await response.json());
+        return data.truncated ? undefined : filterTree(data.tree);
+      });
+      if (mirrored) return keepTree(mirrored, generation);
+    }
     const url = `${apiRepoUrl}/git/trees/${encodedBranch}?recursive=1`;
     try {
       return await withResponse(url, 20_000, async (response) => {
@@ -490,12 +546,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
             'GitHub truncated the repository tree (too many files) -- results would be incomplete',
           );
         }
-        const items = filterTree(data.tree);
-        if (generation === cacheGeneration) {
-          treeCache.set(key, copyItems(items));
-          if (store) storeWrite(key, JSON.stringify(items));
-        }
-        return items;
+        return keepTree(filterTree(data.tree), generation);
       });
     } catch (error) {
       if (error instanceof DocsFetchError) throw error;
@@ -531,6 +582,12 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
       if (generation === cacheGeneration) fileCache.set(path, stored);
       return stored;
     }
+    if (mirror !== undefined) {
+      const mirrored = await fromMirror(mirror, `raw/${encodePath(path)}`, (response) =>
+        response.text(),
+      );
+      if (mirrored !== undefined) return keepFile(path, mirrored, generation);
+    }
     const url = `${rawRepoUrl}/${encodedBranch}/${encodePath(path)}`;
     try {
       return await withResponse(url, 15_000, async (response) => {
@@ -539,10 +596,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient {
           if (stale !== undefined && (await isTransientResponse(response))) return stale;
           throw new DocsFetchError(path, response.status, `HTTP ${String(response.status)}`);
         }
-        const content = await response.text();
-        if (generation === cacheGeneration) fileCache.set(path, content);
-        if (store) storeWrite(`blob-${await blobSha(content)}`, content);
-        return content;
+        return keepFile(path, await response.text(), generation);
       });
     } catch (error) {
       if (error instanceof DocsFetchError) throw error;

src/types.ts

@@ -58,6 +58,7 @@ export interface DocsClientOptions {
     file?: number;
   };
   store?: DocsStore;
+  mirror?: string;
 }
 
 export interface DocsClient {