From 11e8a4a9a483c6b4b227b27def4aa5270812b641 Mon Sep 17 00:00:00 2001 From: m1ngsama Date: Sat, 26 Sep 2026 17:41:55 +0800 Subject: [PATCH] feat: persist the tree and file content through an optional store --- README.md | 15 +++++- scripts/check-package.mjs | 10 ++-- src/__tests__/client.test.ts | 94 ++++++++++++++++++++++++++++++++++ src/client.ts | 98 +++++++++++++++++++++++++++++++++--- src/index.ts | 1 + src/types.ts | 8 +++ 6 files changed, 216 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7ff806d..610f678 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ const matches = await docs.search('repair', { pathPrefix: 'repair' }); | `token` | `GITHUB_TOKEN` or `GH_TOKEN` | GitHub token | | `cacheTtlMs.dir` | `300000` | Directory and tree cache TTL | | `cacheTtlMs.file` | `600000` | File cache TTL | +| `store` | none | Persistent cache, see below | ### `docs.listDir(path?)` @@ -48,7 +49,12 @@ Returns raw file content. ### `docs.listAll()` -Lists every Markdown file through GitHub's recursive tree API. +Lists every Markdown file through GitHub's recursive tree API. Each item carries its Git blob `sha`. + +### `docs.peekAll()` + +Returns the last known tree without a request: the one fetched in this process, else the one in +`store`, else `undefined`. ### `docs.listSections()` @@ -60,6 +66,13 @@ Returns content with its route, section, title, summary, and semantic component metadata covers `PageHero`, `FactStrip`, `LinkCard`, `Split`, `TimelineEntry`, and `Figure` without imposing a renderer. +### `store` + +An object with `read(key): string | undefined` and `write(key, value): void`. The client keeps the +tree under `tree` and file content under `blob-`, where `` is the Git blob id. `getFile` +serves stored content only when its hash matches the blob id in the last known tree, so a changed +file is always refetched. Store errors and corrupt entries are ignored; eviction is up to the store. + ### `docs.search(query, options?)` Searches paths, titles, summaries, Markdown text, and semantic component attributes. Results are diff --git a/scripts/check-package.mjs b/scripts/check-package.mjs index 2ce6408..7365733 100644 --- a/scripts/check-package.mjs +++ b/scripts/check-package.mjs @@ -42,6 +42,7 @@ try { 'const client = createDocsClient();', "if (typeof client.listDir !== 'function') throw new TypeError('invalid client export');", "if (typeof client.listAll !== 'function') throw new TypeError('invalid tree export');", + "if (client.peekAll() !== undefined) throw new TypeError('invalid peek export');", "if (typeof client.listSections !== 'function') throw new TypeError('invalid discovery export');", "if (typeof client.getDocument !== 'function') throw new TypeError('invalid document export');", "if (typeof client.search !== 'function') throw new TypeError('invalid search export');", @@ -55,21 +56,24 @@ try { join(temporaryDirectory, 'consumer.ts'), [ "import { createDocsClient } from '@nbtca/docs';", - "import type { DocComponent, DocItem, DocPage, DocSection, DocsClient, DocsClientOptions, DocsSearchOptions, DocsSearchResult } from '@nbtca/docs';", - "const item = { name: 'guide.md', path: 'repair/guide.md', type: 'file' } satisfies DocItem;", + "import type { DocComponent, DocItem, DocPage, DocSection, DocsClient, DocsClientOptions, DocsSearchOptions, DocsSearchResult, DocsStore } from '@nbtca/docs';", + "const item = { name: 'guide.md', path: 'repair/guide.md', type: 'file', sha: 'b5aaad7d6dda27ea24335cdd4722c8129113f4cd' } satisfies DocItem;", + 'const store: DocsStore = { read: () => undefined, write: () => undefined };', "const component = { name: 'Figure', attributes: { caption: 'Example' } } satisfies DocComponent;", "const page = { components: [component], content: '# Guide', name: item.name, path: item.path, route: '/repair/guide', section: 'repair', summary: 'Repair guide', title: 'Guide' } satisfies DocPage;", "const section = { count: 1, indexPath: 'repair/index.md', path: 'repair' } satisfies DocSection;", "const searchOptions = { pathPrefix: 'repair', limit: 10 } satisfies DocsSearchOptions;", "const searchResult = { excerpt: 'Repair guide', name: item.name, path: item.path, route: page.route, score: 42, section: page.section, summary: page.summary, title: page.title } satisfies DocsSearchResult;", - "const options = { branch: 'main', cacheTtlMs: { dir: 0, file: 0 } } satisfies DocsClientOptions;", + "const options = { branch: 'main', cacheTtlMs: { dir: 0, file: 0 }, store } satisfies DocsClientOptions;", 'const client: DocsClient = createDocsClient(options);', 'async function consumePromptContract(docs: DocsClient): Promise {', ' const items: DocItem[] = await docs.listAll();', + ' const known: DocItem[] | undefined = docs.peekAll();', ' const document: DocPage = await docs.getDocument(item.path);', " const results: DocsSearchResult[] = await docs.search('repair', searchOptions);", ' docs.clear();', ' void items;', + ' void known;', ' void document;', ' void results;', '}', diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index 8c1093e..e2a1801 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -776,6 +776,100 @@ describe('listAll', () => { }); }); +describe('persistent store', () => { + const guideSha = 'b5aaad7d6dda27ea24335cdd4722c8129113f4cd'; + const changedSha = 'b125dfa2b24f5e3865648e4e3ca989bf0804bed1'; + const tree = (sha: string) => ({ + truncated: false, + tree: [{ path: 'repair/guide.md', type: 'blob', sha }], + }); + + function memoryStore(entries: Record = {}) { + const map = new Map(Object.entries(entries)); + return { + map, + read: (key: string) => map.get(key), + write: (key: string, value: string) => { + map.set(key, value); + }, + }; + } + + it('exposes blob shas and persists the tree', async () => { + mockFetch({ ok: true, json: async () => tree(guideSha) }); + const store = memoryStore(); + const items = await createDocsClient({ store }).listAll(); + expect(items).toEqual([ + { name: 'guide.md', path: 'repair/guide.md', type: 'file', sha: guideSha }, + ]); + expect(createDocsClient({ store }).peekAll()).toEqual(items); + }); + + it('drops shas that are not Git object ids', async () => { + mockFetch({ ok: true, json: async () => tree('../escape') }); + const [item] = await createDocsClient().listAll(); + expect(item).not.toHaveProperty('sha'); + }); + + it('peeks nothing without a stored or fetched tree', () => { + expect(createDocsClient().peekAll()).toBeUndefined(); + expect(createDocsClient({ store: memoryStore() }).peekAll()).toBeUndefined(); + }); + + it('serves a stored blob matching the known sha without fetching', async () => { + const spy = vi.fn().mockRejectedValue(new TypeError('fetch failed')); + vi.stubGlobal('fetch', spy); + const store = memoryStore({ + tree: JSON.stringify([ + { name: 'guide.md', path: 'repair/guide.md', type: 'file', sha: guideSha }, + ]), + [`blob-${guideSha}`]: '# Guide', + }); + await expect(createDocsClient({ store }).getFile('repair/guide.md')).resolves.toBe('# Guide'); + expect(spy).not.toHaveBeenCalled(); + }); + + it('refetches a changed file and stores it under its own sha', async () => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValueOnce({ ok: true, json: async () => tree(changedSha) }) + .mockResolvedValueOnce({ ok: true, text: async () => '# Guide v2' }), + ); + const store = memoryStore({ [`blob-${guideSha}`]: '# Guide' }); + const client = createDocsClient({ store }); + await client.listAll(); + await expect(client.getFile('repair/guide.md')).resolves.toBe('# Guide v2'); + expect(store.map.get(`blob-${changedSha}`)).toBe('# Guide v2'); + }); + + it('ignores corrupt entries and failing stores', async () => { + mockFetch({ ok: true, text: async () => '# Guide' }); + const corrupt = memoryStore({ + tree: JSON.stringify([ + { name: 'guide.md', path: 'repair/guide.md', type: 'file', sha: guideSha }, + ]), + [`blob-${guideSha}`]: '# Gui', + }); + await expect(createDocsClient({ store: corrupt }).getFile('repair/guide.md')).resolves.toBe( + '# Guide', + ); + expect(createDocsClient({ store: memoryStore({ tree: '{' }) }).peekAll()).toBeUndefined(); + const failing = { + read: () => { + throw new Error('EACCES'); + }, + write: () => { + throw new Error('ENOSPC'); + }, + }; + await expect(createDocsClient({ store: failing }).getFile('repair/guide.md')).resolves.toBe( + '# Guide', + ); + }); +}); + describe('document discovery', () => { it('groups documents by top-level section and identifies section indexes', async () => { mockFetch({ diff --git a/src/client.ts b/src/client.ts index e5b3b7b..5226b54 100644 --- a/src/client.ts +++ b/src/client.ts @@ -42,6 +42,8 @@ const SKIP = new Set([ 'docs', ]); +const TREE_KEY = 'tree'; +const SHA = /^[0-9a-f]{40}$/; const SEARCH_CONCURRENCY = 6; const SEARCH_RESULT_LIMIT = 20; @@ -57,6 +59,7 @@ function filterAndSort(raw: GitHubItem[]): DocItem[] { name: item.name, path: item.path, type: item.type === 'dir' ? 'dir' : 'file', + ...shaOf(item), })) .sort((a, b) => { if (a.type !== b.type) return a.type === 'dir' ? -1 : 1; @@ -75,10 +78,45 @@ function filterTree(items: GitHubTreeItem[]): DocItem[] { name: item.path.slice(item.path.lastIndexOf('/') + 1), path: item.path, type: 'file' as const, + ...shaOf(item), })) .sort((a, b) => a.path.localeCompare(b.path)); } +function shaOf(item: { sha?: unknown }): { sha?: string } { + return typeof item.sha === 'string' && SHA.test(item.sha) ? { sha: item.sha } : {}; +} + +async function blobSha(content: string): Promise { + const body = new TextEncoder().encode(content); + const header = new TextEncoder().encode(`blob ${String(body.length)}\0`); + const object = new Uint8Array(header.length + body.length); + object.set(header); + object.set(body, header.length); + const digest = new Uint8Array(await crypto.subtle.digest('SHA-1', object)); + return Array.from(digest, (byte) => byte.toString(16).padStart(2, '0')).join(''); +} + +function isDocItem(value: unknown): value is DocItem { + return ( + isRecord(value) && + typeof value.name === 'string' && + typeof value.path === 'string' && + (value.type === 'file' || value.type === 'dir') && + (value.sha === undefined || typeof value.sha === 'string') + ); +} + +function parseStoredTree(value: string | undefined): DocItem[] | undefined { + if (value === undefined) return undefined; + try { + const items: unknown = JSON.parse(value); + return Array.isArray(items) && items.every(isDocItem) ? items : undefined; + } catch { + return undefined; + } +} + function copyItems(items: DocItem[]): DocItem[] { return items.map((item) => ({ ...item })); } @@ -132,11 +170,13 @@ interface GitHubItem { name: string; path: string; type: string; + sha?: unknown; } interface GitHubTreeItem { path: string; type: string; + sha?: unknown; } interface GitHubTreeResponse { @@ -314,8 +354,33 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { const dirRequests = new Map>(); const fileRequests = new Map>(); const treeRequests = new Map>(); + const store = options.store; + let storedTree: DocItem[] | undefined; let cacheGeneration = 0; + function storeRead(key: string): string | undefined { + try { + return store?.read(key); + } catch { + return undefined; + } + } + + function storeWrite(key: string, value: string): void { + try { + store?.write(key, value); + } catch { + return; + } + } + + function knownTree(): DocItem[] | undefined { + const fetched = treeCache.getStale(TREE_KEY); + if (fetched) return fetched; + storedTree ??= parseStoredTree(storeRead(TREE_KEY)); + return storedTree; + } + function headers(): Record { const requestHeaders: Record = { Accept: 'application/vnd.github.v3+json', @@ -406,7 +471,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { } async function loadAll(generation: number): Promise { - const key = '__tree__'; + const key = TREE_KEY; const url = `${apiRepoUrl}/git/trees/${encodedBranch}?recursive=1`; try { return await withResponse(url, 20_000, async (response) => { @@ -426,7 +491,10 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { ); } const items = filterTree(data.tree); - if (generation === cacheGeneration) treeCache.set(key, copyItems(items)); + if (generation === cacheGeneration) { + treeCache.set(key, copyItems(items)); + if (store) storeWrite(key, JSON.stringify(items)); + } return items; }); } catch (error) { @@ -436,17 +504,33 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { } function listAll(): Promise { - const key = '__tree__'; - const hit = treeCache.get(key); + const hit = treeCache.get(TREE_KEY); if (hit) return Promise.resolve(copyItems(hit)); - return shareRequest(treeRequests, key, () => loadAll(cacheGeneration)).then(copyItems); + return shareRequest(treeRequests, TREE_KEY, () => loadAll(cacheGeneration)).then(copyItems); + } + + function peekAll(): DocItem[] | undefined { + const items = knownTree(); + return items && copyItems(items); } async function listSections(): Promise { return sectionsFromItems(await listAll()); } + async function loadStoredFile(path: string): Promise { + const sha = knownTree()?.find((item) => item.path === path)?.sha; + if (sha === undefined) return undefined; + const content = storeRead(`blob-${sha}`); + return content !== undefined && (await blobSha(content)) === sha ? content : undefined; + } + async function loadFile(path: string, generation: number): Promise { + const stored = store && (await loadStoredFile(path)); + if (stored !== undefined) { + if (generation === cacheGeneration) fileCache.set(path, stored); + return stored; + } const url = `${rawRepoUrl}/${encodedBranch}/${encodePath(path)}`; try { return await withResponse(url, 15_000, async (response) => { @@ -457,6 +541,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { } const content = await response.text(); if (generation === cacheGeneration) fileCache.set(path, content); + if (store) storeWrite(`blob-${await blobSha(content)}`, content); return content; }); } catch (error) { @@ -521,6 +606,7 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { function clear(): void { cacheGeneration += 1; + storedTree = undefined; dirCache.clear(); fileCache.clear(); treeCache.clear(); @@ -529,5 +615,5 @@ export function createDocsClient(options: DocsClientOptions = {}): DocsClient { treeRequests.clear(); } - return { listDir, listAll, listSections, getFile, getDocument, search, clear }; + return { listDir, listAll, peekAll, listSections, getFile, getDocument, search, clear }; } diff --git a/src/index.ts b/src/index.ts index dcd6949..8721912 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,5 +9,6 @@ export type { DocsClientOptions, DocsSearchOptions, DocsSearchResult, + DocsStore, } from './types.js'; export { DocsFetchError } from './types.js'; diff --git a/src/types.ts b/src/types.ts index 0eb81f0..a09fa6a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -2,6 +2,7 @@ export interface DocItem { name: string; path: string; type: 'file' | 'dir'; + sha?: string; } export interface DocComponent { @@ -42,6 +43,11 @@ export interface DocsSearchResult { title: string; } +export interface DocsStore { + read(key: string): string | undefined; + write(key: string, value: string): void; +} + export interface DocsClientOptions { owner?: string; repo?: string; @@ -51,11 +57,13 @@ export interface DocsClientOptions { dir?: number; file?: number; }; + store?: DocsStore; } export interface DocsClient { listDir(path?: string): Promise; listAll(): Promise; + peekAll(): DocItem[] | undefined; listSections(): Promise; getFile(path: string): Promise; getDocument(path: string): Promise;