From 2f2afb029cfb138288f4c1ccde7913c3f4b76f08 Mon Sep 17 00:00:00 2001 From: DevBehindYou Date: Thu, 29 Oct 2026 23:21:52 +0530 Subject: [PATCH 1/2] Paginate controller job details and preserve complete exports --- docs/LLMISER_CONTROLLER_PAGINATION.md | 42 ++++++ docs/LLMISER_OPTIMIZATION_PLAN.md | 8 +- docs/START_HERE.md | 2 + docs/ai-handover/HANDOVER.md | 2 + docs/ai-handover/PROGRESS.md | 2 + frontend/app/api/admin/jobs/[id]/route.js | 14 +- frontend/components/admin/ContentTreeView.js | 2 +- frontend/components/admin/JobDetail.js | 33 ++++- frontend/lib/i18n/messages/de.json | 8 +- frontend/lib/i18n/messages/en-US.json | 8 +- frontend/lib/i18n/messages/es.json | 8 +- frontend/lib/i18n/messages/ja.json | 8 +- frontend/lib/i18n/messages/ru.json | 8 +- frontend/lib/i18n/messages/zh-CN.json | 8 +- frontend/lib/models/adminApi.js | 6 +- frontend/lib/models/adminDetail.js | 45 +++++++ frontend/lib/models/contentTree.js | 5 +- frontend/lib/server/adminService.js | 77 +++++++++-- frontend/lib/server/supabaseRest.js | 16 ++- frontend/package-lock.json | 37 +++++ frontend/package.json | 1 + .../scripts/measure-controller-pagination.mjs | 40 ++++++ frontend/test/adminDetail.test.mjs | 127 ++++++++++++++++++ frontend/test/adminDetailHooks.test.mjs | 91 +++++++++++++ frontend/test/adminDetailRoute.test.mjs | 52 +++++++ frontend/test/contentTree.test.mjs | 11 ++ frontend/test/sql/pgliteSupabase.mjs | 28 ++++ frontend/viewmodels/useAdminViewModel.js | 95 +++++++++---- 28 files changed, 728 insertions(+), 56 deletions(-) create mode 100644 docs/LLMISER_CONTROLLER_PAGINATION.md create mode 100644 frontend/lib/models/adminDetail.js create mode 100644 frontend/scripts/measure-controller-pagination.mjs create mode 100644 frontend/test/adminDetail.test.mjs create mode 100644 frontend/test/adminDetailHooks.test.mjs create mode 100644 frontend/test/adminDetailRoute.test.mjs diff --git a/docs/LLMISER_CONTROLLER_PAGINATION.md b/docs/LLMISER_CONTROLLER_PAGINATION.md new file mode 100644 index 0000000..e85d8e8 --- /dev/null +++ b/docs/LLMISER_CONTROLLER_PAGINATION.md @@ -0,0 +1,42 @@ +# Controller detail pagination + +Updated 29 October 2026. Candidate on `codex/controller-detail-pagination`; release and production acceptance are still pending. The previously released runtime is PR #17, `9131a2ac6163f22b630b008b40accb482ed3e0b3`. + +Opening a large job previously returned all content nodes, work items, events and registered files. The controller now starts with 50 records per section, with independent **Load more** and retry controls. Counts distinguish loaded rows from the full section total. The tree and status badges explicitly describe loaded records; Refresh restarts every section. Six interface languages are supported; blogs and their visuals are unchanged. + +## Read and export contracts + +- Authenticated `GET /api/admin/jobs/:id` returns the job summary, four bounded arrays and `pages` metadata (`total`, `next`, and whole-job `active_total` for files). +- `?section=files|nodes|items|events&limit=50&cursor=...` reads one section. Default 50, maximum 100. Files additionally accept `stored=1` and `kind=INPUT|OUTPUT|EXPORT|PREVIEW`. +- Every query is scoped to the validated job UUID. Stable ascending order uses sequence/UUID for nodes and timestamp/UUID for the other sections. Opaque cursors bind job, section and file filters. PostgreSQL timestamp microseconds are preserved. +- Updated clients send `x-llmiser-detail-paging: 1`. An authenticated old cached client receives 409 with reload guidance, preventing its old ZIP code from exporting only page one. Authentication precedes this check. Sessions, compatibility cookies/CSRF headers, no-store/noindex and existing action/audit rules remain intact. +- Exact totals use PostgREST `Prefer: count=exact` and `Content-Range`. Later cursor requests also issue a HEAD count without the cursor. Initial detail reads use six REST calls: job summary, four section reads and active-file count. Counts and pages are separate reads, not a transactionally consistent snapshot; running jobs can change. Refresh reconciles them. +- ZIP actions resolve every matching ACTIVE file page before signing or downloading. Single-job ZIPs include all stored file kinds; selected-job ZIPs include OUTPUT files. Existing 200-file/200 MiB limits remain. Missing pages, duplicate IDs, changing counts, errors and cancellation reject collection rather than silently exporting an incomplete list. A change after collection can still make a signed download fail; this is not a Storage snapshot. + +No migration, index, RLS, retention, worker, pool, cron or service setting is changed. The existing `get_job_tree` RPC remains available for compatibility; this controller stops calling it. + +## Implementation map + +| Location | Responsibility | +| --- | --- | +| `frontend/lib/server/adminService.js` | Field allowlists, validation, stable cursors, bounded detail sections | +| `frontend/lib/server/supabaseRest.js` | GET/HEAD exact-count response handling; existing calls retain their old shape | +| `frontend/app/api/admin/jobs/[id]/route.js` | Authenticated paging negotiation and section routing | +| `frontend/lib/models/adminDetail.js` | Page deduplication and complete export metadata collection | +| `frontend/viewmodels/useAdminViewModel.js` | Independent section state, aborts and generation guards on refresh/job switch | +| `frontend/components/admin/JobDetail.js` | Loaded/total controls, Refresh and partial-tree notice | +| `frontend/lib/models/contentTree.js` | Set membership for PDF partitioning; PROCESSING status summaries | + +## Local evidence and limits + +Focused service, transport, route, export, React hook, localized rendering and tree checks passed: **12 tests**. Migrated PostgreSQL fixtures contain 123 records per section for two jobs, tied keys and microsecond timestamps. Traversal has exact IDs with no gaps or duplicates; filters and cursors cannot be reused across jobs/sections. A simulated server row cap still produces continuation. React tests exercise job switches, late responses despite abort, repeated clicks, concurrent sections, page failure/retry and refresh invalidation. Controls render counts, disabled state, live announcements and retry alerts in all six languages. `react-test-renderer` 18.3.1 is a development-only dependency matching installed React. + +Generated local measurement: 21 jobs and 22,000 rows per section, including 2,000 rows per section in the target job. The previous response shape was **3,701,322 bytes**; the initial bounded response was **65,901 bytes**, about **98.2% smaller**. This compares serialized response sizes, not heap peaks or page latency. All four collections traversed completely in 20 pages of 100 rows. Typed PostgreSQL EXPLAIN queries used existing job indexes for initial/deep reads and scoped counts. Some queries still sort/filter a job's matching rows; exact counts scan matching entries. This is evidence against adding an index in this slice, not a production latency or constant-cost guarantee. Raw generated plans and logs are private under `project-docs/controller-pagination/`. + +Reproduce generated plans/sizes from `frontend` with `node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON scripts/measure-controller-pagination.mjs`. This creates an in-memory migrated PGlite database and prints JSON; it does not contact production. Run `node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --test test/adminDetail*.test.mjs test/contentTree.test.mjs` for focused regression. + +Production baseline: one generated 72-text-file ZIP completed through the public Storage flow. Authenticated detail returned 73 nodes, four registered ACTIVE files, one work item and two events. The job has normal 48-hour retention; no cleanup or retention mutation was performed. After release, compare every section's IDs against this baseline, retrieve every stored file and reopen a ZIP containing them. No user documents are needed for acceptance. + +Full local regression: 276 of 278 passed in the sandbox; the two deployment-scope tests could not create temporary Git configurations. Both passed when rerun outside the sandbox. Lint, six dictionaries (637 messages each), SQL consistency, blog consistency and diff whitespace checks passed. The initial lint/build attempt also encountered local filesystem restrictions; the elevated build is being checked separately. CI, release and production results must be recorded before calling this released. React/SSR acceptance does not establish native browser layout, mobile overflow or representative-device performance. + +PostgREST documents [pagination/count headers and HEAD](https://docs.postgrest.org/en/stable/references/api/pagination_count.html). PostgreSQL explains why [unique ordering is necessary and large offsets remain costly](https://www.postgresql.org/docs/16/queries-limit.html). These contracts guide the cursor design; production measurements must remain separate from local generated evidence. diff --git a/docs/LLMISER_OPTIMIZATION_PLAN.md b/docs/LLMISER_OPTIMIZATION_PLAN.md index 7fae379..ba4e52f 100644 --- a/docs/LLMISER_OPTIMIZATION_PLAN.md +++ b/docs/LLMISER_OPTIMIZATION_PLAN.md @@ -22,7 +22,11 @@ The copied source already contains atomic job startup/finalization, cleanup reco Search migration and public redirects are live; see [search migration](LLMISER_SEARCH_MIGRATION.md). Analytics ingestion is verified; controller login/configuration is verified but authenticated operations are not; see [access and analytics](LLMISER_ANALYTICS_CONTROLLER.md). Keep-warm Vault and cron changes were applied and verified in the earlier cron task. Dependency-aware service retirement remains separate operational work. -## Current slice: retained result recovery +## Current slice: controller detail pagination + +Candidate: `codex/controller-detail-pagination`; see [implementation and evidence](LLMISER_CONTROLLER_PAGINATION.md). Files, nodes, work items and events now load 50 rows each, with stable unique cursors, exact live totals, per-section retry and refresh/job-switch guards. ZIP exports fetch every matching stored file page before signing/downloading. Existing indexes suffice for generated scoped query plans; no production SQL change is required. Focused acceptance passed; full regression, release and authenticated deployed comparison remain pending. + +## Historical slice: retained result recovery Branch: `codex/result-download-recovery`. Implementation and local acceptance are complete; CI/review and a separate release decision remain. This is not the production revision. @@ -57,7 +61,7 @@ Acceptance: expired link, later-part failure, complete Unicode assembly, restart The remaining queue is not an effort percentage. Track implementation, local regression/browser evidence, CI and deployed behavior independently. Do not apply production SQL or retire old services merely to complete a checklist. -## Current verification +## Historical recovery verification - Full frontend regression suite: **232 passed, 0 failed** on 22 October 2026. Log: private `project-docs/optimization-evidence/frontend-tests.log`. - Synthetic browser flow: initial final-download 403, browser reload, metadata-only recent-session recovery, complete Markdown download. Fixture counters remained **1 upload, 1 conversion start, 1 retrieval**. No production conversion services were called. diff --git a/docs/START_HERE.md b/docs/START_HERE.md index 7cf6324..765b5e6 100644 --- a/docs/START_HERE.md +++ b/docs/START_HERE.md @@ -4,6 +4,8 @@ Updated 29 October 2026. **LLMiser is the sole working repository and the source ## Current state +Current candidate: [controller detail pagination](LLMISER_CONTROLLER_PAGINATION.md) on `codex/controller-detail-pagination`. Bounded files/tree/work-items/events, exact totals, independent Load more/retry and complete ZIP metadata collection are implemented. Focused PostgreSQL/React/route checks passed; full regression, release and deployed acceptance are pending. No database migration is required. Production remains the revision recorded below until release is verified. + The app is live at https://llmiser.devbehindyou.com. PRs #1–#17 are merged; the latest verified runtime revision is `9131a2ac6163f22b630b008b40accb482ed3e0b3` from PR #17. All seven CI jobs passed before and after merge, including 270 frontend tests and Linux OCR; all six production Vercel statuses succeeded. Release checks confirmed that exact revision and readiness on every N/O/Z worker. Thirty protocol checks, eight workbook/ZIP equivalence checks, four post-release Storage conversion/recovery jobs and 20 public HTTP/redirect checks passed; all pools were 2/2 ready. See [shared deadlines](LLMISER_SHARED_DEADLINES.md), [Excel/assembly](LLMISER_EXCEL_ASSEMBLY_MEMORY.md), [CSV capacity](LLMISER_PARSER_CAPACITY.md) and [retirement evidence](LLMISER_LEGACY_RETIREMENT.md). The full app supports English (US), German, Japanese, Spanish, Russian and Simplified Chinese. The browser remembers the interface language; documents are not translated and OCR languages are unchanged. Blogs remain English. Eight articles, eight banners and forty figures are complete. The Chrome extension source is merged, but native acceptance and store publication remain pending. diff --git a/docs/ai-handover/HANDOVER.md b/docs/ai-handover/HANDOVER.md index 0de1aee..5054373 100644 --- a/docs/ai-handover/HANDOVER.md +++ b/docs/ai-handover/HANDOVER.md @@ -4,6 +4,8 @@ Updated 29 October 2026. Start with [START_HERE](../START_HERE.md). This replace ## What the owner wants now +Active implementation: [controller detail pagination](../LLMISER_CONTROLLER_PAGINATION.md), branch `codex/controller-detail-pagination`. The owner resumed this task after pausing it. Code and focused acceptance are complete; full validation, PR/merge/deployment and generated authenticated production comparison remain. Keep ZIP exports complete across pages, timestamp precision and job/section/filter cursor binding. Exact counts are live separate reads; Refresh resets all sections. No new SQL/index is needed from the measured existing-index plans. Runtime remains PR #17 until release evidence is recorded. + The owner resumed the optimization queue after the Excel/assembly release. Shared conversion deadlines are now released in PR #17. Continue from the remaining queue rather than repeating either slice. LLMiser must also remain independently maintainable after MDify and MDify-Pro are deleted. Keep immediate permanent redirects on the old public hosts and the minimal LLMiser-only fallback; repository/service deletion is not being executed here. ## Work already shipped diff --git a/docs/ai-handover/PROGRESS.md b/docs/ai-handover/PROGRESS.md index 2db3e3a..4e058bc 100644 --- a/docs/ai-handover/PROGRESS.md +++ b/docs/ai-handover/PROGRESS.md @@ -51,6 +51,8 @@ See [release record](../LLMISER_RELEASE_2026-10-26.md), [localization](../LLMISE ## Remaining work +In progress: [controller detail pagination](../LLMISER_CONTROLLER_PAGINATION.md) on `codex/controller-detail-pagination`. Implemented bounded detail collections, stable cursors/exact counts, per-section Load more/retry, refresh invalidation and complete ZIP collection. Twelve focused checks passed, including migrated PostgreSQL and real React hook races. Generated query plans use existing indexes; no migration is proposed. Authenticated production baseline is captured from one generated ZIP job (73 nodes, four ACTIVE files). Full tests/build, CI/merge and after-release comparison are pending; do not mark this released yet. Six dictionaries now have 637 candidate messages; production still has 631. + Released: [shared conversion deadlines](../LLMISER_SHARED_DEADLINES.md), PR #17 at 9131a2a. All seven CI jobs passed before/after merge, all six Vercel statuses succeeded and all six workers reported the exact revision/readiness. Thirty production auth/expired/invalid/valid/legacy protocol requests passed; eight future-header workbook/ZIP comparisons and two Storage baselines retained identical output/counts. Two additional Storage jobs exercised image-ZIP OCR/merge and scanned-PDF OCR/merge with exact recovery. Twenty public HTTP checks passed and all pools were 2/2 ready. Six generated Storage jobs in total across pre/post/multi-stage probes follow existing retention; cleanup was not manually run. Controller detail pagination is next; no whole-job lifetime or exactly-once write guarantee is claimed. Released: [Excel conversion and final-result assembly memory](../LLMISER_EXCEL_ASSEMBLY_MEMORY.md), explicitly selected by the owner, PR #15 at 3db4114. Local suites: 144 normal/101 archive tests; installed-version checks passed. All seven Linux CI jobs and six Vercel production statuses passed, with exact revision/readiness verified on all six workers including Render. Eight post-release generated multipart probes and two Storage-backed XLSX/ZIP conversions passed, including all four output objects and exact saved-result recovery. Before/after Markdown, counts and index hashes match; manifest job IDs correctly differ. All 20 HTTP checks passed and all pools were 2/2 ready. Four real jobs across before/after tests follow normal retention; cleanup execution itself was not tested. Local Windows OCR readiness-suite failure is preserved in the report; Linux OCR CI passed. diff --git a/frontend/app/api/admin/jobs/[id]/route.js b/frontend/app/api/admin/jobs/[id]/route.js index c16be19..c4acaa3 100644 --- a/frontend/app/api/admin/jobs/[id]/route.js +++ b/frontend/app/api/admin/jobs/[id]/route.js @@ -1,6 +1,16 @@ -import { jobDetail } from '../../../../../lib/server/adminService'; +import { jobDetail, jobDetailPage } from '../../../../../lib/server/adminService'; import { adminJson, withAdmin } from '../../../../../lib/server/adminRoute'; export const dynamic = 'force-dynamic'; -export const GET = withAdmin(async ({ params, db }) => adminJson(await jobDetail(db, params.id)), { name: 'job' }); +export const GET = withAdmin(async ({ request, params, db }) => { + // Old cached clients must reload rather than silently export only page one. + if (request.headers.get('x-llmiser-detail-paging') !== '1') { + return adminJson({ detail: 'Reload the controller to use paginated job details.' }, { status: 409 }); + } + const query = new URL(request.url).searchParams; + const section = query.get('section'); + return adminJson(section + ? await jobDetailPage(db, params.id, section, { limit: query.get('limit') ?? undefined, cursor: query.get('cursor'), stored: query.get('stored'), kind: query.get('kind') }) + : await jobDetail(db, params.id)); +}, { name: 'job' }); diff --git a/frontend/components/admin/ContentTreeView.js b/frontend/components/admin/ContentTreeView.js index 15f1094..4bd9f07 100644 --- a/frontend/components/admin/ContentTreeView.js +++ b/frontend/components/admin/ContentTreeView.js @@ -79,7 +79,7 @@ export default function ContentTreeView({ tree }) {
{counts.total}{uiText(" part(s):")} - {['DONE', 'PENDING', 'RUNNING', 'SKIPPED', 'FAILED'].map((s) => + {['DONE', 'PENDING', 'RUNNING', 'PROCESSING', 'SKIPPED', 'FAILED'].map((s) => counts[s] ? ( {counts[s]} {s.toLowerCase()} diff --git a/frontend/components/admin/JobDetail.js b/frontend/components/admin/JobDetail.js index efb11a8..22ebc58 100644 --- a/frontend/components/admin/JobDetail.js +++ b/frontend/components/admin/JobDetail.js @@ -26,6 +26,23 @@ function Section({ title, children }) { ); } +export function DetailPageControl({ section, vm }) { + const { uiText, locale } = useI18n(); + const meta = vm.detail?.pages?.[section]; + const state = vm.pageState?.[section]; + const loaded = vm.detail?.[section]?.length || 0; + if (!meta) return null; + return ( +
+

+ {uiText("Loaded: {v0} · Total: {v1}", { v0: loaded.toLocaleString(locale), v1: meta.total.toLocaleString(locale) })} +

+ vm.loadMore(section)} /> + {meta.next ?
: null} +
+ ); +} + /** Side panel (full screen on phones) with one job's details and actions. */ export default function JobDetail({ jobId, onClose, handleError, notify, onChanged }) { const { uiText , locale } = useI18n(); @@ -34,7 +51,7 @@ export default function JobDetail({ jobId, onClose, handleError, notify, onChang const panel = useRef(null); const job = vm.detail?.job; const files = vm.detail?.files || []; - const stored = files.filter((f) => f.storage_status === 'ACTIVE'); + const storedTotal = vm.detail?.pages?.files?.active_total ?? 0; const filesGone = Boolean(job?.files_deleted_at); useEffect(() => { @@ -62,6 +79,7 @@ export default function JobDetail({ jobId, onClose, handleError, notify, onChang
{uiText("Job ")}{jobId}
{job?.original_filename || (job ? uiText("(name removed after deletion)") : uiText("Loading…"))}
+ @@ -69,6 +87,7 @@ export default function JobDetail({ jobId, onClose, handleError, notify, onChang
+ {job ?

{uiText("Totals can change while a job runs. Refresh to update all sections.")}

: null} {job ? ( <> @@ -128,7 +147,7 @@ export default function JobDetail({ jobId, onClose, handleError, notify, onChang
-
+
{files.length ? (
    {files.map((f) => { @@ -153,7 +172,8 @@ export default function JobDetail({ jobId, onClose, handleError, notify, onChang ) : (

    {uiText("No files registered.")}

    )} - {stored.length > 1 ? ( + + {storedTotal > 1 ? (
    @@ -183,10 +203,12 @@ export default function JobDetail({ jobId, onClose, handleError, notify, onChang
+

{uiText("Tree and status counts show loaded records only.")}

+
-
+
@@ -215,8 +237,8 @@ export default function JobDetail({ jobId, onClose, handleError, notify, onChang
+
-
    {vm.detail.events.map((e) => ( @@ -228,6 +250,7 @@ export default function JobDetail({ jobId, onClose, handleError, notify, onChang ))}
+
) : vm.loading ? ( diff --git a/frontend/lib/i18n/messages/de.json b/frontend/lib/i18n/messages/de.json index f8a636c..0721cde 100644 --- a/frontend/lib/i18n/messages/de.json +++ b/frontend/lib/i18n/messages/de.json @@ -629,5 +629,11 @@ "🔒 Privacy Policy": "🔒 Datenschutzerklärung", "Large document: formatted preview is paused to keep editing responsive. The complete Markdown is available in Raw mode and exports.": "Großes Dokument: Die formatierte Vorschau ist pausiert, damit die Bearbeitung flüssig bleibt. Das vollständige Markdown ist im Modus „Quelltext“ und in Exporten verfügbar.", "Render full preview (may take a moment)": "Vollständige Vorschau rendern (kann einen Moment dauern)", - "Pause formatted preview": "Formatierte Vorschau pausieren" + "Pause formatted preview": "Formatierte Vorschau pausieren", + "Loaded: {v0} · Total: {v1}": "Geladen: {v0} · Gesamt: {v1}", + "Totals can change while a job runs. Refresh to update all sections.": "Die Gesamtzahlen können sich während der Verarbeitung ändern. Aktualisieren Sie alle Abschnitte über „Aktualisieren“.", + "Tree and status counts show loaded records only.": "Baum und Statuszahlen zeigen nur geladene Einträge.", + "Stored files changed. Refresh and try again.": "Die gespeicherten Dateien haben sich geändert. Aktualisieren Sie die Ansicht und versuchen Sie es erneut.", + "Could not load the complete file list": "Die vollständige Dateiliste konnte nicht geladen werden", + "Reload the controller to use paginated job details.": "Laden Sie den Controller neu, um Auftragsdetails seitenweise anzuzeigen." } diff --git a/frontend/lib/i18n/messages/en-US.json b/frontend/lib/i18n/messages/en-US.json index cac216e..8e3eb7a 100644 --- a/frontend/lib/i18n/messages/en-US.json +++ b/frontend/lib/i18n/messages/en-US.json @@ -629,5 +629,11 @@ "🔒 Privacy Policy": "🔒 Privacy Policy", "Large document: formatted preview is paused to keep editing responsive. The complete Markdown is available in Raw mode and exports.": "Large document: formatted preview is paused to keep editing responsive. The complete Markdown is available in Raw mode and exports.", "Render full preview (may take a moment)": "Render full preview (may take a moment)", - "Pause formatted preview": "Pause formatted preview" + "Pause formatted preview": "Pause formatted preview", + "Loaded: {v0} · Total: {v1}": "Loaded: {v0} · Total: {v1}", + "Totals can change while a job runs. Refresh to update all sections.": "Totals can change while a job runs. Refresh to update all sections.", + "Tree and status counts show loaded records only.": "Tree and status counts show loaded records only.", + "Stored files changed. Refresh and try again.": "Stored files changed. Refresh and try again.", + "Could not load the complete file list": "Could not load the complete file list", + "Reload the controller to use paginated job details.": "Reload the controller to use paginated job details." } diff --git a/frontend/lib/i18n/messages/es.json b/frontend/lib/i18n/messages/es.json index 72ed828..196b9e3 100644 --- a/frontend/lib/i18n/messages/es.json +++ b/frontend/lib/i18n/messages/es.json @@ -629,5 +629,11 @@ "🔒 Privacy Policy": "🔒 Política de privacidad", "Large document: formatted preview is paused to keep editing responsive. The complete Markdown is available in Raw mode and exports.": "Documento grande: la vista previa con formato está en pausa para mantener la edición fluida. El Markdown completo está disponible en el modo Código y en las exportaciones.", "Render full preview (may take a moment)": "Mostrar la vista previa completa (puede tardar un momento)", - "Pause formatted preview": "Pausar la vista previa con formato" + "Pause formatted preview": "Pausar la vista previa con formato", + "Loaded: {v0} · Total: {v1}": "Cargados: {v0} · Total: {v1}", + "Totals can change while a job runs. Refresh to update all sections.": "Los totales pueden cambiar mientras se ejecuta un trabajo. Actualiza para renovar todas las secciones.", + "Tree and status counts show loaded records only.": "El árbol y los recuentos por estado solo muestran los registros cargados.", + "Stored files changed. Refresh and try again.": "Los archivos almacenados han cambiado. Actualiza e inténtalo de nuevo.", + "Could not load the complete file list": "No se pudo cargar la lista completa de archivos", + "Reload the controller to use paginated job details.": "Vuelve a cargar el controlador para ver los detalles del trabajo por páginas." } diff --git a/frontend/lib/i18n/messages/ja.json b/frontend/lib/i18n/messages/ja.json index 893dc25..ea5c41f 100644 --- a/frontend/lib/i18n/messages/ja.json +++ b/frontend/lib/i18n/messages/ja.json @@ -629,5 +629,11 @@ "🔒 Privacy Policy": "🔒 プライバシーポリシー", "Large document: formatted preview is paused to keep editing responsive. The complete Markdown is available in Raw mode and exports.": "大きなドキュメントのため、快適に編集できるよう整形プレビューを一時停止しています。Markdown 全文はソースモードとエクスポートで利用できます。", "Render full preview (may take a moment)": "全文のプレビューを表示（時間がかかる場合があります）", - "Pause formatted preview": "整形プレビューを一時停止" + "Pause formatted preview": "整形プレビューを一時停止", + "Loaded: {v0} · Total: {v1}": "読み込み済み: {v0} · 合計: {v1}", + "Totals can change while a job runs. Refresh to update all sections.": "ジョブの実行中は合計が変わることがあります。更新してすべてのセクションを最新にしてください。", + "Tree and status counts show loaded records only.": "ツリーとステータス件数には読み込み済みのレコードのみが表示されます。", + "Stored files changed. Refresh and try again.": "保存済みファイルが変更されました。更新してもう一度お試しください。", + "Could not load the complete file list": "完全なファイル一覧を読み込めませんでした", + "Reload the controller to use paginated job details.": "ジョブの詳細をページごとに表示するには、コントローラーを再読み込みしてください。" } diff --git a/frontend/lib/i18n/messages/ru.json b/frontend/lib/i18n/messages/ru.json index 8722165..8d337a6 100644 --- a/frontend/lib/i18n/messages/ru.json +++ b/frontend/lib/i18n/messages/ru.json @@ -629,5 +629,11 @@ "🔒 Privacy Policy": "🔒 Политика конфиденциальности", "Large document: formatted preview is paused to keep editing responsive. The complete Markdown is available in Raw mode and exports.": "Большой документ: предпросмотр с форматированием приостановлен, чтобы редактирование оставалось отзывчивым. Полный Markdown доступен в режиме «Исходный» и при экспорте.", "Render full preview (may take a moment)": "Показать полный предпросмотр (может занять некоторое время)", - "Pause formatted preview": "Приостановить предпросмотр с форматированием" + "Pause formatted preview": "Приостановить предпросмотр с форматированием", + "Loaded: {v0} · Total: {v1}": "Загружено: {v0} · Всего: {v1}", + "Totals can change while a job runs. Refresh to update all sections.": "Итоговые значения могут меняться во время выполнения задания. Обновите все разделы.", + "Tree and status counts show loaded records only.": "Дерево и счётчики статусов показывают только загруженные записи.", + "Stored files changed. Refresh and try again.": "Сохранённые файлы изменились. Обновите страницу и повторите попытку.", + "Could not load the complete file list": "Не удалось загрузить полный список файлов", + "Reload the controller to use paginated job details.": "Перезагрузите панель управления для постраничного просмотра задания." } diff --git a/frontend/lib/i18n/messages/zh-CN.json b/frontend/lib/i18n/messages/zh-CN.json index 6f419a3..1b8cd47 100644 --- a/frontend/lib/i18n/messages/zh-CN.json +++ b/frontend/lib/i18n/messages/zh-CN.json @@ -629,5 +629,11 @@ "🔒 Privacy Policy": "🔒 隐私政策", "Large document: formatted preview is paused to keep editing responsive. The complete Markdown is available in Raw mode and exports.": "文档较大：为保持编辑流畅，已暂停格式化预览。完整 Markdown 可在源码模式和导出文件中查看。", "Render full preview (may take a moment)": "显示完整预览（可能需要一些时间）", - "Pause formatted preview": "暂停格式化预览" + "Pause formatted preview": "暂停格式化预览", + "Loaded: {v0} · Total: {v1}": "已加载：{v0} · 总计：{v1}", + "Totals can change while a job runs. Refresh to update all sections.": "任务运行时总数可能发生变化。请刷新以更新所有部分。", + "Tree and status counts show loaded records only.": "目录树和状态计数仅显示已加载的记录。", + "Stored files changed. Refresh and try again.": "已存储的文件发生变化。请刷新后重试。", + "Could not load the complete file list": "无法加载完整的文件列表", + "Reload the controller to use paginated job details.": "请重新加载控制台以分页查看任务详情。" } diff --git a/frontend/lib/models/adminApi.js b/frontend/lib/models/adminApi.js index ab4c349..dffb053 100644 --- a/frontend/lib/models/adminApi.js +++ b/frontend/lib/models/adminApi.js @@ -11,7 +11,7 @@ export class AdminApiError extends Error { } } -async function call(path, { method = 'GET', body, signal } = {}) { +async function call(path, { method = 'GET', body, signal, headers = {} } = {}) { let res; try { res = await fetch(`/api/admin${path}`, { @@ -20,6 +20,7 @@ async function call(path, { method = 'GET', body, signal } = {}) { cache: 'no-store', signal, headers: { + ...headers, ...(method === 'GET' ? {} : { 'x-mdify-admin': '1' }), ...(body === undefined ? {} : { 'Content-Type': 'application/json' }), }, @@ -47,7 +48,8 @@ export const adminApi = { logout: () => call('/session', { method: 'DELETE' }), overview: () => call('/overview'), jobs: (params, signal) => call(`/jobs${query(params)}`, { signal }), - job: (jobId, signal) => call(`/jobs/${encodeURIComponent(jobId)}`, { signal }), + job: (jobId, signal) => call(`/jobs/${encodeURIComponent(jobId)}`, { signal, headers: { 'x-llmiser-detail-paging': '1' } }), + jobPage: (jobId, section, params, signal) => call(`/jobs/${encodeURIComponent(jobId)}${query({ section, ...params })}`, { signal, headers: { 'x-llmiser-detail-paging': '1' } }), action: (jobId, action, hours) => call(`/jobs/${encodeURIComponent(jobId)}/action`, { method: 'POST', body: { action, hours } }), bulk: (jobIds, action, hours) => call('/bulk', { method: 'POST', body: { job_ids: jobIds, action, hours } }), signFile: (jobId, fileId, purpose = 'download') => diff --git a/frontend/lib/models/adminDetail.js b/frontend/lib/models/adminDetail.js new file mode 100644 index 0000000..a9d2e9b --- /dev/null +++ b/frontend/lib/models/adminDetail.js @@ -0,0 +1,45 @@ +// Page merging/export collection are independent of React and testable with delayed requests. +export const DETAIL_SECTIONS = ['files', 'nodes', 'items', 'events']; +export const DETAIL_IDS = { files: 'file_id', nodes: 'node_id', items: 'work_item_id', events: 'event_id' }; + +export function mergeDetailPage(detail, section, page) { + if (!DETAIL_SECTIONS.includes(section) || page.section !== section) throw new Error('Invalid detail page'); + const records = new Map((detail[section] || []).map(row => [row[DETAIL_IDS[section]], row])); + for (const row of page.rows) records.set(row[DETAIL_IDS[section]], row); + const { rows, section: _section, ...metadata } = page; + return { ...detail, [section]: [...records.values()], pages: { ...detail.pages, [section]: metadata } }; +} + +/** Resolve every active registered file before signing/downloading a ZIP. */ +export async function collectStoredFiles(jobIds, { loadPage, outputsOnly = false, maxFiles = 200, signal } = {}) { + const files = []; + const ids = [...new Set(jobIds)]; + for (const jobId of ids) { + let cursor = null; + const seenCursors = new Set(); + const seenFiles = new Set(); + const priorFiles = files.length; + let expectedTotal = null; + do { + signal?.throwIfAborted(); + const page = await loadPage(jobId, 'files', { limit: 100, cursor, stored: '1', ...(outputsOnly ? { kind: 'OUTPUT' } : {}) }, signal); + signal?.throwIfAborted(); + if (page.section !== 'files' || !Array.isArray(page.rows) || !Number.isSafeInteger(page.total) || page.total < page.rows.length) throw new Error('Could not load the complete file list'); + if (expectedTotal !== null && page.total !== expectedTotal) throw new Error('Stored files changed. Refresh and try again.'); + expectedTotal = page.total; + if (page.total + priorFiles > maxFiles) throw new Error(`At most ${maxFiles} files per ZIP`); + for (const file of page.rows) { + if (seenFiles.has(file.file_id)) throw new Error('Stored files changed. Refresh and try again.'); + seenFiles.add(file.file_id); + if (file.storage_status !== 'ACTIVE' || (outputsOnly && file.kind !== 'OUTPUT')) throw new Error('Stored files changed. Refresh and try again.'); + files.push({ jobId, fileId: file.file_id, name: `${ids.length > 1 ? jobId.slice(0, 8) : file.kind.toLowerCase()}/${file.object_path.split('/').pop()}` }); + if (files.length > maxFiles) throw new Error(`At most ${maxFiles} files per ZIP`); + } + if (page.next && (seenCursors.has(page.next) || !page.rows.length)) throw new Error('Could not load the complete file list'); + if (page.next) seenCursors.add(page.next); + cursor = page.next; + } while (cursor); + if (seenFiles.size !== expectedTotal) throw new Error('Could not load the complete file list'); + } + return files; +} diff --git a/frontend/lib/models/contentTree.js b/frontend/lib/models/contentTree.js index 5c5fca5..ebe243d 100644 --- a/frontend/lib/models/contentTree.js +++ b/frontend/lib/models/contentTree.js @@ -8,7 +8,7 @@ // After a job's files are deleted, paths are cleared in the database; nodes // then show their type and position instead of a name. -const STATUS_ORDER = ['FAILED', 'SKIPPED', 'PENDING', 'RUNNING', 'DONE']; +const STATUS_ORDER = ['FAILED', 'SKIPPED', 'PENDING', 'RUNNING', 'PROCESSING', 'DONE']; function folder(name, path) { return { key: `dir:${path}`, name, path, node: null, children: new Map() }; @@ -54,7 +54,8 @@ export function buildContentTree(nodes = [], job = {}) { root.node = rootNode; const paged = rest.filter((n) => n.node_type === 'PDF_PAGE' || n.node_type === 'PDF_SEGMENT'); - const others = rest.filter((n) => !paged.includes(n)); + const pageSet = new Set(paged); + const others = rest.filter((n) => !pageSet.has(n)); for (const n of [...paged].sort((a, b) => (a.page_from ?? 0) - (b.page_from ?? 0))) { root.children.set(`page:${n.node_id}`, { diff --git a/frontend/lib/server/adminService.js b/frontend/lib/server/adminService.js index 3619eb4..b132e28 100644 --- a/frontend/lib/server/adminService.js +++ b/frontend/lib/server/adminService.js @@ -84,18 +84,75 @@ export async function listJobs(db, { limit = 50, before = null, beforeId = null, return { jobs, next: jobs.length === n && last ? { before: last.created_at, before_id: last.job_id } : null }; } -export async function jobDetail(db, jobId) { +export const DETAIL_PAGE_SIZE = 50; +export const DETAIL_MAX_PAGE_SIZE = 100; +const DETAIL_SECTIONS = { + files: { table: 'file_objects', id: 'file_id', key: 'created_at', select: 'file_id,kind,object_path,extension,mime_type,size_bytes,storage_status,created_at,deleted_at' }, + nodes: { table: 'content_nodes', id: 'node_id', key: 'sequence_index', select: 'node_id,parent_node_id,node_type,classification,logical_path,page_from,page_to,sequence_index,archive_depth,size_bytes,status,skip_reason,engine,backend_instance,duration_ms,created_at,completed_at' }, + items: { table: 'work_items', id: 'work_item_id', key: 'created_at', select: 'work_item_id,node_id,task_type,pool,status,is_root,is_final,sequence_index,attempt_count,max_attempts,assigned_backend,error_code,error_message,duration_ms,created_at,claimed_at,completed_at' }, + events: { table: 'job_events', id: 'event_id', key: 'created_at', select: 'event_id,event_type,status,stage,backend_role,backend_instance,message,created_at' }, +}; +const JOB_DETAIL_FIELDS = 'job_id,original_filename,source_type,source_extension,profile,status,stage,progress,created_at,completed_at,processing_ms,backend_instance,engine,items_total,items_done,items_failed,items_skipped,error_code,error_message,warnings,files_deleted_at,retention_mode,retention_extended_until,auto_delete_at,cleanup_state,cleanup_last_error'; + +function detailLimit(limit = DETAIL_PAGE_SIZE) { + if (!/^\d{1,3}$/.test(String(limit)) || Number(limit) < 1) throw new AdminError('Invalid page size'); + return Math.min(Number(limit), DETAIL_MAX_PAGE_SIZE); +} + +function pageCursor(cursor, jobId, section, filter) { + if (cursor === null || cursor === undefined) return null; + if (typeof cursor !== 'string' || !/^[\w-]{1,512}$/.test(cursor)) throw new AdminError('Bad cursor'); + let value; + try { value = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8')); } catch { throw new AdminError('Bad cursor'); } + if (value?.v !== 1 || value.job !== jobId || value.section !== section || value.filter !== filter || !UUID.test(String(value.id || ''))) throw new AdminError('Bad cursor'); + if (section === 'nodes') { + if (!Number.isInteger(value.key) || value.key < -2147483648 || value.key > 2147483647) throw new AdminError('Bad cursor'); + } else if (typeof value.key !== 'string' + || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?(?:Z|[+-]\d{2}:\d{2})$/.test(value.key) + || !Number.isFinite(Date.parse(value.key))) throw new AdminError('Bad cursor'); + return value; +} + +function encodePageCursor(job, section, row, config, filter) { + // Preserve PostgreSQL microseconds; Date.toISOString would round them away. + return Buffer.from(JSON.stringify({ v: 1, job, section, filter, key: row[config.key], id: row[config.id] })).toString('base64url'); +} + +export async function jobDetailPage(db, jobId, section, { limit, cursor = null, stored = null, kind = null, jobKnown = false } = {}) { const id = assertJobId(jobId); - const [tree, files] = await Promise.all([ - db.rpc('get_job_tree', { p_job_id: id }), - db.select('file_objects', { - job_id: `eq.${id}`, - select: 'file_id,kind,object_path,extension,mime_type,size_bytes,storage_status,created_at,deleted_at', - order: 'created_at.asc', - }), + const config = DETAIL_SECTIONS[section]; + if (!Object.hasOwn(DETAIL_SECTIONS, section)) throw new AdminError('Unknown detail section'); + const n = detailLimit(limit); + if (stored !== null && stored !== '1') throw new AdminError('Unknown file filter'); + if (kind !== null && !['INPUT', 'OUTPUT', 'EXPORT', 'PREVIEW'].includes(kind)) throw new AdminError('Unknown file filter'); + if (section !== 'files' && (stored !== null || kind !== null)) throw new AdminError('Unknown file filter'); + const filter = `${stored || ''}:${kind || ''}`; + const after = pageCursor(cursor, id, section, filter); + if (!jobKnown && !(await db.select('jobs', { job_id: `eq.${id}`, select: 'job_id', limit: '1' }))?.length) throw new AdminError('Unknown job', 404); + const base = { job_id: `eq.${id}`, ...(stored ? { storage_status: 'eq.ACTIVE' } : {}), ...(kind ? { kind: `eq.${kind}` } : {}) }; + const filters = { ...base, select: config.select, order: `${config.key}.asc,${config.id}.asc`, limit: String(n) }; + if (after) filters.or = `(${config.key}.gt.${after.key},and(${config.key}.eq.${after.key},${config.id}.gt.${after.id}))`; + const [page, total, active] = await Promise.all([ + db.selectPage(config.table, filters), + after ? db.count(config.table, { ...base, select: config.id, limit: '1' }) : null, + section === 'files' ? db.count('file_objects', { job_id: `eq.${id}`, storage_status: 'eq.ACTIVE', select: 'file_id', limit: '1' }) : null, ]); - if (!tree?.job) throw new AdminError('Unknown job', 404); - return { ...tree, files: files || [] }; + if (page.rows.length > n || !Number.isSafeInteger(page.total) || page.total < page.rows.length) throw new AdminError('Invalid page response', 502); + const last = page.rows.at(-1); + return { section, rows: page.rows, total: total ?? page.total, + next: last && page.total > page.rows.length ? encodePageCursor(id, section, last, config, filter) : null, + ...(section === 'files' ? { active_total: active } : {}), + }; +} + +export async function jobDetail(db, jobId) { + const id = assertJobId(jobId); + const jobs = await db.select('jobs', { job_id: `eq.${id}`, select: JOB_DETAIL_FIELDS, limit: '1' }); + if (!jobs?.[0]) throw new AdminError('Unknown job', 404); + const sections = Object.keys(DETAIL_SECTIONS); + const pages = await Promise.all(sections.map(section => jobDetailPage(db, id, section, { jobKnown: true }))); + return { job: jobs[0], ...Object.fromEntries(pages.map(page => [page.section, page.rows])), + pages: Object.fromEntries(pages.map(({ rows, section, ...page }) => [section, page])) }; } const FILE_STATUSES = new Set(['PENDING', 'ACTIVE', 'DELETE_PENDING', 'DELETED', 'ERROR']); diff --git a/frontend/lib/server/supabaseRest.js b/frontend/lib/server/supabaseRest.js index a7635bb..bdb882f 100644 --- a/frontend/lib/server/supabaseRest.js +++ b/frontend/lib/server/supabaseRest.js @@ -29,7 +29,7 @@ export function createSupabase(config, fetchImpl = fetch, { deadlineMs } = {}) { Authorization: `Bearer ${config.key}`, }; - async function call(path, init = {}) { + async function call(path, init = {}, { count = false } = {}) { const res = await fetchImpl(`${config.url}${path}`, { ...init, headers: { ...headers, ...(init.headers || {}) }, @@ -48,6 +48,14 @@ export function createSupabase(config, fetchImpl = fetch, { deadlineMs } = {}) { const msg = (data && (data.message || data.error)) || `HTTP ${res.status}`; throw new SupabaseError(`Supabase ${init.method || 'GET'} ${path.split('?')[0]} failed: ${msg}`, res.status); } + if (count) { + const match = /\/(\d+)$/.exec(res.headers.get('content-range') || ''); + const total = match ? Number(match[1]) : NaN; + if (!Number.isSafeInteger(total) || (init.method === 'HEAD' ? data !== null : !Array.isArray(data))) { + throw new SupabaseError('Database did not return an exact page count', 502); + } + return { rows: data || [], total }; + } return data; } @@ -77,6 +85,12 @@ export function createSupabase(config, fetchImpl = fetch, { deadlineMs } = {}) { select: (table, filters) => call(`/rest/v1/${table}?${new URLSearchParams(filters)}`), + selectPage: (table, filters) => call(`/rest/v1/${table}?${new URLSearchParams(filters)}`, + { headers: { Prefer: 'count=exact' } }, { count: true }), + + count: (table, filters) => call(`/rest/v1/${table}?${new URLSearchParams(filters)}`, + { method: 'HEAD', headers: { Prefer: 'count=exact' } }, { count: true }).then(page => page.total), + /** DELETE rows matching `filters` (PostgREST syntax); returns the deleted rows. */ remove: (table, filters) => call(`/rest/v1/${table}?${new URLSearchParams(filters)}`, { method: 'DELETE', headers: { Prefer: 'return=representation' } }), diff --git a/frontend/package-lock.json b/frontend/package-lock.json index af0f2d1..6824081 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -23,6 +23,7 @@ "eslint": "^8.57.0", "eslint-config-next": "^14.2.35", "postcss": "^8.4.39", + "react-test-renderer": "18.3.1", "sharp": "^0.35.5", "tailwindcss": "^3.4.6" } @@ -6780,6 +6781,42 @@ "react": ">=18" } }, + "node_modules/react-shallow-renderer": { + "version": "16.15.0", + "resolved": "https://registry.npmjs.org/react-shallow-renderer/-/react-shallow-renderer-16.15.0.tgz", + "integrity": "sha512-oScf2FqQ9LFVQgA73vr86xl2NaOIX73rh+YFqcOp68CWj56tSfgtGKrEbyhCj0rSijyG9M1CYprTh39fBi5hzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "object-assign": "^4.1.1", + "react-is": "^16.12.0 || ^17.0.0 || ^18.0.0" + }, + "peerDependencies": { + "react": "^16.0.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/react-test-renderer": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-test-renderer/-/react-test-renderer-18.3.1.tgz", + "integrity": "sha512-KkAgygexHUkQqtvvx/otwxtuFu5cVjfzTCtjXLH9boS19/Nbtg84zS7wIQn39G8IlrhThBpQsMKkq5ZHZIYFXA==", + "dev": true, + "license": "MIT", + "dependencies": { + "react-is": "^18.3.1", + "react-shallow-renderer": "^16.15.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/react-test-renderer/node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "dev": true, + "license": "MIT" + }, "node_modules/read-cache": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index 8086fa0..9cbbb15 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -33,6 +33,7 @@ "eslint": "^8.57.0", "eslint-config-next": "^14.2.35", "postcss": "^8.4.39", + "react-test-renderer": "18.3.1", "sharp": "^0.35.5", "tailwindcss": "^3.4.6" } diff --git a/frontend/scripts/measure-controller-pagination.mjs b/frontend/scripts/measure-controller-pagination.mjs new file mode 100644 index 0000000..c784ee3 --- /dev/null +++ b/frontend/scripts/measure-controller-pagination.mjs @@ -0,0 +1,40 @@ +// Generated local PostgreSQL plans and response sizes; no network or production data. +import { freshDb } from '../test/sql/pgliteDb.mjs'; +import { createPgliteSupabase } from '../test/sql/pgliteSupabase.mjs'; +import { jobDetail,jobDetailPage } from '../lib/server/adminService.js'; + +const pg=await freshDb(); +try { + const jobs=(await pg.query("insert into jobs(status,original_filename) select 'COMPLETED','generated.zip' from generate_series(1,21) returning job_id")).rows.map(row=>row.job_id); + for(const [index,id] of jobs.entries()) { + const n=index===0?2000:1000; + await pg.query("insert into content_nodes(job_id,node_type,sequence_index,logical_path) select $1,'TEXT_FILE',i,'src/file-' || i || '.txt' from generate_series(1,$2) i",[id,n]); + await pg.query("insert into file_objects(job_id,bucket,object_path,kind,storage_status,created_at) select $1::uuid,'mdify-pro-files',$1::text || '/' || i || '.md','OUTPUT','ACTIVE',now()+i*interval '1 microsecond' from generate_series(1,$2) i",[id,n]); + await pg.query("insert into work_items(job_id,task_type,pool,created_at) select $1,'CONVERT','normal',now()+i*interval '1 microsecond' from generate_series(1,$2) i",[id,n]); + await pg.query("insert into job_events(job_id,event_type,message,details,created_at) select $1,'GENERATED','Generated benchmark event',jsonb_build_object('generated',repeat('x',256)),now()+i*interval '1 microsecond' from generate_series(1,$2) i",[id,n]); + } + await pg.exec('analyze jobs; analyze file_objects; analyze content_nodes; analyze work_items; analyze job_events;'); + const db=createPgliteSupabase(pg),id=jobs[0]; + const old={...(await db.rpc('get_job_tree',{p_job_id:id})),files:await db.select('file_objects',{job_id:`eq.${id}`,select:'file_id,kind,object_path,extension,mime_type,size_bytes,storage_status,created_at,deleted_at',order:'created_at.asc'})}; + const initial=await jobDetail(db,id); + const plans={}; + for(const [section,table,key,pk] of [['files','file_objects','created_at','file_id'],['nodes','content_nodes','sequence_index','node_id'],['items','work_items','created_at','work_item_id'],['events','job_events','created_at','event_id']]) { + const row=(await pg.query(`select row_to_json(p) as value from(select ${key},${pk} from ${table} where job_id=$1 order by ${key},${pk} offset 1499 limit 1)p`,[id])).rows[0].value; + const explain=async(sql,params)=>(await pg.query('explain (analyze,buffers,format json) '+sql,params)).rows[0]['QUERY PLAN'][0]; + plans[section]={ + first:await explain(`select * from ${table} where job_id=$1 order by ${key},${pk} limit 50`,[id]), + deep:await explain(`select * from ${table} where job_id=$1 and (${key}>$2 or (${key}=$2 and ${pk}>$3)) order by ${key},${pk} limit 50`,[id,row[key],row[pk]]), + count:await explain(`select count(*) from ${table} where job_id=$1`,[id]), + }; + let cursor=null,seen=new Set(),pages=0; + do {const page=await jobDetailPage(db,id,section,{limit:100,cursor});for(const record of page.rows){if(seen.has(record[pk]))throw new Error('Duplicate');seen.add(record[pk]);}cursor=page.next;pages++;}while(cursor); + if(seen.size!==2000)throw new Error('Missing '+section); + plans[section].traversal={pages,rows:seen.size}; + } + const result={checkedAt:new Date().toISOString(),engine:(await pg.query('select version() as version')).rows[0].version, + fixture:{jobs:21,targetRowsPerSection:2000,totalRowsPerSection:22000}, + oldBytes:Buffer.byteLength(JSON.stringify(old)),initialBytes:Buffer.byteLength(JSON.stringify(initial)), + initialRows:Object.fromEntries(['files','nodes','items','events'].map(section=>[section,initial[section].length])),plans}; + result.reductionPercent=100*(1-result.initialBytes/result.oldBytes); + console.log(JSON.stringify(result,null,2)); +} finally {await pg.close();} diff --git a/frontend/test/adminDetail.test.mjs b/frontend/test/adminDetail.test.mjs new file mode 100644 index 0000000..b5231ed --- /dev/null +++ b/frontend/test/adminDetail.test.mjs @@ -0,0 +1,127 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { freshDb } from './sql/pgliteDb.mjs'; +import { createPgliteSupabase } from './sql/pgliteSupabase.mjs'; +import { jobDetail, jobDetailPage } from '../lib/server/adminService.js'; +import { collectStoredFiles, mergeDetailPage } from '../lib/models/adminDetail.js'; +import { createSupabase } from '../lib/server/supabaseRest.js'; + +test('detail pages on migrated Postgres retain tied and microsecond keys, scope and complete exports', async t => { + const pg = await freshDb(); + t.after(() => pg.close()); + const db = createPgliteSupabase(pg); + const job = async () => (await pg.query("insert into jobs(status,original_filename) values ('COMPLETED','generated.zip') returning job_id")).rows[0].job_id; + const id = await job(), other = await job(), empty = await job(); + for (const jobId of [id, other]) { + const args = [jobId]; + const stamp = "'2026-10-29T00:00:00Z'::timestamptz + ((i / 3) * interval '1 microsecond')"; + await pg.query(`insert into file_objects(job_id,bucket,object_path,kind,storage_status,created_at) + select $1::uuid,'mdify-pro-files',$1::text || '/' || i || '.md',case when i%2=0 then 'OUTPUT' else 'INPUT' end, + case when i%5=0 then 'DELETED' else 'ACTIVE' end,${stamp} from generate_series(1,123) i`, args); + await pg.query(`insert into content_nodes(job_id,node_type,sequence_index,logical_path) + select $1,'TEXT_FILE',i/3,'part-' || i from generate_series(1,123) i`, args); + await pg.query(`insert into work_items(job_id,task_type,pool,created_at,payload) + select $1,'CONVERT','normal',${stamp},jsonb_build_object('private','omitted') from generate_series(1,123) i`, args); + await pg.query(`insert into job_events(job_id,event_type,created_at,details) + select $1,'GENERATED',${stamp},jsonb_build_object('private','omitted') from generate_series(1,123) i`, args); + } + const detail = await jobDetail(db, id); + const names = {files:['file_objects','file_id','created_at'],nodes:['content_nodes','node_id','sequence_index'],items:['work_items','work_item_id','created_at'],events:['job_events','event_id','created_at']}; + for (const [section,[table,key,order]] of Object.entries(names)) { + assert.equal(detail[section].length,50); + assert.equal(detail.pages[section].total,123); + assert.ok(detail.pages[section].next); + let result = detail; + const lengths = [50]; + while (result.pages[section].next) { + const page = await jobDetailPage(db,id,section,{cursor:result.pages[section].next}); + lengths.push(page.rows.length); + result = mergeDetailPage(result,section,page); + } + assert.deepEqual(lengths,[50,50,23]); + const expected = (await pg.query(`select ${key} from ${table} where job_id=$1 order by ${order},${key}`,[id])).rows.map(row=>row[key]); + assert.deepEqual(result[section].map(row=>row[key]),expected,section); + assert.ok(!('payload' in result[section][0])); + assert.ok(!('details' in result[section][0])); + assert.equal((await jobDetailPage(db,id,section,{limit:'999'})).rows.length,100); + const short = await jobDetailPage({...db,selectPage:(table,filters)=>db.selectPage(table,{...filters,limit:'2'})},id,section,{limit:50}); + assert.equal(short.rows.length,2); + assert.ok(short.next,'a server row cap must not silently end traversal'); + } + assert.equal(detail.pages.files.active_total,99); + assert.ok(!db.calls.some(call=>call[1]==='get_job_tree')); + const filtered = await jobDetailPage(db,id,'files',{limit:1,stored:'1',kind:'OUTPUT'}); + const exported = await collectStoredFiles([id,other],{outputsOnly:true,loadPage:(jobId,section,params)=>jobDetailPage(db,jobId,section,{...params,limit:7})}); + assert.equal(exported.length,98); + assert.equal(new Set(exported.map(file=>file.fileId)).size,98); + await assert.rejects(jobDetailPage(db,other,'files',{cursor:filtered.next,stored:'1',kind:'OUTPUT'}),/Bad cursor/); + await assert.rejects(jobDetailPage(db,id,'files',{cursor:filtered.next,stored:'1'}),/Bad cursor/); + await assert.rejects(jobDetailPage(db,id,'events',{cursor:filtered.next}),/Bad cursor/); + for (const limit of [0,-1,'NaN','1.2','1000','1 or true']) await assert.rejects(jobDetailPage(db,id,'files',{limit}),/page size/); + for (const cursor of ['', '!', 'e30', 'a'.repeat(513)]) await assert.rejects(jobDetailPage(db,id,'files',{cursor}),/Bad cursor/); + await assert.rejects(jobDetailPage(db,id,'__proto__'),/Unknown detail section/); + await assert.rejects(jobDetailPage(db,id,'nodes',{stored:'1'}),/file filter/); + await assert.rejects(jobDetailPage(db,id,'files',{kind:'secret'}),/file filter/); + await assert.rejects(jobDetail(db,crypto.randomUUID()),error=>error.status===404); + const blank = await jobDetail(db,empty); + for (const section of Object.keys(names)) assert.deepEqual(blank.pages[section],{total:0,next:null,...(section==='files'?{active_total:0}:{})}); +}); + +test('PostgREST exact page counts use GET/HEAD, preserve query keys and fail closed',async()=>{ + const calls=[]; + let body='[{"event_id":"x"}]',range='0-0/123',status=200; + const db=createSupabase({url:'https://database.test',key:'test',bucket:'test'},async(url,init)=>{ + calls.push({url:new URL(url),init}); + return new Response(init.method==='HEAD'?null:body,{status,headers:{'content-range':range}}); + }); + const or='(created_at.gt.2026-10-29T00:00:00.000001+00:00,and(created_at.eq.2026-10-29T00:00:00.000001+00:00,event_id.gt.abc))'; + assert.deepEqual(await db.selectPage('job_events',{or,limit:50,order:'created_at.asc,event_id.asc'}),{rows:[{event_id:'x'}],total:123}); + assert.equal(calls[0].url.searchParams.get('or'),or); + assert.equal(calls[0].init.headers.Prefer,'count=exact'); + assert.equal(calls[0].init.cache,'no-store'); + assert.equal(await db.count('job_events',{limit:1}),123); + assert.equal(calls[1].init.method,'HEAD'); + range='*/0'; body='[]'; + assert.deepEqual(await db.selectPage('job_events',{}),{rows:[],total:0}); + for (const value of ['', '0-0/*','0-0/9007199254740992']) { + range=value; await assert.rejects(db.selectPage('job_events',{}),/exact page count/); + } + range='0-0/1'; + for (const value of ['null','{}','not JSON']) {body=value;await assert.rejects(db.selectPage('job_events',{}),/exact page count/);} + status=401; body='{"message":"denied"}'; await assert.rejects(db.selectPage('job_events',{}),error=>error.status===401); +}); + +const file = i=>({file_id:String(i),kind:'OUTPUT',storage_status:'ACTIVE',object_path:`generated/${i}.md`}); +const page = (rows,total=rows.length,next=null)=>({section:'files',rows,total,next}); +test('ZIP collection resolves all pages and refuses incomplete, changed, oversized or aborted lists',async()=>{ + let requests=0; + const all=await collectStoredFiles(['a'],{loadPage:async(_job,_section,params)=>{ + requests++; assert.equal(params.stored,'1'); + return params.cursor?page([file(2)],2):page([file(1)],2,'next'); + }}); + assert.deepEqual(all.map(item=>item.fileId),['1','2']); + assert.equal(requests,2); + await assert.rejects(collectStoredFiles(['a'],{loadPage:async()=>page([file(1)],201)}),/At most 200/); + await assert.rejects(collectStoredFiles(['a','b'],{maxFiles:1,loadPage:async()=>page([file(1)])}),/At most 1/); + for(const loadPage of [ + async(_j,_s,p)=>p.cursor?Promise.reject(new Error('offline')):page([file(1)],2,'next'), + async()=>page([file(1)],2,'next'), + async(_j,_s,p)=>p.cursor?page([],2):page([file(1)],2,'next'), + async(_j,_s,p)=>p.cursor?page([file(2)],3):page([file(1)],2,'next'), + async()=>page([{...file(1),storage_status:'DELETED'}]), + async()=>({rows:[],total:0,next:null}), + ]) await assert.rejects(collectStoredFiles(['a'],{loadPage})); + await assert.rejects(collectStoredFiles(['a'],{outputsOnly:true,loadPage:async()=>page([{...file(1),kind:'INPUT'}])}),/Stored files changed/); + const controller=new AbortController(); + await assert.rejects(collectStoredFiles(['a'],{signal:controller.signal,loadPage:async()=>{controller.abort();return page([file(1)]);}}),error=>error.name==='AbortError'); +}); + +test('merging a page deduplicates IDs while retaining other sections and updated metadata',()=>{ + const initial={files:[file(1)],events:[{event_id:'unchanged'}],pages:{files:{total:3,next:'a'},events:{total:1,next:null}}}; + const result=mergeDetailPage(initial,'files',page([{...file(1),size_bytes:42},file(2)],3,'b')); + assert.equal(result.files.length,2); + assert.equal(result.files[0].size_bytes,42); + assert.equal(result.events,initial.events); + assert.equal(result.pages.files.next,'b'); + assert.equal(initial.files.length,1); +}); diff --git a/frontend/test/adminDetailHooks.test.mjs b/frontend/test/adminDetailHooks.test.mjs new file mode 100644 index 0000000..ed34924 --- /dev/null +++ b/frontend/test/adminDetailHooks.test.mjs @@ -0,0 +1,91 @@ +import { test, after } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import React from 'react'; +import { create, act } from 'react-test-renderer'; +import { renderToStaticMarkup } from 'react-dom/server'; +import { adminApi } from '../lib/models/adminApi.js'; +import { readDictionaries } from '../scripts/check-i18n.mjs'; + +const root = path.dirname(fileURLToPath(import.meta.url)); +const directory = fs.mkdtempSync(path.join(root,'.locale-runtime-')); +after(()=>{assert.equal(path.dirname(directory),root);fs.rmSync(directory,{recursive:true,force:true});}); +const require = createRequire(import.meta.url); +const { transform } = require('next/dist/build/swc'); +const sources = ['components/i18n/I18nContext','viewmodels/useAdminViewModel','components/admin/JobDetail','components/admin/ContentTreeView','components/admin/ui']; +const targets = new Map(sources.map((source,i)=>[fileURLToPath(new URL(`../${source}.js`,import.meta.url)),path.join(directory,`${i}.mjs`)])); +for(const [input,output] of targets) { + const source=fs.readFileSync(input,'utf8').replace(/from\s*(['"])(\.[^'"]+)\1/g,(_match,_quote,specifier)=>{ + const resolved=path.resolve(path.dirname(input),specifier.endsWith('.js')?specifier:`${specifier}.js`); + return `from ${JSON.stringify(pathToFileURL(targets.get(resolved)||resolved).href)}`; + }); + const compiled=await transform(source,{filename:input,jsc:{parser:{syntax:'ecmascript',jsx:true},transform:{react:{runtime:'automatic'}}},module:{type:'es6'}}); + fs.writeFileSync(output,compiled.code); +} +const [contextModule,hookModule,detailModule]=await Promise.all([0,1,2].map(i=>import(pathToFileURL(path.join(directory,`${i}.mjs`))))); +const { I18nContext }=contextModule; +const { useAdminJobDetailViewModel }=hookModule; +const { DetailPageControl }=detailModule; +const noop=()=>{}, options={handleError:()=>false,notify:noop,onChanged:noop}; +const deferred=()=>{let resolve,reject;const promise=new Promise((yes,no)=>{resolve=yes;reject=no;});return{promise,resolve,reject};}; +const makeDetail=jobId=>({job:{job_id:jobId},files:[],nodes:[],items:[],events:[],pages:Object.fromEntries(['files','nodes','items','events'].map(s=>[s,{total:2,next:`${jobId}-${s}`}]))}); + +test('real controller hook ignores stale jobs/refreshes and isolates concurrent section loading and retry',async t=>{ + const originalJob=adminApi.job,originalPage=adminApi.jobPage; + const jobs=[],pages=[]; + adminApi.job=(id,signal)=>{const request={id,signal,...deferred()};jobs.push(request);return request.promise;}; + adminApi.jobPage=(id,section,params,signal)=>{const request={id,section,params,signal,...deferred()};pages.push(request);return request.promise;}; + let vm,renderer; + function Probe({id}){vm=useAdminJobDetailViewModel(id,options);return null;} + t.after(()=>{act(()=>renderer?.unmount());adminApi.job=originalJob;adminApi.jobPage=originalPage;}); + act(()=>{renderer=create(React.createElement(Probe,{id:'a'}));}); + act(()=>renderer.update(React.createElement(Probe,{id:'b'}))); + assert.ok(jobs[0].signal.aborted); + await act(async()=>{jobs[1].resolve(makeDetail('b'));}); + await act(async()=>{jobs[0].resolve(makeDetail('a'));}); + assert.equal(vm.detail.job.job_id,'b'); + let first,duplicate,independent; + act(()=>{first=vm.loadMore('files');duplicate=vm.loadMore('files');independent=vm.loadMore('events');}); + assert.equal(pages.length,2,'same-section duplicate click must issue one request'); + assert.ok(vm.pageState.files.loading && vm.pageState.events.loading); + await act(async()=>{pages[0].reject(new Error('offline'));await first;await duplicate;}); + assert.equal(vm.pageState.files.error,'offline'); + assert.ok(vm.pageState.events.loading); + await act(async()=>{pages[1].resolve({section:'events',rows:[{event_id:'event-b'}],total:2,next:null});await independent;}); + assert.equal(vm.detail.events[0].event_id,'event-b'); + let retry; + act(()=>{retry=vm.loadMore('files');}); + assert.equal(vm.pageState.files.error,null); + let refresh; + act(()=>{refresh=vm.reload();}); + assert.ok(pages[2].signal.aborted); + await act(async()=>{jobs[2].resolve(makeDetail('b'));await refresh;}); + await act(async()=>{pages[2].resolve({section:'files',rows:[{file_id:'stale'}],total:2,next:null});await retry;}); + assert.deepEqual(vm.detail.files,[],'a refresh replaces all four sections and discards stale pages'); + let more; + act(()=>{more=vm.loadMore('nodes');}); + act(()=>renderer.update(React.createElement(Probe,{id:'c'}))); + assert.equal(vm.detail,null); + assert.ok(pages[3].signal.aborted); + await act(async()=>{pages[3].resolve({section:'nodes',rows:[{node_id:'old'}],total:2,next:null});await more;jobs[3].resolve(makeDetail('c'));}); + assert.equal(vm.detail.job.job_id,'c'); + assert.deepEqual(vm.detail.nodes,[]); +}); + +test('pagination controls render localized counts, busy state and accessible retry in all six languages',()=>{ + const dictionaries=readDictionaries(); + for(const [locale,messages] of Object.entries(dictionaries)) { + const vm={detail:{files:[{file_id:'1'}],pages:{files:{total:1234,next:'n'}}},pageState:{files:{loading:true,error:'Network error. Check your connection.'}},loadMore:noop}; + const html=renderToStaticMarkup(React.createElement(I18nContext.Provider,{value:{locale,messages}},React.createElement(DetailPageControl,{section:'files',vm}))); + assert.ok(html.includes('aria-live="polite"')); + assert.ok(html.includes('role="alert"')); + assert.ok(html.includes(messages['Load more'])); + assert.ok(html.includes(messages.Retry)); + assert.ok(html.includes((1234).toLocaleString(locale))); + assert.ok(html.includes('disabled=""')); + assert.ok(!html.includes('{v0}')); + } +}); diff --git a/frontend/test/adminDetailRoute.test.mjs b/frontend/test/adminDetailRoute.test.mjs new file mode 100644 index 0000000..5c1e077 --- /dev/null +++ b/frontend/test/adminDetailRoute.test.mjs @@ -0,0 +1,52 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { NextRequest } from 'next/server.js'; +import { issueSession } from '../lib/server/adminAuth.js'; + +const asModule=source=>`data:text/javascript;base64,${Buffer.from(source).toString('base64')}`; +const routeUrl=new URL('../app/api/admin/jobs/[id]/route.js',import.meta.url); +const guardUrl=new URL('../lib/server/adminRoute.js',import.meta.url); +const guard=asModule(fs.readFileSync(guardUrl,'utf8').replace(/from '([^']+)'/g,(_s,name)=>`from '${name==='next/server'?new URL('../node_modules/next/server.js',import.meta.url).href:new URL(name,guardUrl).href}'`)); +const {GET}=await import(asModule(fs.readFileSync(routeUrl,'utf8').replace(/from '([^']+)'/g,(_s,name)=>`from '${name.endsWith('/adminRoute')?guard:new URL(name+'.js',routeUrl).href}'`))); + +test('detail route authenticates before paging negotiation, bounds reads and keeps private headers',async()=>{ + const keys=['ADMIN_KEY_1','ADMIN_KEY_2','SUPABASE_URL','SUPABASE_SERVICE_ROLE_KEY']; + const before=Object.fromEntries(keys.map(key=>[key,process.env[key]])); + const oldFetch=globalThis.fetch; + Object.assign(process.env,{ADMIN_KEY_1:'test-first-key-0123456789abcdef',ADMIN_KEY_2:'test-second-key-0123456789abcdef',SUPABASE_URL:'https://db.test',SUPABASE_SERVICE_ROLE_KEY:'test-key'}); + try { + const id='00000000-0000-4000-8000-000000000001'; + const token=issueSession([process.env.ADMIN_KEY_1,process.env.ADMIN_KEY_2]).token; + let queries=0; + globalThis.fetch=async url=>{ + queries++; + const parsed=new URL(url); + if(parsed.pathname.endsWith('/jobs')) return Response.json([{job_id:id}]); + assert.equal(parsed.searchParams.get('job_id'),`eq.${id}`); + assert.equal(parsed.searchParams.get('limit'),'1'); + return Response.json([{event_id:'first',created_at:'2026-10-29T00:00:00.000001+00:00'}],{headers:{'Content-Range':'0-0/2'}}); + }; + const request=(query='',headers={})=>new NextRequest(`https://app.test/api/admin/jobs/${id}${query}`,{headers}); + const context={params:{id}}; + const unauth=await GET(request(),context); + assert.equal(unauth.status,401); + const stale=await GET(request('',{cookie:`mdify_admin=${token}`}),context); + assert.equal(stale.status,409); + assert.equal(queries,0,'old clients and unauthenticated reads must not query job content'); + const headers={cookie:`mdify_admin=${token}`,'x-llmiser-detail-paging':'1'}; + const valid=await GET(request('?section=events&limit=1',headers),context); + assert.equal(valid.status,200); + const page=await valid.json(); + assert.equal(page.rows.length,1);assert.equal(page.total,2);assert.ok(page.next); + const invalid=await GET(request('?section=events&limit=0',headers),context); + assert.equal(invalid.status,400); + for(const response of [unauth,stale,valid,invalid]) { + assert.equal(response.headers.get('cache-control'),'no-store'); + assert.equal(response.headers.get('x-robots-tag'),'noindex, nofollow'); + } + } finally { + globalThis.fetch=oldFetch; + keys.forEach(key=>{if(before[key]===undefined)delete process.env[key];else process.env[key]=before[key];}); + } +}); diff --git a/frontend/test/contentTree.test.mjs b/frontend/test/contentTree.test.mjs index 36a367a..e1229c4 100644 --- a/frontend/test/contentTree.test.mjs +++ b/frontend/test/contentTree.test.mjs @@ -53,3 +53,14 @@ test('empty input gives an empty tree', () => { assert.equal(root.children.length, 0); assert.equal(counts.total, 0); }); + +test('processing nodes remain visible in mixed PDF trees and folder summaries',()=>{ + const {root,counts}=buildContentTree([ + node('r',{node_type:'ROOT_FILE',logical_path:'scan.pdf'}), + node('p',{node_type:'PDF_PAGE',page_from:2,page_to:2,status:'PROCESSING'}), + node('f',{node_type:'TEXT_FILE',logical_path:'notes/readme.md'}), + ]); + assert.equal(root.children.length,2); + assert.equal(counts.PROCESSING,1); + assert.equal(worstStatus(root),'PROCESSING'); +}); diff --git a/frontend/test/sql/pgliteSupabase.mjs b/frontend/test/sql/pgliteSupabase.mjs index fae7f68..cc94248 100644 --- a/frontend/test/sql/pgliteSupabase.mjs +++ b/frontend/test/sql/pgliteSupabase.mjs @@ -16,6 +16,13 @@ function where(filters, params) { const clauses = []; for (const [key, value] of Object.entries(filters || {})) { if (['select', 'order', 'limit', 'on_conflict'].includes(key)) continue; + if (key === 'or') { + const match = /^\((\w+)\.gt\.([^,]+),and\(\1\.eq\.([^,]+),(\w+)\.gt\.([^,)]+)\)\)$/.exec(value); + if (!match) throw new Error('unsupported logical filter'); + params.push(match[2], match[3], match[5]); + clauses.push(`(${ident(match[1])} > $${params.length - 2} or (${ident(match[1])} = $${params.length - 1} and ${ident(match[4])} > $${params.length}))`); + continue; + } const m = /^(eq|lt)\.(.*)$/s.exec(String(value)); if (!m) throw new Error(`unsupported filter ${key}=${value}`); params.push(m[2]); @@ -100,6 +107,27 @@ export function createPgliteSupabase(pg, { bucket = 'mdify-pro-files', calls = [ return rows; }, + async selectPage(table, filters = {}) { + const params = []; + const clause = where(filters, params); + const cols = filters.select.split(',').map(ident).join(', '); + const order = filters.order.split(',').map(part => { + const [col, dir] = part.split('.'); + return `${ident(col)} ${dir === 'desc' ? 'desc' : 'asc'}`; + }).join(', '); + const rows = (await pg.query(`select row_to_json(p) as value from (select ${cols} from public.${ident(table)}${clause} order by ${order} limit ${Number(filters.limit)}) p`, params)).rows.map(row => row.value); + const total = Number((await pg.query(`select count(*) as total from public.${ident(table)}${clause}`, params)).rows[0].total); + calls.push(['selectPage', table, rows.length]); + return { rows, total }; + }, + + async count(table, filters = {}) { + const params = []; + const total = Number((await pg.query(`select count(*) as total from public.${ident(table)}${where(filters, params)}`, params)).rows[0].total); + calls.push(['count', table]); + return total; + }, + async remove(table, filters) { const params = []; const { rows } = await pg.query(`delete from public.${ident(table)}${where(filters, params)} returning *`, params); diff --git a/frontend/viewmodels/useAdminViewModel.js b/frontend/viewmodels/useAdminViewModel.js index d230840..8b24510 100644 --- a/frontend/viewmodels/useAdminViewModel.js +++ b/frontend/viewmodels/useAdminViewModel.js @@ -7,6 +7,7 @@ import { useI18n } from '../components/i18n/I18nContext'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { adminApi, downloadFile, downloadZip, previewText } from '../lib/models/adminApi'; import { buildContentTree } from '../lib/models/contentTree'; +import { collectStoredFiles, mergeDetailPage } from '../lib/models/adminDetail'; // ── Session ──────────────────────────────────────────────────────────────── @@ -233,15 +234,7 @@ export function useAdminJobsViewModel({ handleError, notify }) { if (!ids.length) return; setWorking('ZIP'); try { - const items = []; - for (const id of ids) { - const detail = await adminApi.job(id); - for (const f of detail.files) { - if (f.kind === 'OUTPUT' && f.storage_status === 'ACTIVE') { - items.push({ jobId: id, fileId: f.file_id, name: `${id.slice(0, 8)}/${f.object_path.split('/').pop()}` }); - } - } - } + const items = await collectStoredFiles(ids, { loadPage: adminApi.jobPage, outputsOnly: true }); if (!items.length) { notify('The selected jobs have no stored results', 'error'); return; @@ -283,26 +276,77 @@ export function useAdminJobDetailViewModel(jobId, { handleError, notify, onChang const [preview, setPreview] = useState(null); // { name, text, truncated } | { name, loading } const [working, setWorking] = useState(null); + const generation = useRef(0); + const currentJob = useRef(jobId); + currentJob.current = jobId; + const controllers = useRef(new Set()); + const inFlight = useRef(new Set()); + const [pageState, setPageState] = useState({}); + const previewGeneration = useRef(0); + const load = useCallback(async () => { - if (!jobId) return; + if (!jobId || currentJob.current !== jobId) return; + const gen = ++generation.current; + controllers.current.forEach(controller => controller.abort()); + controllers.current.clear(); + inFlight.current = new Set(); + const controller = new AbortController(); + controllers.current.add(controller); setLoading(true); + previewGeneration.current += 1; + setPreview(null); + setPageState({}); setError(null); try { - setDetail(await adminApi.job(jobId)); + const result = await adminApi.job(jobId, controller.signal); + if (gen === generation.current && currentJob.current === jobId) setDetail(result); } catch (err) { - if (!handleError(err)) setError(err.message); + if (gen === generation.current && err?.name !== 'AbortError' && !handleError(err)) setError(err.message); } finally { - setLoading(false); + controllers.current.delete(controller); + if (gen === generation.current) setLoading(false); } }, [jobId, handleError]); useEffect(() => { setDetail(null); setPreview(null); + setWorking(null); load(); + return () => { + generation.current += 1; + previewGeneration.current += 1; + controllers.current.forEach(controller => controller.abort()); + controllers.current.clear(); + }; }, [load]); - const tree = useMemo(() => (detail ? buildContentTree(detail.nodes, detail.job) : null), [detail]); + const visibleDetail = detail?.job?.job_id === jobId ? detail : null; + const tree = useMemo(() => (visibleDetail ? buildContentTree(visibleDetail.nodes, visibleDetail.job) : null), [visibleDetail]); + + const loadMore = useCallback(async section => { + const next = visibleDetail?.pages?.[section]?.next; + const flights = inFlight.current; + if (!next || loading || flights.has(section)) return; + flights.add(section); + const gen = generation.current; + const controller = new AbortController(); + controllers.current.add(controller); + setPageState(state => ({ ...state, [section]: { loading: true, error: null } })); + try { + const page = await adminApi.jobPage(jobId, section, { cursor: next }, controller.signal); + if (gen !== generation.current || currentJob.current !== jobId) return; + setDetail(previous => previous?.job.job_id === jobId ? mergeDetailPage(previous, section, page) : previous); + } catch (err) { + if (gen === generation.current && err?.name !== 'AbortError' && !handleError(err)) { + setPageState(state => ({ ...state, [section]: { loading: false, error: err.message } })); + } + } finally { + flights.delete(section); + controllers.current.delete(controller); + if (gen === generation.current) setPageState(state => ({ ...state, [section]: { ...state[section], loading: false } })); + } + }, [visibleDetail, jobId, loading, handleError]); const act = useCallback( async (action, hours) => { @@ -317,7 +361,7 @@ export function useAdminJobDetailViewModel(jobId, { handleError, notify, onChang } catch (err) { if (!handleError(err)) notify(err.message, 'error'); } finally { - setWorking(null); + if (currentJob.current === jobId) setWorking(null); } }, [jobId, notify, handleError, onChanged, load, uiText] @@ -325,12 +369,15 @@ export function useAdminJobDetailViewModel(jobId, { handleError, notify, onChang const showPreview = useCallback( async (file) => { + const gen = generation.current; + const request = ++previewGeneration.current; const name = file.object_path.split('/').pop(); setPreview({ name, loading: true }); try { - setPreview({ name, ...(await previewText(jobId, file.file_id)) }); + const result = await previewText(jobId, file.file_id); + if (gen === generation.current && request === previewGeneration.current) setPreview({ name, ...result }); } catch (err) { - if (!handleError(err)) setPreview({ name, error: err.message }); + if (gen === generation.current && request === previewGeneration.current && !handleError(err)) setPreview({ name, error: err.message }); } }, [jobId, handleError] @@ -348,21 +395,19 @@ export function useAdminJobDetailViewModel(jobId, { handleError, notify, onChang ); const downloadAll = useCallback(async () => { - const files = (detail?.files || []).filter((f) => f.storage_status === 'ACTIVE'); setWorking('ZIP'); try { - await downloadZip( - files.map((f) => ({ jobId, fileId: f.file_id, name: `${f.kind.toLowerCase()}/${f.object_path.split('/').pop()}` })), - { zipName: `llmiser-job-${jobId.slice(0, 8)}.zip` } - ); + const files = await collectStoredFiles([jobId], { loadPage: adminApi.jobPage }); + await downloadZip(files, { zipName: `llmiser-job-${jobId.slice(0, 8)}.zip` }); } catch (err) { if (!handleError(err)) notify(err.message, 'error'); } finally { - setWorking(null); + if (currentJob.current === jobId) setWorking(null); } - }, [detail, jobId, handleError, notify]); + }, [jobId, handleError, notify]); - return { detail, tree, loading, error, reload: load, act, working, preview, showPreview, closePreview: () => setPreview(null), download, downloadAll }; + return { detail: visibleDetail, tree, loading, error, reload: load, loadMore, pageState, act, working, preview, showPreview, + closePreview: () => { previewGeneration.current += 1; setPreview(null); }, download, downloadAll }; } // ── Toasts ───────────────────────────────────────────────────────────────── From a156dd7a417b853a1c7cde9d8f604967cfcb4ba9 Mon Sep 17 00:00:00 2001 From: DevBehindYou Date: Thu, 29 Oct 2026 23:37:09 +0530 Subject: [PATCH 2/2] Bundle existing app fonts to remove Google build dependency --- docs/LLMISER_CONTROLLER_PAGINATION.md | 11 +- docs/LLMISER_OPTIMIZATION_PLAN.md | 2 +- docs/START_HERE.md | 2 +- docs/ai-handover/HANDOVER.md | 2 +- docs/ai-handover/PROGRESS.md | 2 +- frontend/app/fonts.css | 31 ++++ frontend/app/layout.js | 25 +-- frontend/next.config.js | 1 + frontend/public/fonts/Inter-OFL.txt | 93 ++++++++++ frontend/public/fonts/JetBrainsMono-OFL.txt | 93 ++++++++++ frontend/public/fonts/PatrickHand-OFL.txt | 93 ++++++++++ frontend/public/fonts/README.md | 15 ++ frontend/public/fonts/manifest.json | 171 ++++++++++++++++++ frontend/public/fonts/v1/inter-1.woff2 | Bin 0 -> 25844 bytes frontend/public/fonts/v1/inter-2.woff2 | Bin 0 -> 18744 bytes frontend/public/fonts/v1/inter-3.woff2 | Bin 0 -> 11272 bytes frontend/public/fonts/v1/inter-4.woff2 | Bin 0 -> 19044 bytes frontend/public/fonts/v1/inter-5.woff2 | Bin 0 -> 10280 bytes frontend/public/fonts/v1/inter-6.woff2 | Bin 0 -> 85272 bytes frontend/public/fonts/v1/inter-latin.woff2 | Bin 0 -> 48432 bytes .../public/fonts/v1/jetbrains-mono-1.woff2 | Bin 0 -> 1664 bytes .../public/fonts/v1/jetbrains-mono-2.woff2 | Bin 0 -> 8892 bytes .../public/fonts/v1/jetbrains-mono-3.woff2 | Bin 0 -> 6800 bytes .../public/fonts/v1/jetbrains-mono-4.woff2 | Bin 0 -> 5872 bytes .../public/fonts/v1/jetbrains-mono-5.woff2 | Bin 0 -> 11596 bytes .../fonts/v1/jetbrains-mono-latin.woff2 | Bin 0 -> 31340 bytes frontend/public/fonts/v1/patrick-hand-1.woff2 | Bin 0 -> 4748 bytes frontend/public/fonts/v1/patrick-hand-2.woff2 | Bin 0 -> 10604 bytes .../public/fonts/v1/patrick-hand-latin.woff2 | Bin 0 -> 14224 bytes frontend/tailwind.config.js | 2 +- frontend/test/appFonts.test.mjs | 42 +++++ 31 files changed, 559 insertions(+), 26 deletions(-) create mode 100644 frontend/app/fonts.css create mode 100644 frontend/public/fonts/Inter-OFL.txt create mode 100644 frontend/public/fonts/JetBrainsMono-OFL.txt create mode 100644 frontend/public/fonts/PatrickHand-OFL.txt create mode 100644 frontend/public/fonts/README.md create mode 100644 frontend/public/fonts/manifest.json create mode 100644 frontend/public/fonts/v1/inter-1.woff2 create mode 100644 frontend/public/fonts/v1/inter-2.woff2 create mode 100644 frontend/public/fonts/v1/inter-3.woff2 create mode 100644 frontend/public/fonts/v1/inter-4.woff2 create mode 100644 frontend/public/fonts/v1/inter-5.woff2 create mode 100644 frontend/public/fonts/v1/inter-6.woff2 create mode 100644 frontend/public/fonts/v1/inter-latin.woff2 create mode 100644 frontend/public/fonts/v1/jetbrains-mono-1.woff2 create mode 100644 frontend/public/fonts/v1/jetbrains-mono-2.woff2 create mode 100644 frontend/public/fonts/v1/jetbrains-mono-3.woff2 create mode 100644 frontend/public/fonts/v1/jetbrains-mono-4.woff2 create mode 100644 frontend/public/fonts/v1/jetbrains-mono-5.woff2 create mode 100644 frontend/public/fonts/v1/jetbrains-mono-latin.woff2 create mode 100644 frontend/public/fonts/v1/patrick-hand-1.woff2 create mode 100644 frontend/public/fonts/v1/patrick-hand-2.woff2 create mode 100644 frontend/public/fonts/v1/patrick-hand-latin.woff2 create mode 100644 frontend/test/appFonts.test.mjs diff --git a/docs/LLMISER_CONTROLLER_PAGINATION.md b/docs/LLMISER_CONTROLLER_PAGINATION.md index e85d8e8..8c03d87 100644 --- a/docs/LLMISER_CONTROLLER_PAGINATION.md +++ b/docs/LLMISER_CONTROLLER_PAGINATION.md @@ -1,6 +1,6 @@ # Controller detail pagination -Updated 29 October 2026. Candidate on `codex/controller-detail-pagination`; release and production acceptance are still pending. The previously released runtime is PR #17, `9131a2ac6163f22b630b008b40accb482ed3e0b3`. +Updated 29 October 2026. Candidate in [PR #19](https://github.com/DevBehindYou/LLMiser/pull/19). Initial pagination source `2f2afb0` passed all seven GitHub CI jobs, including 278 frontend tests and build, but failed the Vercel preview in Next.js's Google font loader. The owner supplied the build log; a bundled-font fix is being validated. Do not merge or claim production acceptance until the updated runtime preview passes. Previously released runtime remains PR #17, `9131a2ac6163f22b630b008b40accb482ed3e0b3`. Opening a large job previously returned all content nodes, work items, events and registered files. The controller now starts with 50 records per section, with independent **Load more** and retry controls. Counts distinguish loaded rows from the full section total. The tree and status badges explicitly describe loaded records; Refresh restarts every section. Six interface languages are supported; blogs and their visuals are unchanged. @@ -37,6 +37,13 @@ Reproduce generated plans/sizes from `frontend` with `node --disable-warning=MOD Production baseline: one generated 72-text-file ZIP completed through the public Storage flow. Authenticated detail returned 73 nodes, four registered ACTIVE files, one work item and two events. The job has normal 48-hour retention; no cleanup or retention mutation was performed. After release, compare every section's IDs against this baseline, retrieve every stored file and reopen a ZIP containing them. No user documents are needed for acceptance. -Full local regression: 276 of 278 passed in the sandbox; the two deployment-scope tests could not create temporary Git configurations. Both passed when rerun outside the sandbox. Lint, six dictionaries (637 messages each), SQL consistency, blog consistency and diff whitespace checks passed. The initial lint/build attempt also encountered local filesystem restrictions; the elevated build is being checked separately. CI, release and production results must be recorded before calling this released. React/SSR acceptance does not establish native browser layout, mobile overflow or representative-device performance. +Full local regression: 276 of 278 passed in the sandbox; the two deployment-scope tests could not create temporary Git configurations. Both passed when rerun outside the sandbox. Lint, six dictionaries (637 messages each), SQL consistency, blog consistency and diff whitespace checks passed. The initial lint/build attempt also encountered local filesystem restrictions; elevated lint and production build passed. All seven Linux CI jobs passed; its full frontend run passed all 278 tests and build. React/SSR acceptance does not establish native browser layout, mobile overflow or representative-device performance. + +## Remaining release steps + +1. Validate the bundled-font fix in the real frontend preview (`albedo-projects/llmiser`, root `frontend`). The owner's log for the initial failed preview identifies Next.js 14.2.35's Google loader extension regex returning null at `loader.js:112`. Local/CI builds alone did not reproduce it. This is separate from pagination logic. +2. The app now bundles the exact 16 WOFF2 assets from the successful original build, all 25 face declarations, Unicode ranges, weights, fallback metrics and the same three Latin preloads (93,996 bytes). `frontend/app/fonts.css` and `frontend/public/fonts/` replace the Google loader; official pinned SIL licenses and hash manifest accompany the files. No font/framework upgrade, runtime dependency or blog edit is included. Validate a clean build, font-integrity tests, CI and actual Vercel preview, then merge the pinned passing revision under the owner's auto-merge authorization. A documentation-only skipped build is not acceptance. +3. Verify the production frontend, then run the generated authenticated comparison described above. Private `verify-live.mjs after` reuses the recorded generated job, traverses every section, collects ACTIVE files through one-row pages, signs/downloads all files and reopens an assembled ZIP; it also tests 401/409/400 and private headers. It reads configured keys into memory and never prints keys, cookies or signed links. +4. Run the public HTTP/redirect and uncached pool-readiness checks; record actual results and update current handovers to released only after acceptance. No new conversion job is needed if the baseline is still retained; its `auto_delete_at` is 31 October 2026 17:43:53 UTC. PostgREST documents [pagination/count headers and HEAD](https://docs.postgrest.org/en/stable/references/api/pagination_count.html). PostgreSQL explains why [unique ordering is necessary and large offsets remain costly](https://www.postgresql.org/docs/16/queries-limit.html). These contracts guide the cursor design; production measurements must remain separate from local generated evidence. diff --git a/docs/LLMISER_OPTIMIZATION_PLAN.md b/docs/LLMISER_OPTIMIZATION_PLAN.md index ba4e52f..07fa48e 100644 --- a/docs/LLMISER_OPTIMIZATION_PLAN.md +++ b/docs/LLMISER_OPTIMIZATION_PLAN.md @@ -24,7 +24,7 @@ Search migration and public redirects are live; see [search migration](LLMISER_S ## Current slice: controller detail pagination -Candidate: `codex/controller-detail-pagination`; see [implementation and evidence](LLMISER_CONTROLLER_PAGINATION.md). Files, nodes, work items and events now load 50 rows each, with stable unique cursors, exact live totals, per-section retry and refresh/job-switch guards. ZIP exports fetch every matching stored file page before signing/downloading. Existing indexes suffice for generated scoped query plans; no production SQL change is required. Focused acceptance passed; full regression, release and authenticated deployed comparison remain pending. +Candidate: PR #19 on `codex/controller-detail-pagination`; see [implementation and evidence](LLMISER_CONTROLLER_PAGINATION.md). Files, nodes, work items and events load 50 rows each, with stable unique cursors, exact live totals, independent retry and refresh/job-switch guards. ZIP exports fetch every matching stored file page. Existing indexes suffice; no SQL change. Initial source `2f2afb0` passed all seven CI jobs/278 tests/build, but Vercel's Google font URL parser failed. The owner supplied the error; identical bundled fonts remove that build dependency. Updated checks/preview and deployed comparison remain before release. ## Historical slice: retained result recovery diff --git a/docs/START_HERE.md b/docs/START_HERE.md index 765b5e6..addd8b5 100644 --- a/docs/START_HERE.md +++ b/docs/START_HERE.md @@ -4,7 +4,7 @@ Updated 29 October 2026. **LLMiser is the sole working repository and the source ## Current state -Current candidate: [controller detail pagination](LLMISER_CONTROLLER_PAGINATION.md) on `codex/controller-detail-pagination`. Bounded files/tree/work-items/events, exact totals, independent Load more/retry and complete ZIP metadata collection are implemented. Focused PostgreSQL/React/route checks passed; full regression, release and deployed acceptance are pending. No database migration is required. Production remains the revision recorded below until release is verified. +Current candidate: [controller detail pagination](LLMISER_CONTROLLER_PAGINATION.md), PR #19 on `codex/controller-detail-pagination`. Initial source `2f2afb0` passed all seven CI jobs (278 frontend tests/build), but Vercel's Google-font URL parser failed. The owner supplied its log. An exact-byte bundled-font fix removes the Google build dependency while retaining existing typography/subsets/preloads; updated validation is in progress. Hold merge until the updated preview passes, then compare the authenticated production baseline. No database migration is required. Production remains the revision below. The app is live at https://llmiser.devbehindyou.com. PRs #1–#17 are merged; the latest verified runtime revision is `9131a2ac6163f22b630b008b40accb482ed3e0b3` from PR #17. All seven CI jobs passed before and after merge, including 270 frontend tests and Linux OCR; all six production Vercel statuses succeeded. Release checks confirmed that exact revision and readiness on every N/O/Z worker. Thirty protocol checks, eight workbook/ZIP equivalence checks, four post-release Storage conversion/recovery jobs and 20 public HTTP/redirect checks passed; all pools were 2/2 ready. See [shared deadlines](LLMISER_SHARED_DEADLINES.md), [Excel/assembly](LLMISER_EXCEL_ASSEMBLY_MEMORY.md), [CSV capacity](LLMISER_PARSER_CAPACITY.md) and [retirement evidence](LLMISER_LEGACY_RETIREMENT.md). diff --git a/docs/ai-handover/HANDOVER.md b/docs/ai-handover/HANDOVER.md index 5054373..661997a 100644 --- a/docs/ai-handover/HANDOVER.md +++ b/docs/ai-handover/HANDOVER.md @@ -4,7 +4,7 @@ Updated 29 October 2026. Start with [START_HERE](../START_HERE.md). This replace ## What the owner wants now -Active implementation: [controller detail pagination](../LLMISER_CONTROLLER_PAGINATION.md), branch `codex/controller-detail-pagination`. The owner resumed this task after pausing it. Code and focused acceptance are complete; full validation, PR/merge/deployment and generated authenticated production comparison remain. Keep ZIP exports complete across pages, timestamp precision and job/section/filter cursor binding. Exact counts are live separate reads; Refresh resets all sections. No new SQL/index is needed from the measured existing-index plans. Runtime remains PR #17 until release evidence is recorded. +Active implementation: [controller detail pagination](../LLMISER_CONTROLLER_PAGINATION.md), PR #19 on `codex/controller-detail-pagination`. Initial source `2f2afb0` passed all seven CI jobs (278 frontend tests/build); Vercel's Google font URL extension parser failed separately. The owner supplied the log. Exact existing WOFF2 assets, subset declarations/fallback metrics and preloads are now bundled with licenses and hashes; updated checks/preview remain before merge. No connected Vercel tool/CLI session is assumed. Keep complete ZIP traversal, timestamp precision and job/section/filter cursor binding. Counts are separate live reads; Refresh resets all sections. No new SQL/index is needed. Validate the corrected preview, merge under existing authorization, then compare the generated authenticated baseline and public HTTP/wake results. Runtime remains PR #17 until release evidence is recorded. The owner resumed the optimization queue after the Excel/assembly release. Shared conversion deadlines are now released in PR #17. Continue from the remaining queue rather than repeating either slice. LLMiser must also remain independently maintainable after MDify and MDify-Pro are deleted. Keep immediate permanent redirects on the old public hosts and the minimal LLMiser-only fallback; repository/service deletion is not being executed here. diff --git a/docs/ai-handover/PROGRESS.md b/docs/ai-handover/PROGRESS.md index 4e058bc..f62a83a 100644 --- a/docs/ai-handover/PROGRESS.md +++ b/docs/ai-handover/PROGRESS.md @@ -51,7 +51,7 @@ See [release record](../LLMISER_RELEASE_2026-10-26.md), [localization](../LLMISE ## Remaining work -In progress: [controller detail pagination](../LLMISER_CONTROLLER_PAGINATION.md) on `codex/controller-detail-pagination`. Implemented bounded detail collections, stable cursors/exact counts, per-section Load more/retry, refresh invalidation and complete ZIP collection. Twelve focused checks passed, including migrated PostgreSQL and real React hook races. Generated query plans use existing indexes; no migration is proposed. Authenticated production baseline is captured from one generated ZIP job (73 nodes, four ACTIVE files). Full tests/build, CI/merge and after-release comparison are pending; do not mark this released yet. Six dictionaries now have 637 candidate messages; production still has 631. +In progress: [controller detail pagination](../LLMISER_CONTROLLER_PAGINATION.md), PR #19. Initial source `2f2afb0` passed all seven CI jobs/278 frontend tests/build, but Vercel's Google font URL parser failed. The owner supplied the diagnostic. The fix bundles identical existing fonts, subset/weight ranges, fallback metrics and preloads with licenses/hashes; updated checks and preview are pending. Pagination has bounded collections, exact live totals, independent retry and complete ZIP collection. Existing-index plans need no migration. Generated authenticated baseline: 73 nodes, four ACTIVE files, one work item and two events, normal 48-hour retention. Do not call this released until updated CI/preview and production comparison pass. Six dictionaries have 637 candidate messages; production still has 631. Released: [shared conversion deadlines](../LLMISER_SHARED_DEADLINES.md), PR #17 at 9131a2a. All seven CI jobs passed before/after merge, all six Vercel statuses succeeded and all six workers reported the exact revision/readiness. Thirty production auth/expired/invalid/valid/legacy protocol requests passed; eight future-header workbook/ZIP comparisons and two Storage baselines retained identical output/counts. Two additional Storage jobs exercised image-ZIP OCR/merge and scanned-PDF OCR/merge with exact recovery. Twenty public HTTP checks passed and all pools were 2/2 ready. Six generated Storage jobs in total across pre/post/multi-stage probes follow existing retention; cleanup was not manually run. Controller detail pagination is next; no whole-job lifetime or exactly-once write guarantee is claimed. diff --git a/frontend/app/fonts.css b/frontend/app/fonts.css new file mode 100644 index 0000000..8b43483 --- /dev/null +++ b/frontend/app/fonts.css @@ -0,0 +1,31 @@ +/* Pinned self-hosted fonts; see public/fonts/v1/README.md and licenses. */ +@font-face{font-family:"LLMiser Inter";font-style:normal;font-weight:100 900;font-display:swap;src:url(/fonts/v1/inter-1.woff2) format("woff2");unicode-range:u+0460-052f,u+1c80-1c8a,u+20b4,u+2de0-2dff,u+a640-a69f,u+fe2e-fe2f} +@font-face{font-family:"LLMiser Inter";font-style:normal;font-weight:100 900;font-display:swap;src:url(/fonts/v1/inter-2.woff2) format("woff2");unicode-range:u+0301,u+0400-045f,u+0490-0491,u+04b0-04b1,u+2116} +@font-face{font-family:"LLMiser Inter";font-style:normal;font-weight:100 900;font-display:swap;src:url(/fonts/v1/inter-3.woff2) format("woff2");unicode-range:u+1f??} +@font-face{font-family:"LLMiser Inter";font-style:normal;font-weight:100 900;font-display:swap;src:url(/fonts/v1/inter-4.woff2) format("woff2");unicode-range:u+0370-0377,u+037a-037f,u+0384-038a,u+038c,u+038e-03a1,u+03a3-03ff} +@font-face{font-family:"LLMiser Inter";font-style:normal;font-weight:100 900;font-display:swap;src:url(/fonts/v1/inter-5.woff2) format("woff2");unicode-range:u+0102-0103,u+0110-0111,u+0128-0129,u+0168-0169,u+01a0-01a1,u+01af-01b0,u+0300-0301,u+0303-0304,u+0308-0309,u+0323,u+0329,u+1ea0-1ef9,u+20ab} +@font-face{font-family:"LLMiser Inter";font-style:normal;font-weight:100 900;font-display:swap;src:url(/fonts/v1/inter-6.woff2) format("woff2");unicode-range:u+0100-02ba,u+02bd-02c5,u+02c7-02cc,u+02ce-02d7,u+02dd-02ff,u+0304,u+0308,u+0329,u+1d00-1dbf,u+1e00-1e9f,u+1ef2-1eff,u+2020,u+20a0-20ab,u+20ad-20c4,u+2113,u+2c60-2c7f,u+a720-a7ff} +@font-face{font-family:"LLMiser Inter";font-style:normal;font-weight:100 900;font-display:swap;src:url(/fonts/v1/inter-latin.woff2) format("woff2");unicode-range:u+00??,u+0131,u+0152-0153,u+02bb-02bc,u+02c6,u+02da,u+02dc,u+0304,u+0308,u+0329,u+2000-206f,u+20ac,u+2122,u+2191,u+2193,u+2212,u+2215,u+feff,u+fffd} +@font-face{font-family:"LLMiser Inter Fallback";src:local("Arial");ascent-override:90.49%;descent-override:22.56%;line-gap-override:0.00%;size-adjust:107.06%} +@font-face{font-family:"LLMiser JetBrains Mono";font-style:normal;font-weight:400;font-display:swap;src:url(/fonts/v1/jetbrains-mono-1.woff2) format("woff2");unicode-range:u+0460-052f,u+1c80-1c8a,u+20b4,u+2de0-2dff,u+a640-a69f,u+fe2e-fe2f} +@font-face{font-family:"LLMiser JetBrains Mono";font-style:normal;font-weight:400;font-display:swap;src:url(/fonts/v1/jetbrains-mono-2.woff2) format("woff2");unicode-range:u+0301,u+0400-045f,u+0490-0491,u+04b0-04b1,u+2116} +@font-face{font-family:"LLMiser JetBrains Mono";font-style:normal;font-weight:400;font-display:swap;src:url(/fonts/v1/jetbrains-mono-3.woff2) format("woff2");unicode-range:u+0370-0377,u+037a-037f,u+0384-038a,u+038c,u+038e-03a1,u+03a3-03ff} +@font-face{font-family:"LLMiser JetBrains Mono";font-style:normal;font-weight:400;font-display:swap;src:url(/fonts/v1/jetbrains-mono-4.woff2) format("woff2");unicode-range:u+0102-0103,u+0110-0111,u+0128-0129,u+0168-0169,u+01a0-01a1,u+01af-01b0,u+0300-0301,u+0303-0304,u+0308-0309,u+0323,u+0329,u+1ea0-1ef9,u+20ab} +@font-face{font-family:"LLMiser JetBrains Mono";font-style:normal;font-weight:400;font-display:swap;src:url(/fonts/v1/jetbrains-mono-5.woff2) format("woff2");unicode-range:u+0100-02ba,u+02bd-02c5,u+02c7-02cc,u+02ce-02d7,u+02dd-02ff,u+0304,u+0308,u+0329,u+1d00-1dbf,u+1e00-1e9f,u+1ef2-1eff,u+2020,u+20a0-20ab,u+20ad-20c4,u+2113,u+2c60-2c7f,u+a720-a7ff} +@font-face{font-family:"LLMiser JetBrains Mono";font-style:normal;font-weight:400;font-display:swap;src:url(/fonts/v1/jetbrains-mono-latin.woff2) format("woff2");unicode-range:u+00??,u+0131,u+0152-0153,u+02bb-02bc,u+02c6,u+02da,u+02dc,u+0304,u+0308,u+0329,u+2000-206f,u+20ac,u+2122,u+2191,u+2193,u+2212,u+2215,u+feff,u+fffd} +@font-face{font-family:"LLMiser JetBrains Mono";font-style:normal;font-weight:500;font-display:swap;src:url(/fonts/v1/jetbrains-mono-1.woff2) format("woff2");unicode-range:u+0460-052f,u+1c80-1c8a,u+20b4,u+2de0-2dff,u+a640-a69f,u+fe2e-fe2f} +@font-face{font-family:"LLMiser JetBrains Mono";font-style:normal;font-weight:500;font-display:swap;src:url(/fonts/v1/jetbrains-mono-2.woff2) format("woff2");unicode-range:u+0301,u+0400-045f,u+0490-0491,u+04b0-04b1,u+2116} +@font-face{font-family:"LLMiser JetBrains Mono";font-style:normal;font-weight:500;font-display:swap;src:url(/fonts/v1/jetbrains-mono-3.woff2) format("woff2");unicode-range:u+0370-0377,u+037a-037f,u+0384-038a,u+038c,u+038e-03a1,u+03a3-03ff} +@font-face{font-family:"LLMiser JetBrains Mono";font-style:normal;font-weight:500;font-display:swap;src:url(/fonts/v1/jetbrains-mono-4.woff2) format("woff2");unicode-range:u+0102-0103,u+0110-0111,u+0128-0129,u+0168-0169,u+01a0-01a1,u+01af-01b0,u+0300-0301,u+0303-0304,u+0308-0309,u+0323,u+0329,u+1ea0-1ef9,u+20ab} +@font-face{font-family:"LLMiser JetBrains Mono";font-style:normal;font-weight:500;font-display:swap;src:url(/fonts/v1/jetbrains-mono-5.woff2) format("woff2");unicode-range:u+0100-02ba,u+02bd-02c5,u+02c7-02cc,u+02ce-02d7,u+02dd-02ff,u+0304,u+0308,u+0329,u+1d00-1dbf,u+1e00-1e9f,u+1ef2-1eff,u+2020,u+20a0-20ab,u+20ad-20c4,u+2113,u+2c60-2c7f,u+a720-a7ff} +@font-face{font-family:"LLMiser JetBrains Mono";font-style:normal;font-weight:500;font-display:swap;src:url(/fonts/v1/jetbrains-mono-latin.woff2) format("woff2");unicode-range:u+00??,u+0131,u+0152-0153,u+02bb-02bc,u+02c6,u+02da,u+02dc,u+0304,u+0308,u+0329,u+2000-206f,u+20ac,u+2122,u+2191,u+2193,u+2212,u+2215,u+feff,u+fffd} +@font-face{font-family:"LLMiser JetBrains Mono Fallback";src:local("Arial");ascent-override:75.79%;descent-override:22.29%;line-gap-override:0.00%;size-adjust:134.59%} +@font-face{font-family:"LLMiser Patrick Hand";font-style:normal;font-weight:400;font-display:swap;src:url(/fonts/v1/patrick-hand-1.woff2) format("woff2");unicode-range:u+0102-0103,u+0110-0111,u+0128-0129,u+0168-0169,u+01a0-01a1,u+01af-01b0,u+0300-0301,u+0303-0304,u+0308-0309,u+0323,u+0329,u+1ea0-1ef9,u+20ab} +@font-face{font-family:"LLMiser Patrick Hand";font-style:normal;font-weight:400;font-display:swap;src:url(/fonts/v1/patrick-hand-2.woff2) format("woff2");unicode-range:u+0100-02ba,u+02bd-02c5,u+02c7-02cc,u+02ce-02d7,u+02dd-02ff,u+0304,u+0308,u+0329,u+1d00-1dbf,u+1e00-1e9f,u+1ef2-1eff,u+2020,u+20a0-20ab,u+20ad-20c4,u+2113,u+2c60-2c7f,u+a720-a7ff} +@font-face{font-family:"LLMiser Patrick Hand";font-style:normal;font-weight:400;font-display:swap;src:url(/fonts/v1/patrick-hand-latin.woff2) format("woff2");unicode-range:u+00??,u+0131,u+0152-0153,u+02bb-02bc,u+02c6,u+02da,u+02dc,u+0304,u+0308,u+0329,u+2000-206f,u+20ac,u+2122,u+2191,u+2193,u+2212,u+2215,u+feff,u+fffd} +@font-face{font-family:"LLMiser Patrick Hand Fallback";src:local("Arial");ascent-override:127.97%;descent-override:38.32%;line-gap-override:0.00%;size-adjust:81.43%} +:root { + --font-inter: "LLMiser Inter", "LLMiser Inter Fallback"; + --font-jetbrains-mono: "LLMiser JetBrains Mono", "LLMiser JetBrains Mono Fallback"; + --font-patrick-hand: "LLMiser Patrick Hand", "LLMiser Patrick Hand Fallback"; +} diff --git a/frontend/app/layout.js b/frontend/app/layout.js index 52b07b0..18d1e5f 100644 --- a/frontend/app/layout.js +++ b/frontend/app/layout.js @@ -1,4 +1,4 @@ -import { Inter, JetBrains_Mono, Patrick_Hand } from 'next/font/google'; +import './fonts.css'; import './globals.css'; import ConsentBanner from '../components/ConsentBanner'; import WakeOnOpen from '../components/WakeOnOpen'; @@ -7,23 +7,6 @@ import JsonLd from '../components/JsonLd'; import LanguageProvider from '../components/i18n/LanguageProvider'; import { DESCRIPTION, HOME_TITLE, KEYWORDS, SITE_NAME, SITE_URL, TAGLINE, siteJsonLd } from '../lib/siteContent'; -// next/font downloads these at build time and serves them from our own -// origin, so visitors' browsers never contact Google's font servers (GDPR). -const inter = Inter({ subsets: ['latin'], variable: '--font-inter', display: 'swap' }); -const jetbrainsMono = JetBrains_Mono({ - subsets: ['latin'], - weight: ['400', '500'], - variable: '--font-jetbrains-mono', - display: 'swap', -}); -const patrickHand = Patrick_Hand({ - subsets: ['latin'], - weight: '400', - variable: '--font-patrick-hand', - display: 'swap', -}); -const fontVariables = `${inter.variable} ${jetbrainsMono.variable} ${patrickHand.variable}`; - const BASE_URL = SITE_URL; export const metadata = { @@ -112,8 +95,12 @@ export const viewport = { themeColor: [ export default function RootLayout({ children }) { return ( - + + {/* Exact existing Latin subsets; other scripts load through unicode-range. */} + + + {/* Apply saved theme before paint (default dark), then point the favicon at the matching icon once the head links exist. */}