diff --git a/tools/render/README.md b/tools/render/README.md index d4195a58..7cafc3f5 100644 --- a/tools/render/README.md +++ b/tools/render/README.md @@ -254,6 +254,26 @@ curl -s http://127.0.0.1:4310/health | `RENDER_CONCURRENCY` | 2 | simultaneous renders | | `RENDER_QUEUE` | 8 | pending renders before 503 | | `RENDER_TIMEOUT` | 45000 | ms before a render is abandoned | +| `RENDER_FILES` | unset | a folder laid out as the download site; see below | + +**The release's files.** The render page fetches a diagram's layout +(`{release}/diagram/{id}.json`, `.graph.json`) and an illustration +(`{release}/ehld/{id}.svg`) from the download site, as a reader's browser does. +With `RENDER_FILES` set (`--files` for the CLI), every request the page makes to +the download site is answered from that folder instead, laid out as the site is +(`/98/diagram/R-HSA-73857.json`), and a file the folder lacks is a 404, +never a request to the site. Set it for any bulk run, and for a release's own +figures, whose files are not on the download site until the release is +published. In a container, mount the folder read-only and point the variable at +it: + +```sh +docker run ... -v /path/to/release-files:/files:ro -e RENDER_FILES=/files websiteangular-render +``` + +A file missing from the folder is not reported as such by the page: the render +waits out `RENDER_TIMEOUT`. So check the folder holds every file before a bulk +run. `/health` says which source is in use (`files`). Query parameters: `token`, `scale`, `subpathways=false`, and for GIF `delay` (ms per frame) and `maxSize` (longest side). All of them are part of the cache diff --git a/tools/render/local-files.spec.ts b/tools/render/local-files.spec.ts new file mode 100644 index 00000000..9ff47ec7 --- /dev/null +++ b/tools/render/local-files.spec.ts @@ -0,0 +1,55 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { DOWNLOAD_ORIGIN, isDownload, localAnswer } from './local-files.ts'; + +let root: string; +beforeAll(async () => { + root = await mkdtemp(path.join(tmpdir(), 'render-files-')); + await mkdir(path.join(root, '98', 'diagram'), { recursive: true }); + await mkdir(path.join(root, '98', 'ehld'), { recursive: true }); + await writeFile(path.join(root, '98', 'diagram', 'R-HSA-1.json'), '{"nodes":[]}'); + await writeFile(path.join(root, '98', 'ehld', 'R-HSA-2.svg'), ''); + await writeFile(path.join(root, 'outside.txt'), 'not under 98'); +}); +afterAll(() => rm(root, { recursive: true, force: true })); + +const at = (p: string) => new URL(`${DOWNLOAD_ORIGIN}${p}`); + +describe('the download site from a local folder', () => { + it('answers only requests to the download site', () => { + expect(isDownload(at('/98/diagram/R-HSA-1.json'))).toBe(true); + expect(isDownload(new URL('http://site/ContentService/data/database/version'))).toBe(false); + expect(isDownload(new URL('https://download.reactome.org.example/98/x'))).toBe(false); + }); + + it('serves a diagram and an illustration by their path, with their types', async () => { + const diagram = await localAnswer(root, at('/98/diagram/R-HSA-1.json?v=1')); + expect(diagram).toMatchObject({ status: 200, contentType: 'application/json' }); + expect(String(diagram.body)).toBe('{"nodes":[]}'); + const ehld = await localAnswer(root, at('/98/ehld/R-HSA-2.svg')); + expect(ehld).toMatchObject({ status: 200, contentType: 'image/svg+xml' }); + }); + + it('answers 404 for a file the folder lacks, rather than asking the site', async () => { + expect((await localAnswer(root, at('/98/diagram/R-HSA-3.json'))).status).toBe(404); + expect((await localAnswer(root, at('/98/diagram/'))).status).toBe(404); + }); + + it('never reads outside the folder', async () => { + const sibling = path.basename(root); + for (const escape of [ + '/98/../../outside.txt', + '/%2e%2e/outside.txt', + `/../${sibling}/outside.txt`, + '/%2e%2e%2f%2e%2e%2fetc/passwd', + // An encoded slash survives the URL's own normalising and decodes to a + // step out of the folder, to a file that exists. + '/%2e%2e%2foutside.txt', + '/%E0%A4%A', + ]) { + expect((await localAnswer(path.join(root, '98'), at(escape))).status, escape).toBe(404); + } + }); +}); diff --git a/tools/render/local-files.ts b/tools/render/local-files.ts new file mode 100644 index 00000000..64ef510f --- /dev/null +++ b/tools/render/local-files.ts @@ -0,0 +1,56 @@ +/** + * The release's files from a local folder, in place of the download site. + * + * The render page fetches a diagram's layout (`{release}/diagram/{id}.json` and + * `.graph.json`) and an illustration (`{release}/ehld/{id}.svg`) from the + * download site, as a reader's browser does. A reader's browser should: that is + * what the site's CDN is for. A render should not: it is a server drawing in + * bulk, and for a release step the release's files are not on the download site + * until the release is published. So when a folder is given, every request the + * page makes to the download site is answered from it, laid out as the site is + * (`/98/diagram/R-HSA-69620.json`), and a file the folder lacks is a + * 404 -- never a request to the site. + */ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +/** The download site, as the site's profiles name it (environments.ts, `S3`). */ +export const DOWNLOAD_ORIGIN = 'https://download.reactome.org'; + +const TYPES: Record = { + '.json': 'application/json', + '.svg': 'image/svg+xml', + '.txt': 'text/plain; charset=utf-8', +}; + +/** Whether a request is one the folder answers: anything on the download site. */ +export function isDownload(url: URL): boolean { + return url.origin === DOWNLOAD_ORIGIN; +} + +/** + * The answer to a download-site request from `root`: the file at the URL's + * path beneath it, or a 404. A path that would leave the folder is a 404 too. + */ +export async function localAnswer( + root: string, + url: URL +): Promise<{ status: number; contentType: string; body: Buffer | string }> { + const missing = { status: 404, contentType: 'text/plain', body: 'not in the local files' }; + const base = path.resolve(root); + let relative: string; + try { + relative = decodeURIComponent(url.pathname); + } catch { + return missing; + } + const file = path.resolve(base, `.${relative}`); + if (!file.startsWith(base + path.sep)) return missing; + try { + const body = await readFile(file); + const contentType = TYPES[path.extname(file).toLowerCase()] ?? 'application/octet-stream'; + return { status: 200, contentType, body }; + } catch { + return missing; + } +} diff --git a/tools/render/render-core.spec.ts b/tools/render/render-core.spec.ts index 5521e418..86c4db0f 100644 --- a/tools/render/render-core.spec.ts +++ b/tools/render/render-core.spec.ts @@ -75,3 +75,42 @@ describe('a figure of another species', () => { ).rejects.toThrow('asked for species 10090, the page drew 9606'); }); }); + +describe("the release's files", () => { + // A stand-in page that records its routes and stops at the first navigation. + function routed() { + const routes: { matcher: unknown; handler: unknown }[] = []; + const page = { + on() {}, + off() {}, + async route(matcher: unknown, handler: unknown) { + routes.push({ matcher, handler }); + }, + async unroute(matcher: unknown, handler: unknown) { + const i = routes.findIndex((r) => r.matcher === matcher && r.handler === handler); + if (i >= 0) routes.splice(i, 1); + }, + async goto() { + seen = routes.length; + throw new Error('stop here'); + }, + }; + let seen = 0; + return { page, routes, seen: () => seen }; + } + + it('are read from the folder while the page draws, and the route removed after', async () => { + const { page, routes, seen } = routed(); + await render(stand(page), { base: 'http://site', pathway: 'R-HSA-1', files: '/files' }).catch( + () => undefined + ); + expect(seen()).toBe(1); + expect(routes).toHaveLength(0); + }); + + it('are left to the download site when no folder is given', async () => { + const { page, seen } = routed(); + await render(stand(page), { base: 'http://site', pathway: 'R-HSA-1' }).catch(() => undefined); + expect(seen()).toBe(0); + }); +}); diff --git a/tools/render/render-core.ts b/tools/render/render-core.ts index 0516119a..4f486d32 100644 --- a/tools/render/render-core.ts +++ b/tools/render/render-core.ts @@ -5,8 +5,9 @@ * "wait for the page, ask it for the artefact" -- the whole point of this work * is not having two renderers, and that argument applies to its callers too. */ -import type { ConsoleMessage, Page } from '@playwright/test'; +import type { ConsoleMessage, Page, Route } from '@playwright/test'; import { gifFromPage, DEFAULT_DELAY, MAX_SIZE } from './gif.ts'; +import { isDownload, localAnswer } from './local-files.ts'; import { pptx } from './pptx.ts'; /** Formats the render page can produce. */ @@ -118,6 +119,11 @@ export interface RenderOptions { delay?: number; maxSize?: number; timeout?: number; + /** + * A folder laid out as the download site, answering the page's requests to it + * (local-files.ts). Unset, the page fetches from the site as a browser does. + */ + files?: string; } /** What the export adds to the page's report: the shapes, the picture, the frames. */ @@ -166,6 +172,7 @@ export async function render( delay = DEFAULT_DELAY, maxSize = MAX_SIZE, timeout = 120_000, + files, }: RenderOptions ): Promise<{ bytes: Buffer; state: RenderReport; problems: string[] }> { if (!isFormat(format)) { @@ -179,6 +186,12 @@ export async function render( }; page.on('pageerror', onPageError); page.on('console', onConsole); + const fromFiles = async (route: Route) => { + if (!files) return route.continue(); + const { status, contentType, body } = await localAnswer(files, new URL(route.request().url())); + return route.fulfill({ status, contentType, body }); + }; + if (files) await page.route(isDownload, fromFiles); try { await page.goto( @@ -257,6 +270,7 @@ export async function render( } finally { page.off('pageerror', onPageError); page.off('console', onConsole); + if (files) await page.unroute(isDownload, fromFiles); } } diff --git a/tools/render/render.ts b/tools/render/render.ts index f4de8d12..969f41e1 100755 --- a/tools/render/render.ts +++ b/tools/render/render.ts @@ -23,6 +23,7 @@ * --species genome-wide only: whose overview, by taxId (default: the site's) * --layout reaction, to draw a reaction's own layout rather than a diagram * --highlight comma-separated events to mark on the whole diagram, nothing framed + * --files a folder laid out as the download site, read instead of it (local-files.ts) * * GIF animates one frame per sample of an expression analysis, so it wants a * --token; without one it is a single frame. PPTX is drawn as shapes, one per @@ -68,6 +69,7 @@ const KNOWN = [ 'layout', 'highlight', 'species', + 'files', ]; const unknown = args.filter((arg) => arg.startsWith('--') && !KNOWN.includes(arg.slice(2))); if (unknown.length) { @@ -124,6 +126,7 @@ try { layout: flag('layout', ''), highlight: flag('highlight', ''), species: flag('species', ''), + ...(flag('files', '') ? { files: flag('files', '') } : {}), }); await writeFile(out, bytes); diff --git a/tools/render/service.ts b/tools/render/service.ts index b481d228..43f6c9e0 100644 --- a/tools/render/service.ts +++ b/tools/render/service.ts @@ -104,6 +104,10 @@ const CACHE = process.env['RENDER_CACHE'] || path.resolve('.render-cache'); // result settles on, before it is ready: a figure with a token cached under v11 // may be uncoloured, or coloured for TOTAL where the browser shows another. const CACHE_KEY = process.env['RENDER_CACHE_KEY'] || 'v12'; +// The release's diagram and illustration files, laid out as the download site, +// for the page to read instead of the site (local-files.ts). Unset, it fetches +// them from the site, as a reader's browser does. +const FILES = process.env['RENDER_FILES'] || undefined; const CONCURRENCY = Number(process.env['RENDER_CONCURRENCY'] || 2); // Generous next to a real render, which is 3-8s, but far short of the two // minutes a page that never becomes ready would otherwise hold a browser for. @@ -434,6 +438,7 @@ async function renderCached(params: RenderParams) { const { bytes, state, problems } = await render(page, { base: BASE, timeout: RENDER_TIMEOUT, + ...(FILES ? { files: FILES } : {}), ...params, }); await toCache(key, params.format, bytes); @@ -518,6 +523,7 @@ app.get('/health', (_req, res) => { base: BASE, cache: CACHE, cacheKey: CACHE_KEY, + files: FILES ? 'local' : 'download site', active, waiting: waiting.length, ...stats,