Render service: read the release's files from a local folder (RENDER_FILES)

tools/render/README.md

@@ -254,6 +254,26 @@ curl -s http://127.0.0.1:4310/health
 | `RENDER_CONCURRENCY` | 2                     | simultaneous renders                                                       |
 | `RENDER_QUEUE`       | 8                     | pending renders before 503                                                 |
 | `RENDER_TIMEOUT`     | 45000                 | ms before a render is abandoned                                            |
+| `RENDER_FILES`       | unset                 | a folder laid out as the download site; see below                          |
+
+**The release's files.** The render page fetches a diagram's layout
+(`{release}/diagram/{id}.json`, `.graph.json`) and an illustration
+(`{release}/ehld/{id}.svg`) from the download site, as a reader's browser does.
+With `RENDER_FILES` set (`--files` for the CLI), every request the page makes to
+the download site is answered from that folder instead, laid out as the site is
+(`<folder>/98/diagram/R-HSA-73857.json`), and a file the folder lacks is a 404,
+never a request to the site. Set it for any bulk run, and for a release's own
+figures, whose files are not on the download site until the release is
+published. In a container, mount the folder read-only and point the variable at
+it:
+
+```sh
+docker run ... -v /path/to/release-files:/files:ro -e RENDER_FILES=/files websiteangular-render
+```
+
+A file missing from the folder is not reported as such by the page: the render
+waits out `RENDER_TIMEOUT`. So check the folder holds every file before a bulk
+run. `/health` says which source is in use (`files`).
 
 Query parameters: `token`, `scale`, `subpathways=false`, and for GIF `delay`
 (ms per frame) and `maxSize` (longest side). All of them are part of the cache

tools/render/local-files.spec.ts

@@ -0,0 +1,55 @@
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { DOWNLOAD_ORIGIN, isDownload, localAnswer } from './local-files.ts';
+
+let root: string;
+beforeAll(async () => {
+  root = await mkdtemp(path.join(tmpdir(), 'render-files-'));
+  await mkdir(path.join(root, '98', 'diagram'), { recursive: true });
+  await mkdir(path.join(root, '98', 'ehld'), { recursive: true });
+  await writeFile(path.join(root, '98', 'diagram', 'R-HSA-1.json'), '{"nodes":[]}');
+  await writeFile(path.join(root, '98', 'ehld', 'R-HSA-2.svg'), '<svg/>');
+  await writeFile(path.join(root, 'outside.txt'), 'not under 98');
+});
+afterAll(() => rm(root, { recursive: true, force: true }));
+
+const at = (p: string) => new URL(`${DOWNLOAD_ORIGIN}${p}`);
+
+describe('the download site from a local folder', () => {
+  it('answers only requests to the download site', () => {
+    expect(isDownload(at('/98/diagram/R-HSA-1.json'))).toBe(true);
+    expect(isDownload(new URL('http://site/ContentService/data/database/version'))).toBe(false);
+    expect(isDownload(new URL('https://download.reactome.org.example/98/x'))).toBe(false);
+  });
+
+  it('serves a diagram and an illustration by their path, with their types', async () => {
+    const diagram = await localAnswer(root, at('/98/diagram/R-HSA-1.json?v=1'));
+    expect(diagram).toMatchObject({ status: 200, contentType: 'application/json' });
+    expect(String(diagram.body)).toBe('{"nodes":[]}');
+    const ehld = await localAnswer(root, at('/98/ehld/R-HSA-2.svg'));
+    expect(ehld).toMatchObject({ status: 200, contentType: 'image/svg+xml' });
+  });
+
+  it('answers 404 for a file the folder lacks, rather than asking the site', async () => {
+    expect((await localAnswer(root, at('/98/diagram/R-HSA-3.json'))).status).toBe(404);
+    expect((await localAnswer(root, at('/98/diagram/'))).status).toBe(404);
+  });
+
+  it('never reads outside the folder', async () => {
+    const sibling = path.basename(root);
+    for (const escape of [
+      '/98/../../outside.txt',
+      '/%2e%2e/outside.txt',
+      `/../${sibling}/outside.txt`,
+      '/%2e%2e%2f%2e%2e%2fetc/passwd',
+      // An encoded slash survives the URL's own normalising and decodes to a
+      // step out of the folder, to a file that exists.
+      '/%2e%2e%2foutside.txt',
+      '/%E0%A4%A',
+    ]) {
+      expect((await localAnswer(path.join(root, '98'), at(escape))).status, escape).toBe(404);
+    }
+  });
+});

tools/render/local-files.ts

@@ -0,0 +1,56 @@
+/**
+ * The release's files from a local folder, in place of the download site.
+ *
+ * The render page fetches a diagram's layout (`{release}/diagram/{id}.json` and
+ * `.graph.json`) and an illustration (`{release}/ehld/{id}.svg`) from the
+ * download site, as a reader's browser does. A reader's browser should: that is
+ * what the site's CDN is for. A render should not: it is a server drawing in
+ * bulk, and for a release step the release's files are not on the download site
+ * until the release is published. So when a folder is given, every request the
+ * page makes to the download site is answered from it, laid out as the site is
+ * (`<folder>/98/diagram/R-HSA-69620.json`), and a file the folder lacks is a
+ * 404 -- never a request to the site.
+ */
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+/** The download site, as the site's profiles name it (environments.ts, `S3`). */
+export const DOWNLOAD_ORIGIN = 'https://download.reactome.org';
+
+const TYPES: Record<string, string> = {
+  '.json': 'application/json',
+  '.svg': 'image/svg+xml',
+  '.txt': 'text/plain; charset=utf-8',
+};
+
+/** Whether a request is one the folder answers: anything on the download site. */
+export function isDownload(url: URL): boolean {
+  return url.origin === DOWNLOAD_ORIGIN;
+}
+
+/**
+ * The answer to a download-site request from `root`: the file at the URL's
+ * path beneath it, or a 404. A path that would leave the folder is a 404 too.
+ */
+export async function localAnswer(
+  root: string,
+  url: URL
+): Promise<{ status: number; contentType: string; body: Buffer | string }> {
+  const missing = { status: 404, contentType: 'text/plain', body: 'not in the local files' };
+  const base = path.resolve(root);
+  let relative: string;
+  try {
+    relative = decodeURIComponent(url.pathname);
+  } catch {
+    return missing;
+  }
+  const file = path.resolve(base, `.${relative}`);
+  if (!file.startsWith(base + path.sep)) return missing;
+  try {
+    const body = await readFile(file);
+    const contentType = TYPES[path.extname(file).toLowerCase()] ?? 'application/octet-stream';
+    return { status: 200, contentType, body };
+  } catch {
+    return missing;
+  }
+}

tools/render/render-core.spec.ts

@@ -75,3 +75,42 @@ describe('a figure of another species', () => {
     ).rejects.toThrow('asked for species 10090, the page drew 9606');
   });
 });
+
+describe("the release's files", () => {
+  // A stand-in page that records its routes and stops at the first navigation.
+  function routed() {
+    const routes: { matcher: unknown; handler: unknown }[] = [];
+    const page = {
+      on() {},
+      off() {},
+      async route(matcher: unknown, handler: unknown) {
+        routes.push({ matcher, handler });
+      },
+      async unroute(matcher: unknown, handler: unknown) {
+        const i = routes.findIndex((r) => r.matcher === matcher && r.handler === handler);
+        if (i >= 0) routes.splice(i, 1);
+      },
+      async goto() {
+        seen = routes.length;
+        throw new Error('stop here');
+      },
+    };
+    let seen = 0;
+    return { page, routes, seen: () => seen };
+  }
+
+  it('are read from the folder while the page draws, and the route removed after', async () => {
+    const { page, routes, seen } = routed();
+    await render(stand(page), { base: 'http://site', pathway: 'R-HSA-1', files: '/files' }).catch(
+      () => undefined
+    );
+    expect(seen()).toBe(1);
+    expect(routes).toHaveLength(0);
+  });
+
+  it('are left to the download site when no folder is given', async () => {
+    const { page, seen } = routed();
+    await render(stand(page), { base: 'http://site', pathway: 'R-HSA-1' }).catch(() => undefined);
+    expect(seen()).toBe(0);
+  });
+});

tools/render/render-core.ts

@@ -5,8 +5,9 @@
  * "wait for the page, ask it for the artefact" -- the whole point of this work
  * is not having two renderers, and that argument applies to its callers too.
  */
-import type { ConsoleMessage, Page } from '@playwright/test';
+import type { ConsoleMessage, Page, Route } from '@playwright/test';
 import { gifFromPage, DEFAULT_DELAY, MAX_SIZE } from './gif.ts';
+import { isDownload, localAnswer } from './local-files.ts';
 import { pptx } from './pptx.ts';
 
 /** Formats the render page can produce. */
@@ -118,6 +119,11 @@ export interface RenderOptions {
   delay?: number;
   maxSize?: number;
   timeout?: number;
+  /**
+   * A folder laid out as the download site, answering the page's requests to it
+   * (local-files.ts). Unset, the page fetches from the site as a browser does.
+   */
+  files?: string;
 }
 
 /** What the export adds to the page's report: the shapes, the picture, the frames. */
@@ -166,6 +172,7 @@ export async function render(
     delay = DEFAULT_DELAY,
     maxSize = MAX_SIZE,
     timeout = 120_000,
+    files,
   }: RenderOptions
 ): Promise<{ bytes: Buffer; state: RenderReport; problems: string[] }> {
   if (!isFormat(format)) {
@@ -179,6 +186,12 @@ export async function render(
   };
   page.on('pageerror', onPageError);
   page.on('console', onConsole);
+  const fromFiles = async (route: Route) => {
+    if (!files) return route.continue();
+    const { status, contentType, body } = await localAnswer(files, new URL(route.request().url()));
+    return route.fulfill({ status, contentType, body });
+  };
+  if (files) await page.route(isDownload, fromFiles);
 
   try {
     await page.goto(
@@ -257,6 +270,7 @@ export async function render(
   } finally {
     page.off('pageerror', onPageError);
     page.off('console', onConsole);
+    if (files) await page.unroute(isDownload, fromFiles);
   }
 }
 

tools/render/render.ts

@@ -23,6 +23,7 @@
  *   --species   genome-wide only: whose overview, by taxId (default: the site's)
  *   --layout    reaction, to draw a reaction's own layout rather than a diagram
  *   --highlight comma-separated events to mark on the whole diagram, nothing framed
+ *   --files     a folder laid out as the download site, read instead of it (local-files.ts)
  *
  * GIF animates one frame per sample of an expression analysis, so it wants a
  * --token; without one it is a single frame. PPTX is drawn as shapes, one per
@@ -68,6 +69,7 @@ const KNOWN = [
   'layout',
   'highlight',
   'species',
+  'files',
 ];
 const unknown = args.filter((arg) => arg.startsWith('--') && !KNOWN.includes(arg.slice(2)));
 if (unknown.length) {
@@ -124,6 +126,7 @@ try {
     layout: flag('layout', ''),
     highlight: flag('highlight', ''),
     species: flag('species', ''),
+    ...(flag('files', '') ? { files: flag('files', '') } : {}),
   });
 
   await writeFile(out, bytes);

tools/render/service.ts

@@ -104,6 +104,10 @@ const CACHE = process.env['RENDER_CACHE'] || path.resolve('.render-cache');
 // result settles on, before it is ready: a figure with a token cached under v11
 // may be uncoloured, or coloured for TOTAL where the browser shows another.
 const CACHE_KEY = process.env['RENDER_CACHE_KEY'] || 'v12';
+// The release's diagram and illustration files, laid out as the download site,
+// for the page to read instead of the site (local-files.ts). Unset, it fetches
+// them from the site, as a reader's browser does.
+const FILES = process.env['RENDER_FILES'] || undefined;
 const CONCURRENCY = Number(process.env['RENDER_CONCURRENCY'] || 2);
 // Generous next to a real render, which is 3-8s, but far short of the two
 // minutes a page that never becomes ready would otherwise hold a browser for.
@@ -434,6 +438,7 @@ async function renderCached(params: RenderParams) {
       const { bytes, state, problems } = await render(page, {
         base: BASE,
         timeout: RENDER_TIMEOUT,
+        ...(FILES ? { files: FILES } : {}),
         ...params,
       });
       await toCache(key, params.format, bytes);
@@ -518,6 +523,7 @@ app.get('/health', (_req, res) => {
     base: BASE,
     cache: CACHE,
     cacheKey: CACHE_KEY,
+    files: FILES ? 'local' : 'download site',
     active,
     waiting: waiting.length,
     ...stats,