Fix production 5xx: cut per-request CPU under the Workers Free plan limit

backend/scripts/verify_cloudflare_config.py

@@ -2,10 +2,23 @@
 
 import json
 import sys
-import tomllib
 from pathlib import Path
 from typing import Any
 
+try:
+    import tomllib
+except ModuleNotFoundError:  # Python < 3.11; tomli is the same parser, backported.
+    try:
+        import tomli as tomllib
+    except ModuleNotFoundError as exc:
+        # This runs as predeploy:backend. A bare ImportError here aborts the
+        # deploy before a single check runs, which reads like a broken script
+        # rather than a missing dependency.
+        raise SystemExit(
+            'verify_cloudflare_config needs a TOML parser: use Python 3.11+ '
+            '(stdlib tomllib) or run `pip install tomli`.'
+        ) from exc
+
 REPO_ROOT = Path(__file__).resolve().parents[2]
 
 # Production cutover to studyplanner-db happens with the integrate_new_db branch

backend/src/http_utils.py

@@ -6,6 +6,7 @@
 from workers import Response
 
 from env_config import get_allowed_origins, is_origin_allowed
+from isolate_identity import get_isolate_id, isolate_age_ms, next_response_sequence
 
 
 def get_request_header(request: Any, header_name: str) -> str | None:
@@ -33,6 +34,15 @@ def build_cors_headers(request: Any, env: Any) -> dict[str, str]:
     headers: dict[str, str] = {
         "access-control-allow-methods": "GET,POST,PUT,PATCH,DELETE,OPTIONS",
         "access-control-allow-headers": "Authorization, Content-Type, X-CSRF-Token",
+        # Diagnostic; see isolate_identity. Every response path funnels through
+        # here, which is why the marker is attached at this point rather than in
+        # each responder.
+        "x-isolate-id": get_isolate_id(),
+        "x-isolate-seq": str(next_response_sequence()),
+        "x-isolate-age-ms": str(isolate_age_ms()),
+        # Without this a browser cannot read the above cross-origin, and the
+        # deployed frontend is cross-origin to this Worker.
+        "access-control-expose-headers": "x-isolate-id, x-isolate-seq, x-isolate-age-ms",
     }
 
     if "*" in allowed_origins:
@@ -60,7 +70,38 @@ def json_response(
     if extra_headers:
         headers.update(extra_headers)
 
-    body = json.dumps(payload, ensure_ascii=False)
+    # Encoded here rather than handed over as `str`. Returning a Python string
+    # makes Pyodide convert it at the JS boundary, and that conversion dominates
+    # this Worker's CPU: measured on the real catalog endpoint, a ~500 KB
+    # response cost 98.6 ms of CPU as a `str` and the same bytes cost a small
+    # fraction of that pre-encoded. Since an isolate is killed (1102) once it has
+    # burned roughly two seconds of CPU in total, and then serves 1101s to
+    # everything routed to it afterwards, that conversion was the app's main
+    # source of production 5xx. See docs/load-test-2026-08.md.
+    body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
+    return Response(body, status=status, headers=headers)
+
+
+def encoded_json_response(
+    body: bytes,
+    request: Any,
+    env: Any,
+    status: int = 200,
+    extra_headers: dict[str, str] | None = None,
+) -> Response:
+    """Return an already-serialised JSON body.
+
+    Exists so a caller that has cached the encoded bytes can skip both
+    `json.dumps` and the encode, which is the whole point of caching them.
+    Headers are still built per request, because they carry per-isolate
+    diagnostics and the CORS origin.
+    """
+    headers = {
+        "content-type": "application/json; charset=utf-8",
+        **build_cors_headers(request, env),
+    }
+    if extra_headers:
+        headers.update(extra_headers)
     return Response(body, status=status, headers=headers)
 
 

backend/src/isolate_identity.py

@@ -0,0 +1,64 @@
+"""Per-isolate identity, so a response can be attributed to the isolate that served it.
+
+Cloudflare exposes no isolate identifier. Diagnosing the wedge recorded in
+docs/load-test-2026-08.md needs one: 56 consecutive failures arrived on a single
+TLS connection while nineteen other connections were served normally, and the
+open question is whether the isolate behind that connection was itself dead or
+was fine and only that connection was stuck to it. Those have different causes.
+
+**The id must be generated lazily, not at module scope.** Module-level code runs
+once and is captured in the Pyodide startup snapshot, which every isolate then
+restores — so a module-scope value is *identical* across isolates and identifies
+nothing. Cloudflare enforces this directly: `secrets.token_hex()` at import time
+fails with
+
+    OSError: [Errno 29] Cannot get entropy outside of request context
+
+Generating on first use puts it inside a request context (legal) and after the
+snapshot was taken (correct): the snapshot holds `None`, and each restored
+isolate fills it in once.
+
+Diagnostic only — nothing reads these headers to make a decision.
+"""
+
+from __future__ import annotations
+
+import secrets
+import time
+
+_isolate_id: str | None = None
+_first_seen_at: float | None = None
+_responses_served = 0
+
+
+def get_isolate_id() -> str:
+    """Stable for the life of one isolate, different across isolates."""
+    global _isolate_id
+    if _isolate_id is None:
+        _isolate_id = secrets.token_hex(8)
+    return _isolate_id
+
+
+def next_response_sequence() -> int:
+    """Position of this response in the isolate's lifetime, starting at 1.
+
+    Counts responses rather than requests: it is incremented where headers are
+    built, so a request that hangs before that never takes a number and leaves a
+    gap. The gap is itself the signal.
+    """
+    global _responses_served
+    _responses_served += 1
+    return _responses_served
+
+
+def isolate_age_ms() -> int:
+    """Milliseconds since this isolate served its first response.
+
+    Measured from first use rather than from module import, for the same reason
+    the id is: import time is baked into the snapshot and is the same everywhere.
+    """
+    global _first_seen_at
+    now = time.time()
+    if _first_seen_at is None:
+        _first_seen_at = now
+    return int((now - _first_seen_at) * 1000)

backend/src/router.py

@@ -1,12 +1,20 @@
 from __future__ import annotations
 
+import json
 import traceback
 from typing import Any
 from urllib.parse import parse_qs, unquote, urlparse
 
 from db.d1 import D1ExecutionError, fetch_all, fetch_one, has_database
-from http_utils import empty_response, error_response, html_response, json_response
+from http_utils import (
+    empty_response,
+    encoded_json_response,
+    error_response,
+    html_response,
+    json_response,
+)
 from request_utils import RequestBodyError, read_json_object
+from services import catalog_response_cache
 from services.authentication import (
     AuthConfigurationError,
     AuthenticationError,
@@ -725,17 +733,36 @@ async def route_request(request: Any, env: Any) -> Any:
             except ValueError:
                 limit = 100
 
+            # Searches are cached too. A broad two-character prefix against the
+            # whole catalog costs ~230 ms of CPU, and those prefixes are exactly
+            # what users type first, so they are the entries most worth keeping.
+            cache_key = catalog_response_cache.build_key(limit, period_value, search_value)
+            cached_body = catalog_response_cache.get(cache_key)
+            if cached_body is not None:
+                return encoded_json_response(
+                    cached_body,
+                    request=request,
+                    env=env,
+                    extra_headers=_PUBLIC_CATALOG_CACHE_HEADERS,
+                )
+
             courses = await list_catalog_courses(
                 env,
                 limit=limit,
                 search=search_value,
                 period_id=period_value,
             )
-            return json_response(
-                {
-                    "count": len(courses),
-                    "courses": courses,
-                },
+            payload = {
+                "count": len(courses),
+                "courses": courses,
+            }
+            # Rebuilding this answer is what kills isolates on the Free plan: it
+            # is the same bytes every time, and it costs ~350-500 ms of CPU for
+            # the unfiltered catalog. See services/catalog_response_cache.
+            encoded = json.dumps(payload, ensure_ascii=False).encode("utf-8")
+            catalog_response_cache.put(cache_key, encoded)
+            return encoded_json_response(
+                encoded,
                 request=request,
                 env=env,
                 extra_headers=_PUBLIC_CATALOG_CACHE_HEADERS,

backend/src/services/catalog_response_cache.py

@@ -0,0 +1,97 @@
+"""Per-isolate cache of already-serialised catalog responses.
+
+Why this exists
+---------------
+On the Workers Free plan each isolate is killed once it has accumulated about
+2000 ms of CPU *above* the 10 ms-per-request limit, and it stays dead — later
+requests routed to it fail. Measured, `/api/catalog/courses?limit=1000&period=all`
+costs roughly 350-500 ms of CPU, so it destroys a fresh isolate after five or six
+requests. Since the frontend issues that request on first load, a handful of
+arriving users is enough to take isolates down. See `docs/load-test-2026-08.md`.
+
+Almost all of that cost is rebuilding the same answer: the D1 round-trips plus
+`course_catalog._build_catalog_summary`, which runs per course. The catalog is a
+snapshot that only changes when someone re-imports it, so the work is identical
+every time.
+
+Caching the *encoded* bytes (not the payload object, and not a `str`) skips both
+the rebuild and the UTF-8 conversion at the Python/JS boundary. Measured on an
+equivalent 1.43 MB payload: rebuilding each time exhausted an isolate, while
+serving pre-encoded bytes survived 150 consecutive requests untouched.
+
+Staleness
+---------
+Entries live as long as the isolate does, which is minutes to hours, and isolates
+are replaced continuously. The endpoint already advertises
+`cache-control: public, max-age=300`, so callers are told to expect data up to
+five minutes old; this does not promise anything stronger. After a catalog
+re-import, responses can lag until isolates recycle — deploy the Worker to force
+it.
+"""
+
+from __future__ import annotations
+
+# Searches are cached too, so the key space is caller-controlled and the bound
+# has to be on bytes rather than entries: one entry can be 1.5 MB and another
+# 20 KB. Isolates tolerate far more than this (measured: 179 MB of resident
+# allocation), so the budget is set for politeness, not survival.
+_MAX_BYTES = 16 * 1024 * 1024
+_MAX_ENTRIES = 64
+
+_entries: dict[str, bytes] = {}
+_total_bytes = 0
+
+
+def build_key(limit: int, period_id: str | None, search: str | None = None) -> str:
+    """Identify one cacheable response.
+
+    Search terms are normalised the way the query is, so that `Info`, `info ` and
+    `info` share an entry — broad prefixes are both the most expensive responses
+    to build and the ones most users type, which is what makes caching them
+    worthwhile.
+    """
+    normalized_search = (search or "").strip().lower()
+    return f"{limit}|{period_id or ''}|{normalized_search}"
+
+
+def get(key: str) -> bytes | None:
+    return _entries.get(key)
+
+
+def put(key: str, body: bytes) -> None:
+    """Store one encoded response, evicting oldest-first to stay within budget.
+
+    Insertion order is dict order in CPython, so the first key is the oldest. A
+    body larger than the whole budget is simply not cached, rather than being
+    stored and immediately evicting everything else.
+    """
+    global _total_bytes
+    if len(body) > _MAX_BYTES:
+        return
+
+    existing = _entries.pop(key, None)
+    if existing is not None:
+        _total_bytes -= len(existing)
+
+    _entries[key] = body
+    _total_bytes += len(body)
+
+    while _entries and (_total_bytes > _MAX_BYTES or len(_entries) > _MAX_ENTRIES):
+        oldest = next(iter(_entries))
+        if oldest == key:
+            break
+        _total_bytes -= len(_entries.pop(oldest))
+
+
+def clear() -> None:
+    global _total_bytes
+    _entries.clear()
+    _total_bytes = 0
+
+
+def size() -> int:
+    return len(_entries)
+
+
+def total_bytes() -> int:
+    return _total_bytes