From 02f03f111ac989f4913476dfdbb33cfe18736bdf Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sun, 22 Nov 2026 16:09:32 +0700 Subject: [PATCH] fix(email): lengthen claim timeout with abort logging and harden prod DB JWT ref check (#273) --- apps/web/src/server/email/outbox.test.ts | 54 ++++++++++++++++++++++ apps/web/src/server/email/outbox.ts | 35 +++++++++++--- apps/web/src/server/email/provider.test.ts | 52 +++++++++++++++++++++ apps/web/src/server/email/provider.ts | 37 +++++++++++++-- 4 files changed, 168 insertions(+), 10 deletions(-) diff --git a/apps/web/src/server/email/outbox.test.ts b/apps/web/src/server/email/outbox.test.ts index cd5c2759..4bc2f8f7 100644 --- a/apps/web/src/server/email/outbox.test.ts +++ b/apps/web/src/server/email/outbox.test.ts @@ -4,8 +4,10 @@ import { fakeDb, op, type Call, type Handler } from "@/test/fake-db" import { CLAIM_BATCH, + CLAIM_TIMEOUT_MS, COMPLETE_TRIES, RPC_TIMEOUT_MS, + RUN_BUDGET_MS, RUN_LIMIT_MS, SEND_DEADLINE_MS, SEND_TIMEOUT_MS, @@ -339,6 +341,58 @@ describe("sendDueEmails", () => { await expect(sendDueEmails(db as never, transport, OPTIONS)).rejects.toMatchObject({ code: "42501" }) }) + it("uses a 15 s claim timeout above PostgREST's 8 s statement timeout and logs aborted claims (DOS/Give#273)", async () => { + expect(CLAIM_TIMEOUT_MS).toBe(15_000) + expect(CLAIM_TIMEOUT_MS).toBeLessThan(RUN_BUDGET_MS) + + const errorSpy = vi.mocked(console.error) + errorSpy.mockClear() + const abortDb = fakeDb( + () => ({ error: { code: "", message: "TimeoutError: The operation was aborted due to timeout" } }), + calls + ) + await expect(sendDueEmails(abortDb as never, transport, OPTIONS)).rejects.toMatchObject({ + message: expect.stringMatching(/aborted due to timeout/), + }) + expect(errorSpy).toHaveBeenCalledWith( + expect.stringMatching(/claim_outbox aborted after timeout; DB transaction may still have committed/), + "", + expect.stringMatching(/TimeoutError/) + ) + + // Subsequent claim abort logs the committed-lease warning and stops without losing earlier counts. + errorSpy.mockClear() + let claimCount = 0 + const secondAbortDb = fakeDb((call) => { + if (call.target === "rpc:claim_outbox") { + claimCount += 1 + if (claimCount === 1) return { data: Array.from({ length: CLAIM_BATCH }, (_, i) => row(100 + i)) } + return { error: { code: "", message: "AbortError: This operation was aborted" } } + } + return handler(call) + }, calls) + const outcome = await sendDueEmails(secondAbortDb as never, transport, OPTIONS) + expect(outcome).toMatchObject({ claimed: CLAIM_BATCH, sent: CLAIM_BATCH }) + expect(errorSpy).toHaveBeenCalledWith( + expect.stringMatching(/claim_outbox aborted after timeout; DB transaction may still have committed/), + "", + expect.stringMatching(/AbortError/) + ) + + // Server-side Postgres statement_timeout (57014) rolled back cleanly: log ordinary failure, not abort warning. + errorSpy.mockClear() + const pgTimeoutDb = fakeDb( + () => ({ error: { code: "57014", message: "canceling statement due to statement timeout" } }), + calls + ) + await expect(sendDueEmails(pgTimeoutDb as never, transport, OPTIONS)).rejects.toMatchObject({ code: "57014" }) + expect(errorSpy).toHaveBeenCalledWith( + "claim_outbox failed", + "57014", + "canceling statement due to statement timeout" + ) + }) + it("skips a malformed row", async () => { claims = [[{ id: "x" }, row(8)]] const outcome = await sendDueEmails(fakeDb(handler, calls) as never, transport, OPTIONS) diff --git a/apps/web/src/server/email/outbox.ts b/apps/web/src/server/email/outbox.ts index cc299b0b..316828a1 100644 --- a/apps/web/src/server/email/outbox.ts +++ b/apps/web/src/server/email/outbox.ts @@ -40,8 +40,12 @@ const COMPLETE_RETRY_MS = [250, 1_000] const NO_HOLD_FUNCTION = new Set(["PGRST202", "42883"]) /** Each RPC after a send (complete, release) is abandoned after this long. */ export const RPC_TIMEOUT_MS = 5_000 -/** A claim is abandoned after this long (it may wait on locks while it settles rows). */ -export const CLAIM_TIMEOUT_MS = 10_000 +/** + * A claim is abandoned after this long. Kept well above PostgREST's 8 s + * statement timeout plus connection-pooler / network jitter so a committing + * claim_outbox call is not aborted in flight (DOS/Give#273 item 5). + */ +export const CLAIM_TIMEOUT_MS = 15_000 /** * No retry or release starts unless it can finish by this point, under the * 60 s function limit: a run killed mid-batch loses its results and leaves @@ -296,6 +300,18 @@ async function release(db: Db, rows: OutboxEmail[], clock: Clock): Promise { missing: ["NON_PRODUCTION_DEPLOYMENT_ON_PRODUCTION_DATABASE"], }) }) + + it("catches the Production database via JWT ref even behind an unlisted custom domain (DOS/Give#273)", () => { + const fakeJwt = (payload: unknown) => + `eyJhbGciOiJIUzI1NiJ9.${Buffer.from(JSON.stringify(payload)).toString("base64url")}.sig` + const prodServiceJwt = fakeJwt({ iss: "supabase", ref: "gulptwduchsjcsbndmua", role: "service_role" }) + const prodAnonJwt = fakeJwt({ iss: "supabase", ref: "GULPTWDUCHSJCSBNDMUA", role: "anon" }) + const betaServiceJwt = fakeJwt({ iss: "supabase", ref: "oyajkbadsykigtfrpdpg", role: "service_role" }) + + expect( + usesProductionDatabase({ + NEXT_PUBLIC_SUPABASE_URL: "https://db.custom-dos.example", + SUPABASE_SERVICE_ROLE_KEY: prodServiceJwt, + }) + ).toBe(true) + expect( + usesProductionDatabase({ + NEXT_PUBLIC_SUPABASE_URL: "https://db.custom-dos.example", + NEXT_PUBLIC_SUPABASE_ANON_KEY: prodAnonJwt, + }) + ).toBe(true) + expect(usesProductionDatabase({ SUPABASE_SERVICE_ROLE_KEY: prodServiceJwt })).toBe(true) + expect( + emailSettingsFromEnv({ + ...BETA, + NEXT_PUBLIC_SUPABASE_URL: "https://db.custom-dos.example", + SUPABASE_SERVICE_ROLE_KEY: prodServiceJwt, + }) + ).toMatchObject({ + ok: false, + missing: ["NON_PRODUCTION_DEPLOYMENT_ON_PRODUCTION_DATABASE"], + }) + + // Non-production JWT and opaque sb_secret_ keys do not trigger false positives. + expect( + usesProductionDatabase({ + NEXT_PUBLIC_SUPABASE_URL: "https://db.custom-beta.example", + SUPABASE_SERVICE_ROLE_KEY: betaServiceJwt, + }) + ).toBe(false) + expect( + usesProductionDatabase({ + NEXT_PUBLIC_SUPABASE_URL: "https://db.custom-beta.example", + SUPABASE_SERVICE_ROLE_KEY: "sb_secret_not_a_jwt", + }) + ).toBe(false) + expect( + usesProductionDatabase({ + NEXT_PUBLIC_SUPABASE_URL: "https://db.custom-beta.example", + SUPABASE_SERVICE_ROLE_KEY: "a.not-valid-json.c", + }) + ).toBe(false) + }) }) diff --git a/apps/web/src/server/email/provider.ts b/apps/web/src/server/email/provider.ts index 3c919e29..8e51d703 100644 --- a/apps/web/src/server/email/provider.ts +++ b/apps/web/src/server/email/provider.ts @@ -71,15 +71,44 @@ export function emailLinkOrigin(env: Env = process.env): string | null { return production ? PRODUCTION_ORIGIN : null } +/** + * Extracts the Supabase project ref from a legacy JWT key (`header.payload.sig`), + * or null for opaque keys (`sb_secret_*` / `sb_publishable_*`) and invalid strings. + */ +function jwtProjectRef(token: string | undefined): string | null { + if (!token) return null + const parts = token.trim().split(".") + if (parts.length !== 3 || !parts[1]) return null + try { + const payload: unknown = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) + if (!payload || typeof payload !== "object") return null + const ref = (payload as { ref?: unknown }).ref + return typeof ref === "string" ? ref.trim().toLowerCase() : null + } catch { + return null + } +} + +/** + * Whether env points at the Production DOS Supabase project (DOS/Give#273 item 6). + * Checks both the URL host (project host + vanity domains in PRODUCTION_SUPABASE_HOSTS) + * and the JWT `ref` claim of the Supabase keys so a newly added custom domain is + * still caught when JWT keys are used. We intentionally do not query the DB for an + * environment tag: Production, Beta, and Preview branches replay the same committed + * DOS.Me migrations, so an in-DB marker would be inherited by Preview/Beta. + */ export function usesProductionDatabase(env: Env = process.env): boolean { - let host: string try { // A trailing dot (fully qualified name) reaches the same host. - host = new URL(env.NEXT_PUBLIC_SUPABASE_URL ?? "").hostname.toLowerCase().replace(/\.+$/, "") + const host = new URL(env.NEXT_PUBLIC_SUPABASE_URL ?? "").hostname.toLowerCase().replace(/\.+$/, "") + if (PRODUCTION_SUPABASE_HOSTS.includes(host) || host.split(".")[0] === PRODUCTION_SUPABASE_REF) return true } catch { - return false + // Missing or malformed URL; still check JWT keys below. } - return PRODUCTION_SUPABASE_HOSTS.includes(host) || host.split(".")[0] === PRODUCTION_SUPABASE_REF + return ( + jwtProjectRef(env.SUPABASE_SERVICE_ROLE_KEY) === PRODUCTION_SUPABASE_REF || + jwtProjectRef(env.NEXT_PUBLIC_SUPABASE_ANON_KEY) === PRODUCTION_SUPABASE_REF + ) } export function emailDelivery(env: Env = process.env): EmailDelivery {