Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions apps/web/src/server/email/outbox.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@ import { fakeDb, op, type Call, type Handler } from "@/test/fake-db"

import {
CLAIM_BATCH,
CLAIM_TIMEOUT_MS,
COMPLETE_TRIES,
RPC_TIMEOUT_MS,
RUN_BUDGET_MS,
RUN_LIMIT_MS,
SEND_DEADLINE_MS,
SEND_TIMEOUT_MS,
Expand Down Expand Up @@ -339,6 +341,58 @@ describe("sendDueEmails", () => {
await expect(sendDueEmails(db as never, transport, OPTIONS)).rejects.toMatchObject({ code: "42501" })
})

it("uses a 15 s claim timeout above PostgREST's 8 s statement timeout and logs aborted claims (DOS/Give#273)", async () => {
expect(CLAIM_TIMEOUT_MS).toBe(15_000)
expect(CLAIM_TIMEOUT_MS).toBeLessThan(RUN_BUDGET_MS)

const errorSpy = vi.mocked(console.error)
errorSpy.mockClear()
const abortDb = fakeDb(
() => ({ error: { code: "", message: "TimeoutError: The operation was aborted due to timeout" } }),
calls
)
await expect(sendDueEmails(abortDb as never, transport, OPTIONS)).rejects.toMatchObject({
message: expect.stringMatching(/aborted due to timeout/),
})
expect(errorSpy).toHaveBeenCalledWith(
expect.stringMatching(/claim_outbox aborted after timeout; DB transaction may still have committed/),
"",
expect.stringMatching(/TimeoutError/)
)

// Subsequent claim abort logs the committed-lease warning and stops without losing earlier counts.
errorSpy.mockClear()
let claimCount = 0
const secondAbortDb = fakeDb((call) => {
if (call.target === "rpc:claim_outbox") {
claimCount += 1
if (claimCount === 1) return { data: Array.from({ length: CLAIM_BATCH }, (_, i) => row(100 + i)) }
return { error: { code: "", message: "AbortError: This operation was aborted" } }
}
return handler(call)
}, calls)
const outcome = await sendDueEmails(secondAbortDb as never, transport, OPTIONS)
expect(outcome).toMatchObject({ claimed: CLAIM_BATCH, sent: CLAIM_BATCH })
expect(errorSpy).toHaveBeenCalledWith(
expect.stringMatching(/claim_outbox aborted after timeout; DB transaction may still have committed/),
"",
expect.stringMatching(/AbortError/)
)

// Server-side Postgres statement_timeout (57014) rolled back cleanly: log ordinary failure, not abort warning.
errorSpy.mockClear()
const pgTimeoutDb = fakeDb(
() => ({ error: { code: "57014", message: "canceling statement due to statement timeout" } }),
calls
)
await expect(sendDueEmails(pgTimeoutDb as never, transport, OPTIONS)).rejects.toMatchObject({ code: "57014" })
expect(errorSpy).toHaveBeenCalledWith(
"claim_outbox failed",
"57014",
"canceling statement due to statement timeout"
)
})

it("skips a malformed row", async () => {
claims = [[{ id: "x" }, row(8)]]
const outcome = await sendDueEmails(fakeDb(handler, calls) as never, transport, OPTIONS)
Expand Down
35 changes: 29 additions & 6 deletions apps/web/src/server/email/outbox.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,12 @@ const COMPLETE_RETRY_MS = [250, 1_000]
const NO_HOLD_FUNCTION = new Set(["PGRST202", "42883"])
/** Each RPC after a send (complete, release) is abandoned after this long. */
export const RPC_TIMEOUT_MS = 5_000
/** A claim is abandoned after this long (it may wait on locks while it settles rows). */
export const CLAIM_TIMEOUT_MS = 10_000
/**
* A claim is abandoned after this long. Kept well above PostgREST's 8 s
* statement timeout plus connection-pooler / network jitter so a committing
* claim_outbox call is not aborted in flight (DOS/Give#273 item 5).
*/
export const CLAIM_TIMEOUT_MS = 15_000
/**
* No retry or release starts unless it can finish by this point, under the
* 60 s function limit: a run killed mid-batch loses its results and leaves
Expand Down Expand Up @@ -296,6 +300,18 @@ async function release(db: Db, rows: OutboxEmail[], clock: Clock): Promise<numbe
return released
}

/**
* Whether an RPC error looks like a client-side abort/timeout (as opposed to
* a server-side rollback such as Postgres 57014 statement_timeout), meaning
* the DB transaction may still have committed and leased rows.
*/
function isAbortError(error: { code?: string; name?: string; message?: string; details?: string }): boolean {
if (error.code === "57014") return false
if (error.name === "AbortError" || error.name === "TimeoutError") return true
const text = `${error.message ?? ""} ${error.details ?? ""}`
return /\b(AbortError|TimeoutError|aborted|timed out)\b/i.test(text)
}

/**
* One send-outbox run. Throws only when the first claim fails. After a send
* times out its connection is cut off (transport.abort), the attempt is
Expand All @@ -321,12 +337,19 @@ export async function sendDueEmails(

try {
run: while (now() - started < budget) {
const { data, error } = await db
.rpc("claim_outbox", { p_limit: CLAIM_BATCH })
.abortSignal(AbortSignal.timeout(CLAIM_TIMEOUT_MS))
const claimSignal = AbortSignal.timeout(CLAIM_TIMEOUT_MS)
const { data, error } = await db.rpc("claim_outbox", { p_limit: CLAIM_BATCH }).abortSignal(claimSignal)
if (error) {
if (claimSignal.aborted || isAbortError(error)) {
console.error(
"claim_outbox aborted after timeout; DB transaction may still have committed and leased rows until their 5-minute lease expires",
error.code,
error.message
)
} else {
console.error("claim_outbox failed", error.code, error.message)
}
if (result.claimed === 0) throw error
console.error("claim_outbox failed", error.code, error.message)
break
}
const raws = Array.isArray(data) ? data : []
Expand Down
52 changes: 52 additions & 0 deletions apps/web/src/server/email/provider.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -162,4 +162,56 @@ describe("a deployment outside Production on the Production database", () => {
missing: ["NON_PRODUCTION_DEPLOYMENT_ON_PRODUCTION_DATABASE"],
})
})

it("catches the Production database via JWT ref even behind an unlisted custom domain (DOS/Give#273)", () => {
const fakeJwt = (payload: unknown) =>
`eyJhbGciOiJIUzI1NiJ9.${Buffer.from(JSON.stringify(payload)).toString("base64url")}.sig`
const prodServiceJwt = fakeJwt({ iss: "supabase", ref: "gulptwduchsjcsbndmua", role: "service_role" })
const prodAnonJwt = fakeJwt({ iss: "supabase", ref: "GULPTWDUCHSJCSBNDMUA", role: "anon" })
const betaServiceJwt = fakeJwt({ iss: "supabase", ref: "oyajkbadsykigtfrpdpg", role: "service_role" })

expect(
usesProductionDatabase({
NEXT_PUBLIC_SUPABASE_URL: "https://db.custom-dos.example",
SUPABASE_SERVICE_ROLE_KEY: prodServiceJwt,
})
).toBe(true)
expect(
usesProductionDatabase({
NEXT_PUBLIC_SUPABASE_URL: "https://db.custom-dos.example",
NEXT_PUBLIC_SUPABASE_ANON_KEY: prodAnonJwt,
})
).toBe(true)
expect(usesProductionDatabase({ SUPABASE_SERVICE_ROLE_KEY: prodServiceJwt })).toBe(true)
expect(
emailSettingsFromEnv({
...BETA,
NEXT_PUBLIC_SUPABASE_URL: "https://db.custom-dos.example",
SUPABASE_SERVICE_ROLE_KEY: prodServiceJwt,
})
).toMatchObject({
ok: false,
missing: ["NON_PRODUCTION_DEPLOYMENT_ON_PRODUCTION_DATABASE"],
})

// Non-production JWT and opaque sb_secret_ keys do not trigger false positives.
expect(
usesProductionDatabase({
NEXT_PUBLIC_SUPABASE_URL: "https://db.custom-beta.example",
SUPABASE_SERVICE_ROLE_KEY: betaServiceJwt,
})
).toBe(false)
expect(
usesProductionDatabase({
NEXT_PUBLIC_SUPABASE_URL: "https://db.custom-beta.example",
SUPABASE_SERVICE_ROLE_KEY: "sb_secret_not_a_jwt",
})
).toBe(false)
expect(
usesProductionDatabase({
NEXT_PUBLIC_SUPABASE_URL: "https://db.custom-beta.example",
SUPABASE_SERVICE_ROLE_KEY: "a.not-valid-json.c",
})
).toBe(false)
})
})
37 changes: 33 additions & 4 deletions apps/web/src/server/email/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,44 @@ export function emailLinkOrigin(env: Env = process.env): string | null {
return production ? PRODUCTION_ORIGIN : null
}

/**
* Extracts the Supabase project ref from a legacy JWT key (`header.payload.sig`),
* or null for opaque keys (`sb_secret_*` / `sb_publishable_*`) and invalid strings.
*/
function jwtProjectRef(token: string | undefined): string | null {
if (!token) return null
const parts = token.trim().split(".")
if (parts.length !== 3 || !parts[1]) return null
try {
const payload: unknown = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8"))
if (!payload || typeof payload !== "object") return null
const ref = (payload as { ref?: unknown }).ref
return typeof ref === "string" ? ref.trim().toLowerCase() : null
} catch {
return null
}
}

/**
* Whether env points at the Production DOS Supabase project (DOS/Give#273 item 6).
* Checks both the URL host (project host + vanity domains in PRODUCTION_SUPABASE_HOSTS)
* and the JWT `ref` claim of the Supabase keys so a newly added custom domain is
* still caught when JWT keys are used. We intentionally do not query the DB for an
* environment tag: Production, Beta, and Preview branches replay the same committed
* DOS.Me migrations, so an in-DB marker would be inherited by Preview/Beta.
*/
export function usesProductionDatabase(env: Env = process.env): boolean {
let host: string
try {
// A trailing dot (fully qualified name) reaches the same host.
host = new URL(env.NEXT_PUBLIC_SUPABASE_URL ?? "").hostname.toLowerCase().replace(/\.+$/, "")
const host = new URL(env.NEXT_PUBLIC_SUPABASE_URL ?? "").hostname.toLowerCase().replace(/\.+$/, "")
if (PRODUCTION_SUPABASE_HOSTS.includes(host) || host.split(".")[0] === PRODUCTION_SUPABASE_REF) return true
} catch {
return false
// Missing or malformed URL; still check JWT keys below.
}
return PRODUCTION_SUPABASE_HOSTS.includes(host) || host.split(".")[0] === PRODUCTION_SUPABASE_REF
return (
jwtProjectRef(env.SUPABASE_SERVICE_ROLE_KEY) === PRODUCTION_SUPABASE_REF ||
jwtProjectRef(env.NEXT_PUBLIC_SUPABASE_ANON_KEY) === PRODUCTION_SUPABASE_REF
)
}

export function emailDelivery(env: Env = process.env): EmailDelivery {
Expand Down