From 853274ceb373c86ca8e60a08319916c9adc0feea Mon Sep 17 00:00:00 2001 From: Maciej Piotr Balcerzak Date: Tue, 1 Sep 2026 21:57:10 +0200 Subject: [PATCH] feat(pagination): support bigint and uuid identifiers --- .../content/docs/dev/database/pagination.mdx | 35 ++++- .../src/api/lib/pagination-cursor.test.ts | 111 +++++++++++++++- .../vitnode/src/api/lib/pagination-cursor.ts | 123 ++++++++++++++++-- .../vitnode/src/api/lib/with-pagination.ts | 60 ++++++--- 4 files changed, 297 insertions(+), 32 deletions(-) diff --git a/apps/web/content/docs/dev/database/pagination.mdx b/apps/web/content/docs/dev/database/pagination.mdx index 041c3b179..2bce02083 100644 --- a/apps/web/content/docs/dev/database/pagination.mdx +++ b/apps/web/content/docs/dev/database/pagination.mdx @@ -13,6 +13,7 @@ VitNode uses cursor-based pagination for optimal performance with large datasets ```ts import z from "zod"; +import { getTableColumns } from "drizzle-orm"; import { buildRoute } from "@/api/lib/route"; import { withPagination, @@ -66,10 +67,10 @@ export const getCronsRoute = buildRoute({ }, c, primaryCursor: core_cron.id, - query: async ({ limit, where, orderBy }) => + query: async ({ cursorSelection, limit, where, orderBy }) => await c .get("db") - .select() + .select({ ...getTableColumns(core_cron), ...cursorSelection }) .from(core_cron) .where(where) .orderBy(orderBy) @@ -93,11 +94,39 @@ The `withPagination` function accepts the following parameters: | Parameter | Type | Description | | --------------- | -------- | -------------------------------------------- | | `params` | Object | Contains the query parameters for pagination | -| `primaryCursor` | Column | The primary key column used for pagination | +| `primaryCursor` | Column | Integer, bigint or UUID row identifier | | `query` | Function | The database query function | | `table` | Table | The database table being queried | | `orderBy` | Object | The column and order to sort by | +### Integer, bigint and UUID identifiers + +`primaryCursor` accepts integer, bigint and UUID columns. Bigints use decimal +strings inside the opaque cursor, so values above JavaScript's safe-integer +limit keep their precision. UUIDs stay strings and are validated before they +reach PostgreSQL. + +The order column and identifier can use different types. For example, a bigint +order column can use a UUID primary key as its stable tiebreaker: + +```ts +import { bigint, pgTable, uuid } from "drizzle-orm/pg-core"; + +const events = pgTable("events", { + id: uuid().defaultRandom().primaryKey(), + sequence: bigint({ mode: "bigint" }).notNull(), +}); + +await withPagination({ + // ... + primaryCursor: events.id, + orderBy: { column: events.sequence, order: "asc" }, +}); +``` + +Keep the cursor opaque on the client and return it unchanged in the next +request. + ### Zod Schemas VitNode provides pre-defined Zod schemas for pagination: diff --git a/packages/vitnode/src/api/lib/pagination-cursor.test.ts b/packages/vitnode/src/api/lib/pagination-cursor.test.ts index 85da253d6..c58a098ae 100644 --- a/packages/vitnode/src/api/lib/pagination-cursor.test.ts +++ b/packages/vitnode/src/api/lib/pagination-cursor.test.ts @@ -5,6 +5,7 @@ import { text, time, timestamp, + uuid, } from "drizzle-orm/pg-core"; // @vitest-environment node import { HTTPException } from "hono/http-exception"; @@ -12,13 +13,18 @@ import { describe, expect, it } from "vitest"; import { core_users } from "@/database/users"; +import type { PaginationCursorColumn } from "./with-pagination"; + import { + cursorIdentifierForColumn, + cursorIdentifierOf, cursorValueForColumn, cursorValueIsCanonicalText, cursorValueOf, decodePaginationCursor, encodePaginationCursor, isCursorSortableColumn, + isPaginationIdentifierColumn, } from "./pagination-cursor"; const probes = camelCase.table("cursor_probes", { @@ -28,8 +34,11 @@ const probes = camelCase.table("cursor_probes", { day: date(), moment: timestamp(), tags: text().array("[]"), + uuid: uuid(), }); +const UUID_ID = "0198f6f7-d4a2-7ce1-a2ee-4f5f1f2f3a4b"; + const statusOf = (error: unknown): number => error instanceof HTTPException ? error.status : 0; @@ -44,7 +53,7 @@ describe("encoding", () => { expect( decodePaginationCursor(encodePaginationCursor(cursor), { column: "updatedAt", - primaryKey: "id", + primary: core_users.id, }), ).toEqual(cursor); }); @@ -66,7 +75,7 @@ describe("encoding", () => { expect( decodePaginationCursor(encodePaginationCursor(cursor), { column: "publishedAt", - primaryKey: "id", + primary: core_users.id, }), ).toEqual(cursor); }); @@ -81,15 +90,45 @@ describe("encoding", () => { expect( decodePaginationCursor(encodePaginationCursor(cursor), { column: "name", - primaryKey: "id", + primary: core_users.id, }).value, ).toEqual(value); }); + + it("round-trips a bigint order value with a UUID tiebreaker", () => { + const cursor = { + column: "big", + id: UUID_ID, + value: "9007199254740993", + }; + + expect( + decodePaginationCursor(encodePaginationCursor(cursor), { + column: "big", + primary: probes.uuid, + }), + ).toEqual(cursor); + }); + + it("round-trips a bigint identifier without losing precision", () => { + const cursor = { + column: "big", + id: "9007199254740993", + value: "9007199254740993", + }; + + expect( + decodePaginationCursor(encodePaginationCursor(cursor), { + column: "big", + primary: probes.big, + }), + ).toEqual(cursor); + }); }); describe("decoding refuses what it cannot trust", () => { const decode = (raw: string, column = "updatedAt") => - decodePaginationCursor(raw, { column, primaryKey: "id" }); + decodePaginationCursor(raw, { column, primary: core_users.id }); it.each([ ["garbage", "not-a-cursor!!"], @@ -149,17 +188,69 @@ describe("decoding refuses what it cannot trust", () => { describe("legacy numeric cursors", () => { it("still works when the list is ordered by its identifier", () => { expect( - decodePaginationCursor("42", { column: "id", primaryKey: "id" }), + decodePaginationCursor("42", { column: "id", primary: core_users.id }), ).toEqual({ column: "id", id: 42, value: 42 }); }); it("is refused for any other ordering rather than guessed at", () => { expect(() => - decodePaginationCursor("42", { column: "updatedAt", primaryKey: "id" }), + decodePaginationCursor("42", { + column: "updatedAt", + primary: core_users.id, + }), ).toThrow(/cannot be used with the "updatedAt" ordering/); }); }); +describe("cursor identifiers", () => { + it("accepts integer, bigint and UUID columns", () => { + const columns: PaginationCursorColumn[] = [ + core_users.id, + probes.big, + probes.uuid, + ]; + + expect(columns.every(isPaginationIdentifierColumn)).toBe(true); + }); + + it("serializes and restores every supported identifier type", () => { + expect(cursorIdentifierOf(core_users.id, 42)).toBe(42); + expect(cursorIdentifierForColumn(core_users.id, 42)).toBe(42); + + expect(cursorIdentifierOf(probes.big, 9007199254740993n)).toBe( + "9007199254740993", + ); + expect(cursorIdentifierForColumn(probes.big, "9007199254740993")).toBe( + 9007199254740993n, + ); + + expect(cursorIdentifierOf(probes.uuid, UUID_ID)).toBe(UUID_ID); + expect(cursorIdentifierForColumn(probes.uuid, UUID_ID)).toBe(UUID_ID); + }); + + it.each([ + ["a bigint sent as a number", probes.big, 42], + ["a zero bigint", probes.big, "0"], + ["a malformed bigint", probes.big, "42n"], + ["a bigint outside PostgreSQL's range", probes.big, "9223372036854775808"], + ["a UUID sent as a number", probes.uuid, 42], + ["a malformed UUID", probes.uuid, "not-a-uuid"], + ])("refuses %s", (_why, column, value) => { + expect(() => cursorIdentifierForColumn(column, value)).toThrow( + HTTPException, + ); + }); + + it("does not treat a legacy numeric cursor as a UUID", () => { + expect(() => + decodePaginationCursor("42", { + column: "uuid", + primary: probes.uuid, + }), + ).toThrow(HTTPException); + }); +}); + describe("column values", () => { it("keeps a timestamp as text on both sides", () => { const flattened = cursorValueOf( @@ -178,6 +269,13 @@ describe("column values", () => { expect(cursorValueForColumn(core_users.name, "Ada")).toBe("Ada"); }); + it("validates a UUID before it can reach Postgres", () => { + expect(cursorValueForColumn(probes.uuid, UUID_ID)).toBe(UUID_ID); + expect(() => cursorValueForColumn(probes.uuid, "not-a-uuid")).toThrow( + HTTPException, + ); + }); + it("keeps a number a number", () => { expect(cursorValueOf(core_users.id, 7)).toBe(7); expect(cursorValueForColumn(core_users.id, 7)).toBe(7); @@ -273,6 +371,7 @@ describe("a tampered cursor value is refused, never coerced", () => { ["a number", 12], ["a boolean", false], ["whitespace", " 12 "], + ["outside PostgreSQL's range", "9223372036854775808"], ])("refuses %s", (_why, value) => { refuses(probes.big, value); }); diff --git a/packages/vitnode/src/api/lib/pagination-cursor.ts b/packages/vitnode/src/api/lib/pagination-cursor.ts index d2fcd19cb..50c7bec8c 100644 --- a/packages/vitnode/src/api/lib/pagination-cursor.ts +++ b/packages/vitnode/src/api/lib/pagination-cursor.ts @@ -31,11 +31,14 @@ import { HTTPException } from "hono/http-exception"; /** What an order column's value can be, once it has been through JSON. */ export type PaginationCursorValue = boolean | null | number | string; +/** Integer, bigint and UUID identifiers all have a JSON-safe wire form. */ +export type PaginationCursorIdentifier = number | string; + export interface PaginationCursor { /** The order column this cursor was minted for. */ column: string; /** The row's primary key - the tiebreaker half of the ordered tuple. */ - id: number; + id: PaginationCursorIdentifier; /** The order column's value on that row. `null` is a real position. */ value: PaginationCursorValue; } @@ -48,7 +51,8 @@ export interface PaginationCursor { * answering it in one place is what stops the second answer being looser than * the first. */ -type CursorKind = "bigint" | "boolean" | "number" | "string" | "temporal"; +type CursorKind = + "bigint" | "boolean" | "number" | "string" | "temporal" | "uuid"; const KIND_BY_DATA_TYPE: Record = { bigint: "bigint", @@ -70,6 +74,9 @@ const KIND_BY_DATA_TYPE: Record = { const baseDataTypeOf = (column: PgColumn): string => column.dataType.split(" ")[0]; +const isUuidColumn = (column: PgColumn): boolean => + column.getSQLType().toLowerCase() === "uuid"; + /** * Whether the column holds an array. * @@ -126,6 +133,7 @@ export const isCursorSortableColumn = (column: PgColumn): boolean => const kindOf = (column: PgColumn): CursorKind => { if (!isArrayColumn(column) && temporalTypeOf(column)) return "temporal"; + if (!isArrayColumn(column) && isUuidColumn(column)) return "uuid"; const kind = isArrayColumn(column) ? undefined @@ -251,6 +259,18 @@ const isRealTemporal = ( }; const DECIMAL_INTEGER = /^-?\d+$/; +const POSITIVE_DECIMAL_INTEGER = /^[1-9]\d*$/; +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const PG_BIGINT_MIN = -9223372036854775808n; +const PG_BIGINT_MAX = 9223372036854775807n; + +const postgresBigint = (value: string): bigint | undefined => { + const parsed = BigInt(value); + + return parsed >= PG_BIGINT_MIN && parsed <= PG_BIGINT_MAX + ? parsed + : undefined; +}; const pad = (value: number, width = 2): string => String(value).padStart(width, "0"); @@ -337,6 +357,10 @@ export const cursorValueOf = ( if (typeof value === "string") return value; break; } + case "uuid": { + if (typeof value === "string" && UUID.test(value)) return value; + break; + } default: { if (typeof value === "string") return value; break; @@ -384,7 +408,10 @@ export const cursorValueForColumn = ( throw badRequest(INVALID_CURSOR); } - return BigInt(value); + const parsed = postgresBigint(value); + if (parsed === undefined) throw badRequest(INVALID_CURSOR); + + return parsed; } case "boolean": { if (typeof value !== "boolean") throw badRequest(INVALID_CURSOR); @@ -412,6 +439,13 @@ export const cursorValueForColumn = ( // Postgres does the parsing - at the precision it stored. return value; } + case "uuid": { + if (typeof value !== "string" || !UUID.test(value)) { + throw badRequest(INVALID_CURSOR); + } + + return value; + } default: { if (typeof value !== "string") throw badRequest(INVALID_CURSOR); @@ -432,6 +466,78 @@ export const cursorValueForColumn = ( export const cursorValueIsCanonicalText = (column: PgColumn): boolean => kindOf(column) === "temporal"; +type IdentifierKind = "bigint" | "number" | "uuid"; + +const identifierKindOf = (column: PgColumn): IdentifierKind | undefined => { + if (isArrayColumn(column)) return undefined; + if (isUuidColumn(column)) return "uuid"; + + const baseDataType = baseDataTypeOf(column); + if (baseDataType === "bigint" || baseDataType === "number") { + return baseDataType; + } + + return undefined; +}; + +export const isPaginationIdentifierColumn = (column: PgColumn): boolean => + identifierKindOf(column) !== undefined; + +export const cursorIdentifierOf = ( + column: PgColumn, + value: unknown, +): PaginationCursorIdentifier => { + switch (identifierKindOf(column)) { + case "bigint": + if (typeof value === "bigint" && value > 0n) return value.toString(); + break; + case "number": + if ( + typeof value === "number" && + Number.isSafeInteger(value) && + value > 0 + ) { + return value; + } + break; + case "uuid": + if (typeof value === "string" && UUID.test(value)) return value; + break; + } + + throw new Error( + `Cannot build a pagination cursor from the identifier on "${column.name}".`, + ); +}; + +export const cursorIdentifierForColumn = ( + column: PgColumn, + value: PaginationCursorIdentifier, +): bigint | number | string => { + switch (identifierKindOf(column)) { + case "bigint": + if (typeof value === "string" && POSITIVE_DECIMAL_INTEGER.test(value)) { + const parsed = postgresBigint(value); + if (parsed !== undefined) return parsed; + } + break; + case "number": + if ( + typeof value === "number" && + Number.isSafeInteger(value) && + value > 0 + ) { + return value; + } + break; + case "uuid": + if (typeof value === "string" && UUID.test(value)) return value; + break; + } + + throw badRequest(INVALID_CURSOR); +}; + export const encodePaginationCursor = (cursor: PaginationCursor): string => Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url"); @@ -464,18 +570,19 @@ const isCursorValue = (value: unknown): value is PaginationCursorValue => */ export const decodePaginationCursor = ( raw: string, - { column, primaryKey }: { column: string; primaryKey: string }, + { column, primary }: { column: string; primary: PgColumn }, ): PaginationCursor => { const trimmed = raw.trim(); if (trimmed === "") throw badRequest(INVALID_CURSOR); if (LEGACY_CURSOR.test(trimmed)) { - if (column !== primaryKey) { + if (column !== primary.name) { throw badRequest( `This cursor cannot be used with the "${column}" ordering. Start from the first page.`, ); } const id = Number(trimmed); + cursorIdentifierForColumn(primary, id); return { column, id, value: id }; } @@ -497,14 +604,14 @@ export const decodePaginationCursor = ( const id = candidate.id; if ( typeof candidate.column !== "string" || - typeof id !== "number" || - !Number.isSafeInteger(id) || - id <= 0 || + (typeof id !== "number" && typeof id !== "string") || !isCursorValue(candidate.value) ) { throw badRequest(INVALID_CURSOR); } + cursorIdentifierForColumn(primary, id); + if (candidate.column !== column) { throw badRequest( `This cursor was issued for a different ordering. Start from the first page.`, diff --git a/packages/vitnode/src/api/lib/with-pagination.ts b/packages/vitnode/src/api/lib/with-pagination.ts index 29c9d4933..2330818df 100644 --- a/packages/vitnode/src/api/lib/with-pagination.ts +++ b/packages/vitnode/src/api/lib/with-pagination.ts @@ -1,4 +1,9 @@ -import type { ColumnDataNumberConstraint, Placeholder, SQL } from "drizzle-orm"; +import type { + ColumnDataBigIntConstraint, + ColumnDataNumberConstraint, + Placeholder, + SQL, +} from "drizzle-orm"; import type { PgColumn, PgColumnBaseConfig, @@ -28,12 +33,15 @@ import { HTTPException } from "hono/http-exception"; import type { PaginationCursor } from "./pagination-cursor"; import { + cursorIdentifierForColumn, + cursorIdentifierOf, cursorValueForColumn, cursorValueIsCanonicalText, cursorValueOf, decodePaginationCursor, encodePaginationCursor, isCursorSortableColumn, + isPaginationIdentifierColumn, } from "./pagination-cursor"; /** Nobody may ask for more than this in one page, whatever they send. */ @@ -165,10 +173,14 @@ function buildCursorCondition({ primary: PgColumn; }): SQL { const after = direction === "asc" ? gt : lt; + const identifier = sql`${sql.param( + cursorIdentifierForColumn(primary, cursor.id), + primary, + )}`; // The identifier is the whole tuple when the list is ordered by it, so there // is no second half to compare and no null block to worry about. - if (isPrimaryOrder) return after(primary, cursor.id); + if (isPrimaryOrder) return after(primary, identifier); const boundary = boundaryValue(column, cursor); @@ -176,13 +188,13 @@ function buildCursorCondition({ // NULLS LAST: a null cursor is inside the trailing block, and everything // that is not null is already behind us. if (cursor.value === null) { - return required(and(isNull(column), gt(primary, cursor.id))); + return required(and(isNull(column), gt(primary, identifier))); } return required( or( gt(column, boundary), - and(eq(column, boundary), gt(primary, cursor.id)), + and(eq(column, boundary), gt(primary, identifier)), isNull(column), ), ); @@ -192,12 +204,15 @@ function buildCursorCondition({ // that block comes first and every non-null row follows it. if (cursor.value === null) { return required( - or(and(isNull(column), lt(primary, cursor.id)), isNotNull(column)), + or(and(isNull(column), lt(primary, identifier)), isNotNull(column)), ); } return required( - or(lt(column, boundary), and(eq(column, boundary), lt(primary, cursor.id))), + or( + lt(column, boundary), + and(eq(column, boundary), lt(primary, identifier)), + ), ); } @@ -252,18 +267,24 @@ async function fetchTotalCount( } /** - * A column that can carry the cursor: any numeric one. + * A stable row identifier: an integer, bigint or UUID column. * - * Drizzle refines numeric column types (a `serial` is `number int32`, a - * `doublePrecision` is `number double`), so the bound admits the whole - * `number ...` family rather than the bare `number`. The data type is asserted - * through the config - Drizzle leaves `PgColumn`'s first argument as `any` on - * built columns, exactly as its own `AnyPgColumn` does. + * Drizzle refines these types (`serial` is `number int32`, bigint is + * `bigint int64`, UUID is `string uuid`), so the bound includes their refined + * forms. The first generic stays broad because built `PgColumn`s expose it that + * way, as Drizzle's own `AnyPgColumn` does. */ +type PaginationIdentifierDataType = + | "bigint" + | "number" + | "string uuid" + | `bigint ${ColumnDataBigIntConstraint}` + | `number ${ColumnDataNumberConstraint}`; + export type PaginationCursorColumn = PgColumn< // eslint-disable-next-line @typescript-eslint/no-explicit-any any, - PgColumnBaseConfig<"number" | `number ${ColumnDataNumberConstraint}`> + PgColumnBaseConfig >; export async function withPagination< @@ -332,6 +353,12 @@ export async function withPagination< const orderColumn = table[orderName] as PgColumn; const isPrimaryOrder = orderName === primaryCursor.name; + if (!isPaginationIdentifierColumn(primary)) { + throw new HTTPException(400, { + message: `The "${primaryCursor.name}" primary cursor must be an integer, bigint or UUID column.`, + }); + } + // A column with no total order Postgres and JavaScript agree on cannot be // paged at all, so it is refused rather than served for one page and then // quietly wrong on the next. @@ -359,7 +386,7 @@ export async function withPagination< ? undefined : decodePaginationCursor(rawCursor, { column: orderName, - primaryKey: primaryCursor.name, + primary, }); const searchWhere = buildSearchWhere(search, params.query.search); @@ -411,6 +438,7 @@ export async function withPagination< edges: finalEdges, orderColumn, orderName, + primaryColumn: primary, primaryName: primaryCursor.name, }); @@ -469,11 +497,13 @@ function cursorsFrom({ edges, orderColumn, orderName, + primaryColumn, primaryName, }: { edges: readonly Record[]; orderColumn: PgColumn; orderName: string; + primaryColumn: PgColumn; primaryName: string; }): { endCursor: null | string; startCursor: null | string } { const first = edges[0]; @@ -505,7 +535,7 @@ function cursorsFrom({ const mint = (row: Record): string => encodePaginationCursor({ column: orderName, - id: Number(row[primaryName]), + id: cursorIdentifierOf(primaryColumn, row[primaryName]), value: cursorValueOf(orderColumn, valueOf(row)), });