diff --git a/apps/server/package.json b/apps/server/package.json index f2016a4863..707b4158bf 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -4,7 +4,7 @@ "type": "module", "private": true, "scripts": { - "build": "node ../../scripts/build-node-entry.mjs src/index.ts dist/index.js --clean-dist --external ./start-server.js --copy-dir ../../packages/db/drizzle dist/drizzle --copy-dir src/assets dist/assets && node --import tsx scripts/copy-builtin-skills.ts && node ../../scripts/build-node-entry.mjs src/start-server.ts dist/start-server.js && node ../../scripts/build-node-entry.mjs ../../packages/plugin-sdk/src/index.ts dist/plugin-sdk-runtime.js", + "build": "node ../../scripts/build-node-entry.mjs src/index.ts dist/index.js --clean-dist --external ./start-server.js --copy-dir ../../packages/db/drizzle dist/drizzle --copy-dir src/assets dist/assets && node --import tsx scripts/copy-builtin-skills.ts && node ../../scripts/build-node-entry.mjs src/start-server.ts dist/start-server.js && node ../../scripts/build-node-entry.mjs src/sqlite-read-worker.ts dist/sqlite-read-worker.js && node ../../scripts/build-node-entry.mjs ../../packages/plugin-sdk/src/index.ts dist/plugin-sdk-runtime.js", "start": "node dist/index.js", "start:prod": "cross-env NODE_ENV=production node dist/index.js", "dev": "node --conditions=source --import tsx scripts/dev-supervisor.mjs", diff --git a/apps/server/src/routes/projects.ts b/apps/server/src/routes/projects.ts index ab1c464a7b..e400774cdd 100644 --- a/apps/server/src/routes/projects.ts +++ b/apps/server/src/routes/projects.ts @@ -13,7 +13,7 @@ import { listPublicProjects, listProjectSourcesByProjectIds, listThreadSections, - listThreadsWithPendingInteractionStateForProjects, + listThreadsWithPendingInteractionStateForProjectsOffThread, reorderProject, updateProject, updateProjectSource, @@ -201,26 +201,27 @@ function parseProjectListIncludes( return includes; } -function buildProjectsWithThreadsResponse( +async function buildProjectsWithThreadsResponse( deps: AppDeps, options: ProjectListOptions, -): ProjectWithThreadsResponse[] { +): Promise { return buildProjectsWithThreadsResponseFromRows( deps, listDiscoverableProjects(deps, options), ); } -function buildProjectsWithThreadsResponseFromRows( +async function buildProjectsWithThreadsResponseFromRows( deps: AppDeps, projectRows: ProjectResponseProjectFields[], -): ProjectWithThreadsResponse[] { +): Promise { const projects = buildProjectResponsesFromRows(deps, projectRows); const projectIds = projects.map((project) => project.id); - const threadRows = listThreadsWithPendingInteractionStateForProjects( - deps.db, - { archived: false, projectIds }, - ); + const threadRows = + await listThreadsWithPendingInteractionStateForProjectsOffThread(deps.db, { + archived: false, + projectIds, + }); const threadResponses = toThreadListEntryResponses(deps, { threads: threadRows, }); @@ -253,7 +254,7 @@ function buildProjectsWithThreadsResponseFromRows( })); } -function buildSidebarBootstrapResponse(deps: AppDeps) { +async function buildSidebarBootstrapResponse(deps: AppDeps) { const personalProject = getPersonalProject(deps.db); if (!personalProject) { throw new ApiError( @@ -262,9 +263,8 @@ function buildSidebarBootstrapResponse(deps: AppDeps) { "Personal project is not initialized", ); } - const personalProjectResponse = buildProjectsWithThreadsResponseFromRows( - deps, - [personalProject], + const personalProjectResponse = ( + await buildProjectsWithThreadsResponseFromRows(deps, [personalProject]) )[0]; if (!personalProjectResponse) { throw new ApiError( @@ -275,7 +275,7 @@ function buildSidebarBootstrapResponse(deps: AppDeps) { } return { sections: listThreadSections(deps.db), - projects: buildProjectsWithThreadsResponseFromRows( + projects: await buildProjectsWithThreadsResponseFromRows( deps, listPublicProjects(deps.db), ), @@ -338,13 +338,15 @@ export function registerProjectRoutes(app: Hono, deps: AppDeps): void { }); const routes = publicApiRoutes.projects; - get(routes.list, (context, query) => { + get(routes.list, async (context, query) => { const includes = parseProjectListIncludes(query); const options: ProjectListOptions = { includePersonal: query.includePersonal === "true", }; if (includes.has("threads")) { - return context.json(buildProjectsWithThreadsResponse(deps, options)); + return context.json( + await buildProjectsWithThreadsResponse(deps, options), + ); } return context.json( buildProjectResponsesFromRows( @@ -354,8 +356,8 @@ export function registerProjectRoutes(app: Hono, deps: AppDeps): void { ); }); - get(routes.sidebarBootstrap, (context) => - context.json(buildSidebarBootstrapResponse(deps)), + get(routes.sidebarBootstrap, async (context) => + context.json(await buildSidebarBootstrapResponse(deps)), ); post(routes.create, async (context, payload) => { diff --git a/apps/server/src/routes/threads/base.ts b/apps/server/src/routes/threads/base.ts index 77c7f32eb8..aaba545452 100644 --- a/apps/server/src/routes/threads/base.ts +++ b/apps/server/src/routes/threads/base.ts @@ -6,7 +6,7 @@ import { getEnvironment, getThreadSectionById, listThreadMentionRowsByIds, - listThreadsWithPendingInteractionState, + listThreadsWithPendingInteractionStateOffThread, markThreadDeleted, searchThreadsWithPendingInteractionState, updateThread, @@ -255,7 +255,7 @@ export function registerThreadBaseRoutes(app: Hono, deps: AppDeps): void { ); }); - get(routes.list, (context, query) => { + get(routes.list, async (context, query) => { const limit = parseOptionalInteger(query.limit, "limit"); if (limit !== undefined && limit <= 0) { throw new ApiError(400, "invalid_request", "limit must be positive"); @@ -277,22 +277,33 @@ export function registerThreadBaseRoutes(app: Hono, deps: AppDeps): void { if (query.sectionId) { requireThreadSection(deps, query.sectionId); } - const threads = listThreadsWithPendingInteractionState(deps.db, { - ...(query.projectId ? { projectId: query.projectId } : {}), - ...(query.parentThreadId ? { parentThreadId: query.parentThreadId } : {}), - ...(query.sourceThreadId ? { sourceThreadId: query.sourceThreadId } : {}), - ...(query.sectionId ? { sectionId: query.sectionId } : {}), - ...(query.unsectioned === "true" ? { unsectioned: true } : {}), - ...(query.originKind ? { originKind: query.originKind } : {}), - ...(query.originPluginId ? { originPluginId: query.originPluginId } : {}), - includeHidden: query.includeHidden === "true", - archived: - query.archived === undefined ? undefined : query.archived === "true", - hasParent: - query.hasParent === undefined ? undefined : query.hasParent === "true", - ...(limit !== undefined ? { limit } : {}), - ...(offset !== undefined ? { offset } : {}), - }); + const threads = await listThreadsWithPendingInteractionStateOffThread( + deps.db, + { + ...(query.projectId ? { projectId: query.projectId } : {}), + ...(query.parentThreadId + ? { parentThreadId: query.parentThreadId } + : {}), + ...(query.sourceThreadId + ? { sourceThreadId: query.sourceThreadId } + : {}), + ...(query.sectionId ? { sectionId: query.sectionId } : {}), + ...(query.unsectioned === "true" ? { unsectioned: true } : {}), + ...(query.originKind ? { originKind: query.originKind } : {}), + ...(query.originPluginId + ? { originPluginId: query.originPluginId } + : {}), + includeHidden: query.includeHidden === "true", + archived: + query.archived === undefined ? undefined : query.archived === "true", + hasParent: + query.hasParent === undefined + ? undefined + : query.hasParent === "true", + ...(limit !== undefined ? { limit } : {}), + ...(offset !== undefined ? { offset } : {}), + }, + ); return context.json( toThreadListEntryResponses(deps, { threads }) satisfies ThreadListEntry[], ); diff --git a/apps/server/src/sqlite-read-worker.ts b/apps/server/src/sqlite-read-worker.ts new file mode 100644 index 0000000000..b5e6793d15 --- /dev/null +++ b/apps/server/src/sqlite-read-worker.ts @@ -0,0 +1 @@ +import "@bb/db/sqlite-read-worker"; diff --git a/apps/server/src/start-server.ts b/apps/server/src/start-server.ts index 7162aae15a..f02919cb65 100644 --- a/apps/server/src/start-server.ts +++ b/apps/server/src/start-server.ts @@ -6,7 +6,7 @@ import type { ServerConfig } from "@bb/config/server"; import { isLoopbackHostname } from "@bb/config/loopback"; import { toOptionalString } from "@bb/config/strings"; import { createLogger } from "@bb/logger"; -import { getAppSettings } from "@bb/db"; +import { getAppSettings, startSqliteReadWorker } from "@bb/db"; import { initDb } from "./db.js"; import { createApp } from "./server.js"; import { PendingInteractionLifecycle } from "./services/interactions/pending-interactions.js"; @@ -56,6 +56,25 @@ export async function runServer(serverConfig: ServerConfig): Promise { dataDir: serverConfig.BB_DATA_DIR, logger, }); + if (serverConfig.databasePath !== ":memory:") { + try { + const jsFilename = fileURLToPath( + new URL("./sqlite-read-worker.js", import.meta.url), + ); + const tsFilename = fileURLToPath( + new URL("./sqlite-read-worker.ts", import.meta.url), + ); + startSqliteReadWorker({ + source: serverConfig.databasePath, + workerFilename: existsSync(jsFilename) ? jsFilename : tsFilename, + }); + } catch (error) { + logger.error( + { err: error }, + "Failed to start sqlite read worker; heavy reads will stay on the serving thread", + ); + } + } const hub = new NotificationHub(); const watchInterests = new WatchInterestCoordinator({ db, hub }); const sharedPorts = new HostSharedPortCoordinator({ db, hub }); diff --git a/packages/db/package.json b/packages/db/package.json index e9648816e6..640b253961 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -12,6 +12,11 @@ "source": "./src/internal-environment-lifecycle.ts", "types": "./src/internal-environment-lifecycle.ts", "default": "./src/internal-environment-lifecycle.ts" + }, + "./sqlite-read-worker": { + "source": "./src/sqlite-read-worker.ts", + "types": "./src/sqlite-read-worker.ts", + "default": "./src/sqlite-read-worker.ts" } }, "types": "./src/index.ts", diff --git a/packages/db/src/connection.ts b/packages/db/src/connection.ts index b324522076..c38eb817f3 100644 --- a/packages/db/src/connection.ts +++ b/packages/db/src/connection.ts @@ -16,6 +16,7 @@ export interface SlowDbQueryLogger { } export interface CreateConnectionOptions { + readonly?: boolean; slowQueryLogger?: SlowDbQueryLogger; slowQueryThresholdMs?: number; } @@ -156,12 +157,16 @@ export function createConnection( source: string | Buffer = "bb.db", options: CreateConnectionOptions = {}, ) { - const sqlite = new Database(source); - - sqlite.pragma("auto_vacuum = INCREMENTAL"); - sqlite.pragma("journal_mode = WAL"); + const sqlite = options.readonly + ? new Database(source, { fileMustExist: true, readonly: true }) + : new Database(source); + + if (!options.readonly) { + sqlite.pragma("auto_vacuum = INCREMENTAL"); + sqlite.pragma("journal_mode = WAL"); + sqlite.pragma("synchronous = NORMAL"); + } sqlite.pragma("foreign_keys = ON"); - sqlite.pragma("synchronous = NORMAL"); sqlite.pragma(`cache_size = -${SQLITE_CACHE_SIZE_KIB}`); sqlite.pragma(`mmap_size = ${SQLITE_MMAP_SIZE_BYTES}`); sqlite.pragma(`busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 604d0e10cb..93cc7927a9 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,4 +1,10 @@ export { createConnection } from "./connection.js"; +export { + listThreadsWithPendingInteractionStateForProjectsOffThread, + listThreadsWithPendingInteractionStateOffThread, + startSqliteReadWorker, + stopSqliteReadWorker, +} from "./sqlite-read-queue.js"; export type { DbConnection, DbQueryConnection, diff --git a/packages/db/src/sqlite-read-queue.ts b/packages/db/src/sqlite-read-queue.ts new file mode 100644 index 0000000000..91bd68a271 --- /dev/null +++ b/packages/db/src/sqlite-read-queue.ts @@ -0,0 +1,144 @@ +import { existsSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { Worker } from "node:worker_threads"; +import type { DbConnection } from "./connection.js"; +import { + listThreadsWithPendingInteractionState, + listThreadsWithPendingInteractionStateForProjects, + type ListThreadsForProjectsOptions, + type ListThreadsOptions, + type ThreadWithPendingInteractionState, +} from "./data/threads.js"; +import type { + SqliteReadRequest, + SqliteReadResponse, +} from "./sqlite-read-worker.js"; + +type PendingRead = { + reject: (error: Error) => void; + resolve: (result: ThreadWithPendingInteractionState[]) => void; +}; + +let worker: Worker | null = null; +const pending = new Map(); +let nextRequestId = 1; + +function workerFilename(): string { + const tsFilename = fileURLToPath( + new URL("./sqlite-read-worker.ts", import.meta.url), + ); + if (existsSync(tsFilename)) { + return tsFilename; + } + const jsFilename = fileURLToPath( + new URL("./sqlite-read-worker.js", import.meta.url), + ); + if (existsSync(jsFilename)) { + return jsFilename; + } + throw new Error("sqlite read worker file is missing"); +} + +function failAll(error: Error): void { + for (const request of pending.values()) { + request.reject(error); + } + pending.clear(); +} + +export function isSqliteReadWorkerActive(): boolean { + return worker !== null; +} + +export function startSqliteReadWorker(args: { + source: string | Buffer; + workerFilename?: string; +}): void { + if (worker !== null) { + return; + } + if (typeof args.source !== "string" || args.source === ":memory:") { + return; + } + const filename = args.workerFilename ?? workerFilename(); + const next = new Worker(filename, { + execArgv: filename.endsWith(".ts") + ? ["--import", fileURLToPath(import.meta.resolve("tsx"))] + : [], + workerData: { source: args.source }, + }); + next.on("message", (response: SqliteReadResponse) => { + const request = pending.get(response.id); + if (request === undefined) { + return; + } + pending.delete(response.id); + if (response.ok) { + request.resolve(response.result as ThreadWithPendingInteractionState[]); + return; + } + request.reject(new Error(response.error)); + }); + next.on("error", (error) => { + failAll(error); + worker = null; + }); + next.on("exit", (code) => { + if (pending.size > 0) { + failAll(new Error(`sqlite read worker exited with code ${code}`)); + } + worker = null; + }); + worker = next; +} + +export async function stopSqliteReadWorker(): Promise { + const current = worker; + worker = null; + failAll(new Error("sqlite read worker closed")); + if (current === null) { + return; + } + await current.terminate(); +} + +async function request( + message: Omit, +): Promise { + const current = worker; + if (current === null) { + throw new Error("sqlite read worker is not running"); + } + const id = nextRequestId; + nextRequestId += 1; + return new Promise((resolve, reject) => { + pending.set(id, { reject, resolve }); + current.postMessage({ id, ...message } as SqliteReadRequest); + }); +} + +export async function listThreadsWithPendingInteractionStateOffThread( + db: DbConnection, + options: ListThreadsOptions, +): Promise { + if (worker === null) { + return listThreadsWithPendingInteractionState(db, options); + } + return request({ + name: "listThreadsWithPendingInteractionState", + args: options, + }); +} + +export async function listThreadsWithPendingInteractionStateForProjectsOffThread( + db: DbConnection, + options: ListThreadsForProjectsOptions, +): Promise { + if (worker === null) { + return listThreadsWithPendingInteractionStateForProjects(db, options); + } + return request({ + name: "listThreadsWithPendingInteractionStateForProjects", + args: options, + }); +} diff --git a/packages/db/src/sqlite-read-worker.ts b/packages/db/src/sqlite-read-worker.ts new file mode 100644 index 0000000000..f2b8e47a07 --- /dev/null +++ b/packages/db/src/sqlite-read-worker.ts @@ -0,0 +1,50 @@ +import { parentPort, workerData } from "node:worker_threads"; +import { createConnection } from "./connection.js"; +import { + listThreadsWithPendingInteractionState, + listThreadsWithPendingInteractionStateForProjects, + type ListThreadsForProjectsOptions, + type ListThreadsOptions, +} from "./data/threads.js"; + +export type SqliteReadRequest = + | { + id: number; + name: "listThreadsWithPendingInteractionState"; + args: ListThreadsOptions; + } + | { + id: number; + name: "listThreadsWithPendingInteractionStateForProjects"; + args: ListThreadsForProjectsOptions; + }; + +export type SqliteReadResponse = + | { id: number; ok: true; result: unknown } + | { id: number; ok: false; error: string }; + +const port = parentPort; +if (port === null) { + throw new Error("sqlite read worker must run as a worker thread"); +} + +const source = (workerData as { source: string }).source; +const db = createConnection(source, { readonly: true }); + +port.on("message", (request: SqliteReadRequest) => { + try { + const result = + request.name === "listThreadsWithPendingInteractionState" + ? listThreadsWithPendingInteractionState(db, request.args) + : listThreadsWithPendingInteractionStateForProjects(db, request.args); + const response: SqliteReadResponse = { id: request.id, ok: true, result }; + port.postMessage(response); + } catch (error) { + const response: SqliteReadResponse = { + id: request.id, + ok: false, + error: error instanceof Error ? error.message : String(error), + }; + port.postMessage(response); + } +}); diff --git a/packages/db/test/sqlite-read-queue.test.ts b/packages/db/test/sqlite-read-queue.test.ts new file mode 100644 index 0000000000..781835f418 --- /dev/null +++ b/packages/db/test/sqlite-read-queue.test.ts @@ -0,0 +1,104 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.stubGlobal("__sqliteReadQueueTest", true); +import { createConnection, type DbConnection } from "../src/connection.js"; +import { upsertHost } from "../src/data/hosts.js"; +import { createProject } from "../src/data/projects.js"; +import { + createThread, + listThreadsWithPendingInteractionState, + listThreadsWithPendingInteractionStateForProjects, +} from "../src/data/threads.js"; +import { migrate } from "../src/migrate.js"; +import { noopNotifier } from "../src/notifier.js"; +import { + isSqliteReadWorkerActive, + listThreadsWithPendingInteractionStateForProjectsOffThread, + listThreadsWithPendingInteractionStateOffThread, + startSqliteReadWorker, + stopSqliteReadWorker, +} from "../src/sqlite-read-queue.js"; + +const tempDirs: string[] = []; +const connections: DbConnection[] = []; + +afterEach(async () => { + await stopSqliteReadWorker(); + while (connections.length > 0) { + connections.pop()?.$client.close(); + } + while (tempDirs.length > 0) { + const directory = tempDirs.pop(); + if (directory !== undefined) { + rmSync(directory, { force: true, recursive: true }); + } + } +}); + +function createFileDatabase() { + const directory = mkdtempSync(join(tmpdir(), "bb-sqlite-read-queue-")); + tempDirs.push(directory); + const source = join(directory, "bb.db"); + const db = createConnection(source); + connections.push(db); + migrate(db); + const host = upsertHost(db, noopNotifier, { + name: "test-host", + type: "persistent", + }); + const { project } = createProject(db, noopNotifier, { + name: "test-project", + source: { type: "local_path", hostId: host.id, path: "/tmp/test" }, + }); + const thread = createThread(db, noopNotifier, { + projectId: project.id, + providerId: "codex", + }); + return { db, project, source, thread }; +} + +describe("sqlite read queue", () => { + it("keeps in-memory reads on the calling connection", async () => { + const db = createConnection(":memory:"); + connections.push(db); + migrate(db); + startSqliteReadWorker({ source: ":memory:" }); + + expect(isSqliteReadWorkerActive()).toBe(false); + await expect( + listThreadsWithPendingInteractionStateOffThread(db, {}), + ).resolves.toEqual(listThreadsWithPendingInteractionState(db, {})); + }); + + it("returns the same thread list from a file-backed worker as the serving connection", async () => { + const { db, project, source, thread } = createFileDatabase(); + startSqliteReadWorker({ source }); + + expect(isSqliteReadWorkerActive()).toBe(true); + + const fromWorker = await listThreadsWithPendingInteractionStateOffThread( + db, + { projectId: project.id }, + ); + const fromServing = listThreadsWithPendingInteractionState(db, { + projectId: project.id, + }); + const fromProjectsWorker = + await listThreadsWithPendingInteractionStateForProjectsOffThread(db, { + archived: false, + projectIds: [project.id], + }); + const fromProjectsServing = + listThreadsWithPendingInteractionStateForProjects(db, { + archived: false, + projectIds: [project.id], + }); + + expect(fromWorker.map((row) => row.id)).toEqual([thread.id]); + expect(fromWorker).toEqual(fromServing); + expect(fromProjectsWorker).toEqual(fromProjectsServing); + }); +});