From 7c155bdd4b69fc3c945bdb535d2e9c5bc3458c12 Mon Sep 17 00:00:00 2001 From: Amp Date: Wed, 18 Nov 2026 20:05:24 +0000 Subject: [PATCH] Add bounded history pages and an entity-attribute history index Triples.historyPage reads one entity's history, or one attribute's, in index-seek pages pinned to the first page's commit position, with an opaque cursor bound to scope, entity, attribute and position. KV gets entity and entity-attribute history keys with a bounded one-time backfill; SQL backends seek idx_entity_recorded_position or the new idx_entity_attribute_history. The host protocol gains a HistoryPage operation. The new index (SQL migration 2, Cloudflare migration 8) also serves Datalog optional projections, which SQLite planned on an attribute-wide index: a 50-row explorer page sorted by an optional attribute drops from 6.2-8.7 s to 38-56 ms. runMigrations upgrades a 0.5.0 database in place; non-migrating layers report the pending migration. Refs #72. Amp-Thread-ID: https://ampcode.com/threads/T-01a1f02a-e1b5-7578-bc7a-392b7507001f Co-authored-by: Amp --- .changeset/bounded-history-pages.md | 28 +++ docs/concepts.md | 5 +- packages/cloudflare/src/migrations.ts | 6 + packages/cloudflare/src/schema.ts | 3 + .../src/storage/CloudflareAdapter.ts | 10 + packages/core/src/HistoryPage.ts | 175 ++++++++++++++++++ packages/core/src/index.ts | 2 + .../core/src/kv/hexastore/KvTripleStore.ts | 78 ++++++++ packages/core/src/kv/hexastore/indexes.ts | 69 +++++++ packages/core/src/kv/layers/KvTriplesLive.ts | 44 +++++ packages/core/src/storage/StorageAdapter.ts | 13 ++ packages/core/src/store/Triples.ts | 22 +++ packages/core/src/store/TriplesLive.ts | 46 +++++ packages/core/test/runtime/runtime.test.ts | 1 + packages/host/src/Protocol.ts | 17 ++ packages/host/test/host.test.ts | 36 +++- packages/postgres/src/PostgresqlAdapter.ts | 5 + packages/sql/src/HistoryPage.ts | 41 ++++ packages/sql/src/SchemaVersion.ts | 8 +- packages/sql/src/index.ts | 2 + packages/sql/src/migrations.ts | 62 ++++--- packages/sql/src/schema.ts | 10 + packages/sql/test/sql-contract.test.ts | 9 +- packages/sqlite/src/SqliteAdapter.ts | 5 + .../sqlite/test/concurrent-first-open.test.ts | 3 +- packages/sqlite/test/migrations.test.ts | 67 ++++++- packages/testkit/src/historyPageCases.ts | 150 +++++++++++++++ packages/testkit/src/index.ts | 2 + 28 files changed, 883 insertions(+), 36 deletions(-) create mode 100644 .changeset/bounded-history-pages.md create mode 100644 packages/core/src/HistoryPage.ts create mode 100644 packages/sql/src/HistoryPage.ts create mode 100644 packages/testkit/src/historyPageCases.ts diff --git a/.changeset/bounded-history-pages.md b/.changeset/bounded-history-pages.md new file mode 100644 index 00000000..056b9b3b --- /dev/null +++ b/.changeset/bounded-history-pages.md @@ -0,0 +1,28 @@ +--- +"@triplex-build/triplex": minor +--- + +**Bounded history pages, and an index that makes optional projections fast on SQL.** + +`Triples.historyPage(entityId, { attribute?, limit?, cursor?, recordedPosition? })` returns one page +of an entity's history, or of one attribute's history, oldest first by commit position and then fact +ID, including retracted assertions. The first page captures a commit position (the head, or +`recordedPosition`) and every continuation reads at that position, so writes between pages neither +repeat nor skip facts: a later assertion is not shown, and a fact retracted after the snapshot is shown +unretracted, as it was then. Pages default to 100 facts (maximum 1,000) and are read by an index seek, +so a page costs the same for an entity with 10 facts as for one with 10,000. The cursor is opaque and +bound to its database scope, entity, attribute and position; reusing it elsewhere fails with +`PaginationCursorError`. Sealed values stay sealed. The host protocol has a matching `HistoryPage` +operation. `history(entityId)` is unchanged. + +SQL migration 2 (`triplex_entity_attribute_history_index`) and Cloudflare migration 8 add +`idx_entity_attribute_history ON triples(entity_id, attribute, recorded_position)`. Besides serving +per-attribute history pages, it lets a Datalog `optionalProjection` look up each row's value directly; +before, SQLite planned those per-row lookups on an attribute-wide index. On the explorer's +1,200-worker demo database, a 50-row page sorted by an optional attribute drops from 6.2–8.7 s to +38–56 ms. + +`runMigrations` and the migrating layers upgrade a 0.5.0 database in place. The non-migrating and +read-only layers refuse it with `SchemaVersionError` (`pending: true`) until migrations run. In-memory +and FoundationDB KV stores backfill the new history keys once, in bounded batches, on the first history +page. diff --git a/docs/concepts.md b/docs/concepts.md index bc7bc9bc..83b98f52 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -38,7 +38,10 @@ Corrections therefore append evidence: or one attributed raw `Triples.transact` call. `history(entityId)` includes retracted assertions. Current `entity` and `match` reads show only -facts visible at their requested bitemporal basis. +facts visible at their requested bitemporal basis. For an entity with a long history, read it in +bounded pages with `historyPage(entityId, { attribute?, limit?, cursor? })`: pages are oldest first, +come from an index seek, and stay pinned to the first page's commit position, so writes between pages +neither repeat nor skip facts. ### End a fact at a time diff --git a/packages/cloudflare/src/migrations.ts b/packages/cloudflare/src/migrations.ts index 75c3f73d..17dbd298 100644 --- a/packages/cloudflare/src/migrations.ts +++ b/packages/cloudflare/src/migrations.ts @@ -8,6 +8,7 @@ import { import { NUMERIC_VALUE_INDEX_DDL, ENTITY_RECORDED_POSITION_INDEX_DDL, + ENTITY_ATTRIBUTE_HISTORY_INDEX_DDL, ENTITY_BLOBS_TABLE_DDL, ENTITY_SNAPSHOTS_TABLE_DDL, SNAPSHOT_INDEX_DDLS, @@ -68,4 +69,9 @@ export const migrations: readonly Migration[] = [ name: "triplex_journal_store", up: [...JOURNAL_DDLS, "DROP TABLE IF EXISTS triplex_command_receipts"], }, + { + version: 8, + name: "triplex_entity_attribute_history_index", + up: [ENTITY_ATTRIBUTE_HISTORY_INDEX_DDL], + }, ]; diff --git a/packages/cloudflare/src/schema.ts b/packages/cloudflare/src/schema.ts index 66900d7e..31e06ca1 100644 --- a/packages/cloudflare/src/schema.ts +++ b/packages/cloudflare/src/schema.ts @@ -2,6 +2,7 @@ import { NUMERIC_VALUE_INDEX_DDL, SEALED_BLIND_INDEX_DDL, ENTITY_RECORDED_POSITION_INDEX_DDL, + ENTITY_ATTRIBUTE_HISTORY_INDEX_DDL, } from "@triplex-build/triplex-sql"; /** @@ -79,6 +80,7 @@ export const INDEX_DDLS = [ NUMERIC_VALUE_INDEX_DDL, SEALED_BLIND_INDEX_DDL, ENTITY_RECORDED_POSITION_INDEX_DDL, + ENTITY_ATTRIBUTE_HISTORY_INDEX_DDL, ] as const; const TRIPLES_V3_COLUMNS = @@ -120,6 +122,7 @@ export const INDEX_NAMES = [ "idx_temporal", "idx_recorded_position", "idx_entity_recorded_position", + "idx_entity_attribute_history", "idx_entity_attr", "idx_tx_id", ] as const; diff --git a/packages/cloudflare/src/storage/CloudflareAdapter.ts b/packages/cloudflare/src/storage/CloudflareAdapter.ts index 24baadb7..c6e12721 100644 --- a/packages/cloudflare/src/storage/CloudflareAdapter.ts +++ b/packages/cloudflare/src/storage/CloudflareAdapter.ts @@ -28,6 +28,7 @@ import { makeSqlOperationalEntryStorage, makeSqlLivePreconditionLookup, makeSqlLiveFactsLookup, + makeSqlHistoryPage, makeSqlTransactionJournal, SqliteDialect, } from "@triplex-build/triplex-sql"; @@ -668,6 +669,15 @@ export function makeCloudflareAdapter(ctx: DOState): SqlStorageAdapterService { }, SqliteDialect, ), + historyPage: makeSqlHistoryPage( + { + run: >(statement: string, params: readonly unknown[]) => + Effect.try(() => + cursorToArray(sqlStorage.exec(statement, ...(params as SqlStorageValue[]))), + ), + }, + SqliteDialect, + ), livePreconditionFacts: makeSqlLivePreconditionLookup( { run: >(statement: string, params: readonly unknown[]) => diff --git a/packages/core/src/HistoryPage.ts b/packages/core/src/HistoryPage.ts new file mode 100644 index 00000000..26ee1838 --- /dev/null +++ b/packages/core/src/HistoryPage.ts @@ -0,0 +1,175 @@ +import { Result } from "effect"; +import { Base64Url } from "effect/encoding"; + +import { PaginationCursorError } from "./errors/index.js"; +import * as ContentId from "./content/ContentId.js"; + +export type HistoryPageCursor = string & { readonly HistoryPageCursor: unique symbol }; + +export interface HistoryPageRequest { + /** Only this attribute's assertions. Omit for every attribute of the entity. */ + readonly attribute?: string; + /** Defaults to 100; maximum 1,000. */ + readonly limit?: number; + /** Opaque continuation returned by the previous page of the same entity and attribute. */ + readonly cursor?: string; + /** + * Read history as recorded at this commit position. Defaults to the current head; a + * position beyond the head reads the head. A continuation must repeat the first page's + * value or omit it. + */ + readonly recordedPosition?: number; +} + +/** Where a history page resumes: after this assertion in (position, ID) order. */ +export interface HistoryPageAfter { + readonly position: number; + readonly id: string; +} + +interface CursorEnvelope { + readonly version: 1; + readonly scope: string; + readonly entityId: string; + readonly attribute: string | null; + readonly snapshotPosition: number; + readonly afterPosition: number; + readonly afterId: string; +} + +export interface PreparedHistoryPage { + readonly entityId: string; + readonly attribute?: string; + readonly limit: number; + readonly snapshotPosition: number; + readonly after?: HistoryPageAfter; + readonly scope: string; +} + +const decoder = new TextDecoder(); +const encoder = new TextEncoder(); + +const malformed = (message: string, cause?: unknown) => + new PaginationCursorError({ + reason: "malformed", + message, + ...(cause === undefined ? {} : { cause }), + }); + +const isPosition = (value: unknown): value is number => + typeof value === "number" && Number.isSafeInteger(value) && value >= 0; + +const decode = (cursor: string): CursorEnvelope => { + try { + const bytes = Base64Url.decode(cursor); + if (Result.isFailure(bytes)) throw bytes.failure; + const value: unknown = JSON.parse(decoder.decode(bytes.success)); + if (typeof value !== "object" || value === null || !("version" in value)) + throw malformed("History page cursor has an invalid shape"); + if (value.version !== 1) + throw new PaginationCursorError({ + reason: "unsupported_version", + message: "History page cursor has an unsupported version", + }); + const v = value as Partial; + if ( + typeof v.scope !== "string" || + typeof v.entityId !== "string" || + (v.attribute !== null && typeof v.attribute !== "string") || + !isPosition(v.snapshotPosition) || + !isPosition(v.afterPosition) || + v.afterPosition > v.snapshotPosition || + typeof v.afterId !== "string" + ) + throw malformed("History page cursor has an invalid shape"); + return v as CursorEnvelope; + } catch (cause) { + if (cause instanceof PaginationCursorError) throw cause; + throw malformed("History page cursor is not valid base64url JSON", cause); + } +}; + +const scopeFingerprint = (scope: string): string => + ContentId.hash(ContentId.Domain.paginationScope, `history-page\u0000${scope}`); + +/** + * Validate a history page request against the current head. Throws + * `PaginationCursorError` for invalid limits, positions and cursors. + */ +export const prepareHistoryPage = (input: { + readonly entityId: string; + readonly request: HistoryPageRequest; + readonly recordedPosition: number; + readonly scope: string; +}): PreparedHistoryPage => { + const { request } = input; + const limit = request.limit ?? 100; + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw malformed("History page limit must be between 1 and 1000"); + if (request.recordedPosition !== undefined && !isPosition(request.recordedPosition)) + throw malformed("History page recordedPosition must be a non-negative safe integer"); + if (request.attribute !== undefined && typeof request.attribute !== "string") + throw malformed("History page attribute must be a string"); + const scope = scopeFingerprint(input.scope); + const base = { + entityId: input.entityId, + ...(request.attribute === undefined ? {} : { attribute: request.attribute }), + limit, + scope, + }; + if (request.cursor === undefined) + return { + ...base, + snapshotPosition: Math.min( + request.recordedPosition ?? input.recordedPosition, + input.recordedPosition, + ), + }; + const envelope = decode(request.cursor); + if (envelope.scope !== scope) + throw new PaginationCursorError({ + reason: "scope_mismatch", + message: "History page cursor belongs to a different database scope", + }); + if (envelope.entityId !== input.entityId || envelope.attribute !== (request.attribute ?? null)) + throw new PaginationCursorError({ + reason: "query_mismatch", + message: "History page cursor belongs to a different entity or attribute", + }); + if ( + request.recordedPosition !== undefined && + Math.min(request.recordedPosition, input.recordedPosition) !== envelope.snapshotPosition + ) + throw new PaginationCursorError({ + reason: "basis_mismatch", + message: "History page cursor cannot be reused with a different recordedPosition", + }); + if (envelope.snapshotPosition > input.recordedPosition) + throw new PaginationCursorError({ + reason: "basis_mismatch", + message: "History page cursor is ahead of this database's committed head", + }); + return { + ...base, + snapshotPosition: envelope.snapshotPosition, + after: { position: envelope.afterPosition, id: envelope.afterId }, + }; +}; + +export const encodeHistoryPageCursor = ( + prepared: PreparedHistoryPage, + after: HistoryPageAfter, +): HistoryPageCursor => + Base64Url.encode( + encoder.encode( + JSON.stringify({ + version: 1, + scope: prepared.scope, + entityId: prepared.entityId, + attribute: prepared.attribute ?? null, + snapshotPosition: prepared.snapshotPosition, + afterPosition: after.position, + afterId: after.id, + } satisfies CursorEnvelope), + ), + ) as HistoryPageCursor; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index cea5acc0..4ce63f71 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -19,6 +19,7 @@ export { type EntityPageRequest, type EntityPageSnapshot, } from "./EntityPage.js"; +export { type HistoryPageCursor, type HistoryPageRequest } from "./HistoryPage.js"; export * from "./Value.js"; export * as Constraint from "./Constraint.js"; export * as CommitValidation from "./store/CommitValidation.js"; @@ -88,6 +89,7 @@ export { type EntityTransactionPage, type DependencyState, type EntityPage, + type HistoryPage, type QueryOptions, type PagedQueryOptions, type QueryResponse, diff --git a/packages/core/src/kv/hexastore/KvTripleStore.ts b/packages/core/src/kv/hexastore/KvTripleStore.ts index 6c0b75af..9bf58c5c 100644 --- a/packages/core/src/kv/hexastore/KvTripleStore.ts +++ b/packages/core/src/kv/hexastore/KvTripleStore.ts @@ -32,6 +32,10 @@ import { kvJournalAsync, kvJournalSync, type KvJournalEntry } from "../kv/Journa import { intervalKey, INTERVAL_READY_KEY, + historyIndexEntries, + historyRange, + HISTORY_READY_KEY, + HISTORY_CURSOR_KEY, INTERVAL_CURSOR_KEY, liveFactKey, liveFactKeyStr, @@ -641,6 +645,7 @@ export class KvTripleStore { ]); entries.push([metaKey(datom.tripleId), serializeDatom(datom)]); entries.push(...readIndexEntries(datom)); + entries.push(...historyIndexEntries(datom)); // TYPE index for entityType queries if (datom.entityType !== null) { @@ -698,6 +703,7 @@ export class KvTripleStore { ]); entries.push([metaKey(datom.tripleId), serializeDatom(datom)]); entries.push(...readIndexEntries(datom)); + entries.push(...historyIndexEntries(datom)); if (datom.entityType !== null) { entries.push([typeKey(datom.entityType, datom.tripleId), tripleIdValue]); @@ -756,6 +762,8 @@ export class KvTripleStore { ]; entries[idx++] = [metaKeyStr(tripleIdStr), serializeDatom(datom)]; for (const [key, value] of readIndexEntries(datom)) entries[idx++] = [keyString(key), value]; + for (const [key, value] of historyIndexEntries(datom)) + entries[idx++] = [keyString(key), value]; if (datom.entityType !== null) { const etStr = encodeStringStr(datom.entityType); @@ -1062,6 +1070,76 @@ export class KvTripleStore { }); } + private historyIndexesReady = false; + + /** Resumable bounded backfill of the history indexes for stores written before them. */ + ensureHistoryIndexes(): Effect.Effect { + return Effect.gen({ self: this }, function* () { + if (this.historyIndexesReady) return; + let complete = false; + while (!complete) { + complete = yield* this.kv.transact((tx) => + Effect.gen(function* () { + if ((yield* tx.get(HISTORY_READY_KEY))?.[0] === 1) return true; + const cursor = yield* tx.get(HISTORY_CURSOR_KEY); + const entries = yield* Stream.runCollect( + tx.getRange({ + start: cursor === null ? META_PREFIX : concat(cursor, new Uint8Array([0])), + end: new Uint8Array([0x21]), + limit: 501, + }), + ); + const batch = entries.slice(0, 500); + for (const [, bytes] of batch) + for (const [key, value] of historyIndexEntries(deserializeDatom(bytes))) + yield* tx.set(key, value); + if (entries.length <= 500) { + yield* tx.set(HISTORY_READY_KEY, new Uint8Array([1])); + yield* tx.delete(HISTORY_CURSOR_KEY); + return true; + } + yield* tx.set(HISTORY_CURSOR_KEY, batch.at(-1)![0]); + return false; + }), + ); + } + this.historyIndexesReady = true; + }); + } + + /** + * One page of retained history in (position, ID) order, read from the history + * index range and then by ID, so it never visits facts outside the page. + */ + historyPage( + entity: string, + attribute: string | undefined, + snapshotPosition: number, + after: { readonly position: number; readonly id: string } | undefined, + limit: number, + ): Effect.Effect { + return Effect.gen({ self: this }, function* () { + yield* this.ensureHistoryIndexes(); + const rows = yield* Stream.runCollect( + this.kv.getRange({ ...historyRange(entity, attribute, snapshotPosition, after), limit }), + ); + const facts: Datom[] = []; + for (const [, value] of rows) { + const fact = yield* this.getById(textDecoder.decode(value)); + if (fact === null) + return yield* new ReadError({ message: "Inconsistent KV history index" }); + facts.push(fact); + } + return facts; + }).pipe( + Effect.mapError((cause) => + cause instanceof ReadError + ? cause + : new ReadError({ message: "Failed to read KV history page", cause }), + ), + ); + } + private intervalIndexesReady = false; /** Resumable bounded backfill. No planner uses a partially built index. */ diff --git a/packages/core/src/kv/hexastore/indexes.ts b/packages/core/src/kv/hexastore/indexes.ts index c19f309c..87311ad4 100644 --- a/packages/core/src/kv/hexastore/indexes.ts +++ b/packages/core/src/kv/hexastore/indexes.ts @@ -48,6 +48,75 @@ export const liveFactRange = (entity: string, attribute?: string) => { return { start, end: increment(start) }; }; +/** + * Retained history in commit order: (entity, position, ID) and + * (entity, attribute, position, ID). Entries never change after the assert, + * because a retraction keeps the fact's position. + */ +export const ENTITY_HISTORY_PREFIX = new Uint8Array([0x0e]); +export const ENTITY_ATTRIBUTE_HISTORY_PREFIX = new Uint8Array([0x0f]); +export const HISTORY_READY_KEY = new Uint8Array([0x29, 1]); +export const HISTORY_CURSOR_KEY = new Uint8Array([0x29, 2]); +export const historyIndexEntries = (datom: { + readonly entity: string; + readonly attribute: string; + readonly recordedPosition: number; + readonly tripleId: string; +}): Array => { + const id = new TextEncoder().encode(datom.tripleId); + return [ + [ + concat( + ENTITY_HISTORY_PREFIX, + encodeTuple( + tString(datom.entity), + tNumber(datom.recordedPosition), + tString(datom.tripleId), + ), + ), + id, + ], + [ + concat( + ENTITY_ATTRIBUTE_HISTORY_PREFIX, + encodeTuple( + tString(datom.entity), + tString(datom.attribute), + tNumber(datom.recordedPosition), + tString(datom.tripleId), + ), + ), + id, + ], + ]; +}; +/** + * Key range of one history page: facts after `after` (exclusive) up to and + * including `snapshotPosition`, in (position, ID) order. + */ +export const historyRange = ( + entity: string, + attribute: string | undefined, + snapshotPosition: number, + after: { readonly position: number; readonly id: string } | undefined, +) => { + const prefix = + attribute === undefined + ? concat(ENTITY_HISTORY_PREFIX, encodeTuple(tString(entity))) + : concat(ENTITY_ATTRIBUTE_HISTORY_PREFIX, encodeTuple(tString(entity), tString(attribute))); + return { + start: + after === undefined + ? prefix + : concat( + prefix, + encodeTuple(tNumber(after.position), tString(after.id)), + new Uint8Array([0]), + ), + end: concat(prefix, encodeTuple(tNumber(snapshotPosition + 1))), + }; +}; + /** Historical interval access paths; retractions retain entries. */ export const INTERVAL_FROM_PREFIX = new Uint8Array([0x07]); export const INTERVAL_TO_PREFIX = new Uint8Array([0x08]); diff --git a/packages/core/src/kv/layers/KvTriplesLive.ts b/packages/core/src/kv/layers/KvTriplesLive.ts index 9c687010..71b4b2ba 100644 --- a/packages/core/src/kv/layers/KvTriplesLive.ts +++ b/packages/core/src/kv/layers/KvTriplesLive.ts @@ -88,6 +88,7 @@ import * as ContentIds from "../../content/ContentId.js"; import { unsafe } from "../../Branded.js"; import { transactionsForEntity } from "../../store/entityTransactionHistory.js"; import { encodeEntityPageCursor, prepareEntityPage } from "../../EntityPage.js"; +import { encodeHistoryPageCursor, prepareHistoryPage } from "../../HistoryPage.js"; import { tripleAtRecordedPosition } from "../../store/temporalView.js"; import { makeKvOperationalEntryStorage } from "../kv/OperationalEntries.js"; import { @@ -1029,6 +1030,49 @@ const makeKvTriplesService = Effect.gen(function* () { ), ), + historyPage: (entityId, request = {}) => + Effect.gen(function* () { + const head = yield* currentKvCommitPosition(kvBackend); + const prepared = yield* Effect.try({ + try: () => + prepareHistoryPage({ entityId, request, recordedPosition: head, scope: runtime.scope }), + catch: (cause) => + cause instanceof PaginationCursorError + ? cause + : new PaginationCursorError({ + reason: "malformed", + message: `Failed to prepare history page: ${String(cause)}`, + cause, + }), + }); + // One fact past the limit says whether another page exists. + const datoms = yield* hexaStore.historyPage( + entityId, + prepared.attribute, + prepared.snapshotPosition, + prepared.after, + prepared.limit + 1, + ); + const page = datoms.slice(0, prepared.limit); + const last = page.at(-1); + return { + facts: page.map((datom) => + datom.retractedPosition === null || datom.retractedPosition > prepared.snapshotPosition + ? tripleAtRecordedPosition(datomToTriple(datom), prepared.snapshotPosition) + : datomToTriple(datom), + ), + snapshotPosition: prepared.snapshotPosition, + ...(datoms.length > prepared.limit && last !== undefined + ? { + nextCursor: encodeHistoryPageCursor(prepared, { + position: last.recordedPosition, + id: last.tripleId, + }), + } + : {}), + }; + }), + transaction: (txId) => kvJournalEntry(kvBackend, txId).pipe(Effect.flatMap(hydrateOne)), transactionByCommand: (commandId) => diff --git a/packages/core/src/storage/StorageAdapter.ts b/packages/core/src/storage/StorageAdapter.ts index f46f3de7..de3b9dc6 100644 --- a/packages/core/src/storage/StorageAdapter.ts +++ b/packages/core/src/storage/StorageAdapter.ts @@ -20,6 +20,7 @@ import type { OperationalEntryRecord, PendingEntriesScan } from "../operational/ import type { ContentId } from "../content/ContentId.js"; import type { TransactionId } from "../Branded.js"; import type { JournalAppend, JournalEntry } from "../store/journal.js"; +import type { HistoryPageAfter } from "../HistoryPage.js"; /** * StorageAdapterService - Backend-agnostic storage primitives @@ -149,6 +150,18 @@ export interface StorageAdapterService { basis?: ResolvedTemporalBasis, ) => Effect.Effect; readonly history: (entityId: string) => Effect.Effect; + /** + * Indexed history page: one entity's facts (one attribute's, when given), retracted or + * not, recorded at or before `snapshotPosition` and after `after`, ordered by + * (recorded position, ID), at most `limit`. Must seek, not load the entity's history. + */ + readonly historyPage: ( + entityId: string, + attribute: string | undefined, + snapshotPosition: number, + after: HistoryPageAfter | undefined, + limit: number, + ) => Effect.Effect; readonly initialize: () => Effect.Effect; readonly close: () => Effect.Effect; } diff --git a/packages/core/src/store/Triples.ts b/packages/core/src/store/Triples.ts index 43cede96..8d34285e 100644 --- a/packages/core/src/store/Triples.ts +++ b/packages/core/src/store/Triples.ts @@ -49,6 +49,7 @@ import type { OperationalEntryConflictError } from "../errors/index.js"; import type { TripleValue } from "../Value.js"; import type { Rule as ConstraintRule } from "../Constraint.js"; import type { EntityPageCursor, EntityPageRequest, EntityPageSnapshot } from "../EntityPage.js"; +import type { HistoryPageCursor, HistoryPageRequest } from "../HistoryPage.js"; import type { QueryResult, QueryDebugInfo, @@ -189,6 +190,18 @@ export interface EntityPage { readonly nextCursor?: EntityPageCursor; } +export interface HistoryPage { + /** + * Assertions recorded at or before `snapshotPosition`, oldest first by commit position + * and then fact ID, including retracted ones. A retraction committed after the snapshot + * is not shown, so every page of one read sees the same history. + */ + readonly facts: readonly Triple[]; + /** Commit position the first page captured; continuations keep it. */ + readonly snapshotPosition: number; + readonly nextCursor?: HistoryPageCursor; +} + /** * Options accepted by the Datalog read methods. */ @@ -292,6 +305,15 @@ export interface TriplesService { ) => Effect.Effect; /** Full history (including retracted) for an entity. */ readonly history: (entityId: EntityId) => Effect.Effect; + /** + * One bounded page of an entity's history, or one attribute's, from an index seek. + * Pages are pinned to the first page's commit position, so writes between pages + * neither repeat nor skip facts. + */ + readonly historyPage: ( + entityId: EntityId, + request?: HistoryPageRequest, + ) => Effect.Effect; /** Read a persisted causal transaction envelope and its fact changes. */ readonly transaction: (txId: TransactionId) => Effect.Effect; /** Lookup the durable receipt for an atomically unique command ID. */ diff --git a/packages/core/src/store/TriplesLive.ts b/packages/core/src/store/TriplesLive.ts index a655d048..e15911c1 100644 --- a/packages/core/src/store/TriplesLive.ts +++ b/packages/core/src/store/TriplesLive.ts @@ -78,6 +78,7 @@ import { resolveTemporalBasis } from "../Temporal.js"; import { finishPagination, preparePagination, wrapDatalogQuery } from "../Pagination.js"; import { transactionsForEntity } from "./entityTransactionHistory.js"; import { encodeEntityPageCursor, prepareEntityPage } from "../EntityPage.js"; +import { encodeHistoryPageCursor, prepareHistoryPage } from "../HistoryPage.js"; import { commitOperationalView } from "./commitOperationalView.js"; import type * as CommitValidation from "./CommitValidation.js"; import { attemptFrom, runValidators, viewBasis } from "./commitValidationRun.js"; @@ -721,6 +722,50 @@ export const TriplesLive = Layer.effect( return rows.map(rowToTriple); }); + const historyPage: TriplesService["historyPage"] = (entityId, request = {}) => + Effect.gen(function* () { + const head = yield* adapter.currentCommitPosition(); + const prepared = yield* Effect.try({ + try: () => + prepareHistoryPage({ entityId, request, recordedPosition: head, scope: runtime.scope }), + catch: (cause) => + cause instanceof PaginationCursorError + ? cause + : new PaginationCursorError({ + reason: "malformed", + message: `Failed to prepare history page: ${String(cause)}`, + cause, + }), + }); + // One row past the limit says whether another page exists. + const rows = yield* adapter.historyPage( + entityId, + prepared.attribute, + prepared.snapshotPosition, + prepared.after, + prepared.limit + 1, + ); + const page = rows.slice(0, prepared.limit); + const last = page.at(-1); + return { + facts: page.map((row) => + row.retracted_position === null || + Number(row.retracted_position) > prepared.snapshotPosition + ? tripleAtRecordedPosition(rowToTriple(row), prepared.snapshotPosition) + : rowToTriple(row), + ), + snapshotPosition: prepared.snapshotPosition, + ...(rows.length > prepared.limit && last !== undefined + ? { + nextCursor: encodeHistoryPageCursor(prepared, { + position: Number(last.recorded_position), + id: last.id, + }), + } + : {}), + }; + }); + /** One batched primary-key read of every fact a page of entries references. */ const hydrate = (entries: readonly JournalEntry[]) => Effect.gen(function* () { @@ -1000,6 +1045,7 @@ export const TriplesLive = Layer.effect( entityPage, match, history, + historyPage, liveFacts: (entityId) => adapter.liveFacts(entityId).pipe(Effect.map((rows) => rows.map(rowToTriple))), transaction, diff --git a/packages/core/test/runtime/runtime.test.ts b/packages/core/test/runtime/runtime.test.ts index 57bc2021..04a6adbd 100644 --- a/packages/core/test/runtime/runtime.test.ts +++ b/packages/core/test/runtime/runtime.test.ts @@ -249,6 +249,7 @@ describe("public runtime composition", () => { getByEntities: unused, query: unused, history: unused, + historyPage: unused, initialize: () => Effect.fail(failure), close: () => Effect.void, }), diff --git a/packages/host/src/Protocol.ts b/packages/host/src/Protocol.ts index 15ec9f44..fe232fe8 100644 --- a/packages/host/src/Protocol.ts +++ b/packages/host/src/Protocol.ts @@ -35,6 +35,14 @@ export const DataOperation = Schema.Union([ basis: Schema.optional(TemporalBasis), }), Schema.Struct({ _tag: Schema.Literal("History"), entityId: EntityId }), + Schema.Struct({ + _tag: Schema.Literal("HistoryPage"), + entityId: EntityId, + attribute: Schema.optional(Schema.String), + limit: Schema.optional(Schema.Number), + cursor: Schema.optional(Schema.String), + recordedPosition: Schema.optional(Schema.Number), + }), Schema.Struct({ _tag: Schema.Literal("Query"), query: DatalogQuery, @@ -140,6 +148,15 @@ export const executeDataOperation = ( return triples.match(queryToPattern(operation.pattern), operation.basis); case "History": return triples.history(operation.entityId); + case "HistoryPage": + return triples.historyPage(operation.entityId, { + ...(operation.attribute === undefined ? {} : { attribute: operation.attribute }), + ...(operation.limit === undefined ? {} : { limit: operation.limit }), + ...(operation.cursor === undefined ? {} : { cursor: operation.cursor }), + ...(operation.recordedPosition === undefined + ? {} + : { recordedPosition: operation.recordedPosition }), + }); case "Query": return triples.query(operation.query, { ...queryOptions(operation.options), diff --git a/packages/host/test/host.test.ts b/packages/host/test/host.test.ts index 873cad24..df0e0fdb 100644 --- a/packages/host/test/host.test.ts +++ b/packages/host/test/host.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from "vitest"; -import { Cause, Effect, Exit, Layer, Schema } from "effect"; -import { EntityId, Triples, KvTriples, type PagedQueryResponse } from "@triplex-internal/core"; +import { Cause, Effect, Exit, Layer, Option, Schema } from "effect"; +import { + EntityId, + Triples, + KvTriples, + type HistoryPage, + type PagedQueryResponse, +} from "@triplex-internal/core"; import { SqliteTriples } from "@triplex-build/triplex-sqlite"; import { AuthorizationError, @@ -254,6 +260,32 @@ describe("tenant request boundary", () => { ).pipe(Effect.result); expect(ambiguous._tag).toBe("Failure"); } + // History pages travel through the decoded protocol with their cursor. + const names = (page: HistoryPage) => + page.facts.map((fact) => [ + (fact.value as { value: string }).value, + Option.isSome(fact.retractedAt), + ]); + const history = (yield* request({ + _tag: "HistoryPage", + entityId: home, + limit: 1, + })) as HistoryPage; + expect(names(history)).toEqual([["Alice", true]]); + const rest = (yield* request({ + _tag: "HistoryPage", + entityId: home, + limit: 1, + cursor: history.nextCursor!, + })) as HistoryPage; + expect(names(rest)).toEqual([["Alicia", false]]); + expect(rest.nextCursor).toBeUndefined(); + const asRecorded = (yield* request({ + _tag: "HistoryPage", + entityId: home, + recordedPosition: first.position, + })) as HistoryPage; + expect(names(asRecorded)).toEqual([["Alice", false]]); }).pipe(Effect.provide(backendLayer), Effect.provide(AuthorizerLive)), ); }, diff --git a/packages/postgres/src/PostgresqlAdapter.ts b/packages/postgres/src/PostgresqlAdapter.ts index f6690faf..6b4c6302 100644 --- a/packages/postgres/src/PostgresqlAdapter.ts +++ b/packages/postgres/src/PostgresqlAdapter.ts @@ -27,6 +27,7 @@ import { makeSqlOperationalEntryStorage, makeSqlLivePreconditionLookup, makeSqlLiveFactsLookup, + makeSqlHistoryPage, makeSqlTransactionJournal, packValue, runMigrations, @@ -632,6 +633,10 @@ export const makePostgresqlAdapter = (config: PostgresqlAdapterConfig = {}) => { run: (statement, params) => provide(sql.unsafe(statement, [...params])) }, PostgresqlDialect, ), + historyPage: makeSqlHistoryPage( + { run: (statement, params) => provide(sql.unsafe(statement, [...params])) }, + PostgresqlDialect, + ), livePreconditionFacts: makeSqlLivePreconditionLookup( { run: (statement, params) => provide(sql.unsafe(statement, [...params])) }, PostgresqlDialect, diff --git a/packages/sql/src/HistoryPage.ts b/packages/sql/src/HistoryPage.ts new file mode 100644 index 00000000..755bf0ab --- /dev/null +++ b/packages/sql/src/HistoryPage.ts @@ -0,0 +1,41 @@ +import { Effect } from "effect"; +import { + ReadError, + createParamCollector, + type SqlDialect, + type TripleRow, +} from "@triplex-internal/core/internal"; +import type { StorageAdapterService } from "@triplex-internal/core/runtime"; +import type { SqlStatementRunner } from "./SqlQueryExecutor.js"; + +/** + * A history page seeks `idx_entity_recorded_position`, or + * `idx_entity_attribute_history` for one attribute, from the cursor position and + * stops after `limit` rows, so its cost follows the page, not the entity's history. + */ +export const makeSqlHistoryPage = + (runner: SqlStatementRunner, dialect: SqlDialect): StorageAdapterService["historyPage"] => + (entityId, attribute, snapshotPosition, after, limit) => { + const p = createParamCollector(dialect); + const where = [`entity_id = ${p.add(entityId)}`]; + if (attribute !== undefined) where.push(`attribute = ${p.add(attribute)}`); + where.push(`recorded_position <= ${p.add(snapshotPosition)}`); + if (after !== undefined) { + const position = p.add(after.position); + // The leading range keeps the index seek; the disjunction orders ties by ID. + where.push( + `recorded_position >= ${position}`, + `(recorded_position > ${position} OR id > ${p.add(after.id)})`, + ); + } + return runner + .run( + `SELECT * FROM triples WHERE ${where.join(" AND ")} ORDER BY recorded_position, id LIMIT ${p.add(limit)}`, + p.params, + ) + .pipe( + Effect.mapError( + (cause) => new ReadError({ message: "Failed to read history page", cause }), + ), + ); + }; diff --git a/packages/sql/src/SchemaVersion.ts b/packages/sql/src/SchemaVersion.ts index a4361640..0680f3ce 100644 --- a/packages/sql/src/SchemaVersion.ts +++ b/packages/sql/src/SchemaVersion.ts @@ -3,7 +3,8 @@ import { Data } from "effect"; /** * The database's schema is not the one this release uses. * - * `actualVersion` is zero when no Triplex schema exists: run `runMigrations` + * `actualVersion` is zero when no Triplex schema exists, and `pending` is set + * when the database has an earlier schema of this release: run `runMigrations` * (or use a migrating layer) first. Any other history, including every * database written by Triplex 0.4 or earlier, is refused: this release stores * the transaction journal in its own tables and does not migrate older @@ -15,11 +16,16 @@ export class SchemaVersionError extends Data.TaggedError("SchemaVersionError")<{ readonly actualVersion: number; /** Name of the latest recorded migration, when there is one. */ readonly actualName?: string; + /** The database has an earlier schema of this release that `runMigrations` upgrades. */ + readonly pending?: boolean; }> { override get message(): string { if (this.actualVersion === 0) { return `Triplex requires schema version ${this.expectedVersion}, found no Triplex schema. Run runMigrations from @triplex-build/triplex/sql before constructing the layer.`; } + if (this.pending === true) { + return `Triplex requires schema version ${this.expectedVersion}, found version ${this.actualVersion}${this.actualName === undefined ? "" : ` (${this.actualName})`}. Run runMigrations from @triplex-build/triplex/sql to apply the pending migrations before constructing the layer.`; + } return `Triplex requires schema version ${this.expectedVersion}, found version ${this.actualVersion}${this.actualName === undefined ? "" : ` (${this.actualName})`}. This database uses an incompatible storage layout: databases created by Triplex 0.4 or earlier keep the transaction journal in the triples table and cannot be opened or migrated by this release. Start a fresh database, or use the Triplex release that created it.`; } } diff --git a/packages/sql/src/index.ts b/packages/sql/src/index.ts index 128446bb..441974d0 100644 --- a/packages/sql/src/index.ts +++ b/packages/sql/src/index.ts @@ -26,6 +26,7 @@ export { NUMERIC_VALUE_INDEX_DDL, SEALED_BLIND_INDEX_DDL, ENTITY_RECORDED_POSITION_INDEX_DDL, + ENTITY_ATTRIBUTE_HISTORY_INDEX_DDL, INDEX_NAMES, ENTITY_BLOBS_TABLE_DDL, ENTITY_SNAPSHOTS_TABLE_DDL, @@ -76,6 +77,7 @@ export { DatabaseManagerLive } from "./DatabaseManagerLayer.js"; export { makeSqlOperationalEntryStorage } from "./OperationalEntries.js"; export { makeSqlLivePreconditionLookup, makeSqlLiveFactsLookup } from "./LivePreconditions.js"; +export { makeSqlHistoryPage } from "./HistoryPage.js"; export { makeSqlTransactionJournal, diff --git a/packages/sql/src/migrations.ts b/packages/sql/src/migrations.ts index 030b4b39..1a7d1fd5 100644 --- a/packages/sql/src/migrations.ts +++ b/packages/sql/src/migrations.ts @@ -10,6 +10,7 @@ import { NUMERIC_VALUE_INDEX_DDL, SEALED_BLIND_INDEX_DDL, ENTITY_RECORDED_POSITION_INDEX_DDL, + ENTITY_ATTRIBUTE_HISTORY_INDEX_DDL, ENTITY_BLOBS_TABLE_DDL, ENTITY_SNAPSHOTS_TABLE_DDL, SNAPSHOT_INDEX_DDLS, @@ -47,7 +48,8 @@ export interface Migration { export const BASELINE_MIGRATION_NAME = "triplex_journal_store_baseline"; /** - * The Triplex schema: one baseline that creates the current layout directly. + * The Triplex schema: a baseline that creates the 0.5 layout directly, then + * additive migrations that a baseline database applies in order. * * DDL is imported from schema.ts so explicit host-owned migration execution and * convenience auto-migration use the same definitions. PostgreSQL creates its @@ -84,6 +86,12 @@ export const migrations: readonly Migration[] = [ ...KEY_STORE_DDLS, ], }, + { + // Bounded per-attribute history pages and optional-projection lookups. + version: 2, + name: "triplex_entity_attribute_history_index", + up: [ENTITY_ATTRIBUTE_HISTORY_INDEX_DDL], + }, ]; /** The schema version this release requires. */ @@ -97,27 +105,27 @@ export interface SchemaHistoryRow { /** * Compare a database's recorded schema history with this release's. Returns * `"empty"` for a database with no history, `"current"` for exactly this - * release's baseline, and a `SchemaVersionError` for anything else, including - * every database written by Triplex 0.4 or earlier. + * release's migrations, `"pending"` for a strict prefix of them that + * `runMigrations` completes, and a `SchemaVersionError` for anything else, + * including every database written by Triplex 0.4 or earlier. */ export const checkSchemaHistory = ( history: ReadonlyArray, -): "empty" | "current" | SchemaVersionError => { +): "empty" | "current" | "pending" | SchemaVersionError => { if (history.length === 0) return "empty"; const rows = [...history].sort((a, b) => Number(a.version) - Number(b.version)); - const matches = - rows.length === migrations.length && + const prefix = + rows.length <= migrations.length && rows.every( (row, index) => Number(row.version) === migrations[index]!.version && row.name === migrations[index]!.name, ); - return matches - ? "current" - : new SchemaVersionError({ - expectedVersion: SCHEMA_VERSION, - actualVersion: Math.max(...rows.map((row) => Number(row.version))), - actualName: rows.at(-1)!.name, - }); + if (prefix) return rows.length === migrations.length ? "current" : "pending"; + return new SchemaVersionError({ + expectedVersion: SCHEMA_VERSION, + actualVersion: Math.max(...rows.map((row) => Number(row.version))), + actualName: rows.at(-1)!.name, + }); }; /** @@ -147,19 +155,27 @@ export const readSchemaHistory = Effect.gen(function* () { /** * Fail with `SchemaVersionError` unless the database has this release's schema, - * or (when `allowEmpty`) no Triplex schema yet. Every layer, migrating or not, - * runs this before touching the database, so a 0.4 database is refused rather - * than half-upgraded. + * or (when `allowEmpty`, for a layer that migrates) no Triplex schema yet or + * pending migrations. Every layer, migrating or not, runs this before touching + * the database, so a 0.4 database is refused rather than half-upgraded. */ export const checkSchema = (options: { readonly allowEmpty: boolean }) => readSchemaHistory.pipe( Effect.flatMap((history) => { const status = checkSchemaHistory(history); - if (status === "current" || (status === "empty" && options.allowEmpty)) return Effect.void; + if (status === "current") return Effect.void; + if (status instanceof SchemaVersionError) return Effect.fail(status); + if (options.allowEmpty) return Effect.void; + const latest = [...history].sort((a, b) => Number(a.version) - Number(b.version)).at(-1); return Effect.fail( - status === "empty" + latest === undefined ? new SchemaVersionError({ expectedVersion: SCHEMA_VERSION, actualVersion: 0 }) - : status, + : new SchemaVersionError({ + expectedVersion: SCHEMA_VERSION, + actualVersion: Number(latest.version), + actualName: latest.name, + pending: true, + }), ); }), ); @@ -167,9 +183,11 @@ export const checkSchema = (options: { readonly allowEmpty: boolean }) => export const runMigrations = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - const status = checkSchemaHistory(yield* readSchemaHistory); + const history = yield* readSchemaHistory; + const status = checkSchemaHistory(history); if (status === "current") return; - if (status !== "empty") return yield* status; + if (status !== "empty" && status !== "pending") return yield* status; + const applied = Math.max(0, ...history.map((row) => Number(row.version))); yield* sql` CREATE TABLE IF NOT EXISTS triplex_schema_migrations ( @@ -179,7 +197,7 @@ export const runMigrations = Effect.gen(function* () { ) `; - for (const migration of migrations) { + for (const migration of migrations.filter(({ version }) => version > applied)) { const dialectStatements = sql.onDialectOrElse({ orElse: () => [] as ReadonlyArray, sqlite: () => migration.dialects?.sqlite ?? [], diff --git a/packages/sql/src/schema.ts b/packages/sql/src/schema.ts index 36e12913..ff2631df 100644 --- a/packages/sql/src/schema.ts +++ b/packages/sql/src/schema.ts @@ -207,6 +207,14 @@ export const SEALED_BLIND_INDEX_DDL = export const ENTITY_RECORDED_POSITION_INDEX_DDL = "CREATE INDEX IF NOT EXISTS idx_entity_recorded_position ON triples(entity_id, recorded_position)"; +/** + * One attribute of one entity at any recorded cut, retracted rows included: bounded + * history pages for one attribute, and Datalog optional projections, which otherwise + * plan on an attribute-wide index and scan every entity's values for each row. + */ +export const ENTITY_ATTRIBUTE_HISTORY_INDEX_DDL = + "CREATE INDEX IF NOT EXISTS idx_entity_attribute_history ON triples(entity_id, attribute, recorded_position)"; + /** Non-partial indexes preserve interval access at historical recorded cuts. */ export const INTERVAL_HISTORY_INDEX_DDLS = [ "CREATE INDEX IF NOT EXISTS idx_attribute_valid_to_history ON triples (attribute, valid_to, recorded_position)", @@ -226,6 +234,7 @@ export const INDEX_DDLS = [ SEALED_BLIND_INDEX_DDL, ENTITY_RECORDED_POSITION_INDEX_DDL, ...INTERVAL_HISTORY_INDEX_DDLS, + ENTITY_ATTRIBUTE_HISTORY_INDEX_DDL, ] as const; /** @@ -248,6 +257,7 @@ export const INDEX_NAMES = [ "idx_tx_id", "idx_attribute_valid_to_history", "idx_attribute_valid_from_history", + "idx_entity_attribute_history", ] as const; // ============================================================================= diff --git a/packages/sql/test/sql-contract.test.ts b/packages/sql/test/sql-contract.test.ts index af03e781..fa1918fa 100644 --- a/packages/sql/test/sql-contract.test.ts +++ b/packages/sql/test/sql-contract.test.ts @@ -11,9 +11,10 @@ import { } from "../src/index.js"; describe("SQL package contract", () => { - it("keeps one baseline migration with every index declared", () => { + it("keeps the baseline and additive migrations with every index declared", () => { expect(migrations.map((migration) => [migration.version, migration.name])).toEqual([ [1, BASELINE_MIGRATION_NAME], + [2, "triplex_entity_attribute_history_index"], ]); const statements = migrations.flatMap((migration) => migration.up); expect(new Set(statements).size).toBe(statements.length); @@ -27,7 +28,10 @@ describe("SQL package contract", () => { it("accepts only this release's schema history", () => { expect(checkSchemaHistory([])).toBe("empty"); - expect(checkSchemaHistory([{ version: 1, name: BASELINE_MIGRATION_NAME }])).toBe("current"); + const current = migrations.map(({ version, name }) => ({ version, name })); + expect(checkSchemaHistory(current)).toBe("current"); + // A 0.5 baseline database upgrades in place. + expect(checkSchemaHistory([{ version: 1, name: BASELINE_MIGRATION_NAME }])).toBe("pending"); for (const history of [ [{ version: 1, name: "triplex_baseline" }], Array.from({ length: 13 }, (_, index) => ({ version: index + 1, name: `v${index + 1}` })), @@ -35,6 +39,7 @@ describe("SQL package contract", () => { { version: 1, name: BASELINE_MIGRATION_NAME }, { version: 2, name: "from_a_newer_release" }, ], + [...current, { version: 3, name: "from_a_newer_release" }], ]) { const status = checkSchemaHistory(history); expect(status).toBeInstanceOf(SchemaVersionError); diff --git a/packages/sqlite/src/SqliteAdapter.ts b/packages/sqlite/src/SqliteAdapter.ts index 2735efdb..75892df3 100644 --- a/packages/sqlite/src/SqliteAdapter.ts +++ b/packages/sqlite/src/SqliteAdapter.ts @@ -27,6 +27,7 @@ import { makeSqlOperationalEntryStorage, makeSqlLivePreconditionLookup, makeSqlLiveFactsLookup, + makeSqlHistoryPage, makeSqlTransactionJournal, SqliteDialect, checkSchema, @@ -721,6 +722,10 @@ export const makeSqliteAdapter = (config: SqliteAdapterConfig = {}) => { run: (statement, params) => provide(sql.unsafe(statement, [...params])) }, SqliteDialect, ), + historyPage: makeSqlHistoryPage( + { run: (statement, params) => provide(sql.unsafe(statement, [...params])) }, + SqliteDialect, + ), livePreconditionFacts: makeSqlLivePreconditionLookup( { run: (statement, params) => provide(sql.unsafe(statement, [...params])) }, SqliteDialect, diff --git a/packages/sqlite/test/concurrent-first-open.test.ts b/packages/sqlite/test/concurrent-first-open.test.ts index 029ec85b..c2584303 100644 --- a/packages/sqlite/test/concurrent-first-open.test.ts +++ b/packages/sqlite/test/concurrent-first-open.test.ts @@ -9,6 +9,7 @@ import { afterEach, expect, it } from "vitest"; import { Effect, Layer } from "effect"; import { SqlClient } from "effect/sql"; import { Triples } from "@triplex-internal/core"; +import { migrations } from "@triplex-build/triplex-sql"; import { SqliteTriples } from "../src/SqliteTriples.js"; const directories: string[] = []; @@ -42,7 +43,7 @@ it( Effect.all([open, open, open], { concurrency: "unbounded" }), ); expect(first!.position).toBe(0); - expect(first!.history).toHaveLength(1); + expect(first!.history).toHaveLength(migrations.length); expect(second).toEqual(first); } }, diff --git a/packages/sqlite/test/migrations.test.ts b/packages/sqlite/test/migrations.test.ts index f1b96346..04229918 100644 --- a/packages/sqlite/test/migrations.test.ts +++ b/packages/sqlite/test/migrations.test.ts @@ -73,7 +73,7 @@ const schemaVersionErrorOf = async (effect: Effect.Effect) => }; describe("host-owned SQLite migrations", () => { - it("starts without DDL and applies the one baseline idempotently", async () => { + it("starts without DDL and applies every migration idempotently", async () => { const result = await runUnmigrated( Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -105,12 +105,12 @@ describe("host-owned SQLite migrations", () => { ); expect(result.before).toHaveLength(0); - expect(migrations.map(({ version, name }) => [version, name])).toEqual([ + const expected = [ [1, BASELINE_MIGRATION_NAME], - ]); - expect(result.applied.map(({ version, name }) => [version, name])).toEqual([ - [1, BASELINE_MIGRATION_NAME], - ]); + [2, "triplex_entity_attribute_history_index"], + ]; + expect(migrations.map(({ version, name }) => [version, name])).toEqual(expected); + expect(result.applied.map(({ version, name }) => [version, name])).toEqual(expected); expect(result.columns.map(({ name }) => name)).toEqual( expect.arrayContaining([ "recorded_at", @@ -163,6 +163,7 @@ describe("host-owned SQLite migrations", () => { "idx_attr_blind", "idx_entity", "idx_entity_recorded_position", + "idx_entity_attribute_history", "idx_tx_id", ]), ); @@ -181,7 +182,7 @@ describe("host-owned SQLite migrations", () => { const client = makeSqliteLayerUnmigrated(filename); const migrated = await schemaVersionErrorOf(runMigrations.pipe(Effect.provide(client))); - expect(migrated).toMatchObject({ expectedVersion: 1, actualVersion: 13 }); + expect(migrated).toMatchObject({ expectedVersion: 2, actualVersion: 13 }); expect(migrated.message).toContain("Triplex 0.4 or earlier"); expect(migrated.message).toContain("Start a fresh database"); @@ -225,6 +226,58 @@ describe("host-owned SQLite migrations", () => { expect(untouched.tables).toEqual([]); }); + it("upgrades a 0.5 baseline database in place and keeps its facts", async () => { + const filename = await tempFile(); + const fact = await Effect.runPromise( + Effect.gen(function* () { + const triples = yield* Triples; + const sql = yield* SqlClient.SqlClient; + const written = yield* triples.assert({ + entityId: "upgrade:a" as never, + attribute: ":upgrade/name", + value: { type: "string", value: "Ada" }, + }); + // Leave the file as the 0.5.0 baseline wrote it. + yield* sql`DROP INDEX idx_entity_attribute_history`; + yield* sql`DELETE FROM triplex_schema_migrations WHERE version = 2`; + return written; + }).pipe( + Effect.provide(Layer.fresh(SqliteTriples.layerWithSqlClient({ filename }))), + Effect.scoped, + ), + ); + + const pending = await schemaVersionErrorOf( + Triples.pipe( + Effect.provide(Layer.fresh(SqliteTriples.layerReadOnly({ filename }))), + Effect.scoped, + ), + ); + expect(pending).toMatchObject({ expectedVersion: 2, actualVersion: 1, pending: true }); + expect(pending.message).toContain("Run runMigrations"); + + const upgraded = await Effect.runPromise( + Effect.gen(function* () { + const triples = yield* Triples; + const sql = yield* SqlClient.SqlClient; + return { + history: (yield* triples.historyPage(fact.entityId, { attribute: ":upgrade/name" })) + .facts, + versions: yield* sql<{ version: number }>` + SELECT version FROM triplex_schema_migrations ORDER BY version`, + index: yield* sql<{ name: string }>` + SELECT name FROM sqlite_master WHERE name = 'idx_entity_attribute_history'`, + }; + }).pipe( + Effect.provide(Layer.fresh(SqliteTriples.layerWithSqlClient({ filename }))), + Effect.scoped, + ), + ); + expect(upgraded.history.map((f) => f.id)).toEqual([fact.id]); + expect(upgraded.versions.map(({ version }) => Number(version))).toEqual([1, 2]); + expect(upgraded.index).toHaveLength(1); + }); + it("refuses the 0.1 baseline history too, though its version number matches", async () => { const filename = await tempFile(); await write04Database(filename, ["triplex_baseline"]); diff --git a/packages/testkit/src/historyPageCases.ts b/packages/testkit/src/historyPageCases.ts new file mode 100644 index 00000000..47d9a65f --- /dev/null +++ b/packages/testkit/src/historyPageCases.ts @@ -0,0 +1,150 @@ +import { Effect, Option } from "effect"; +import { + EntityId, + PaginationCursorError, + Triples, + string, + type Triple, +} from "@triplex-internal/core"; + +const check = (condition: boolean, message: string) => + condition ? Effect.void : Effect.fail(new Error(message)); + +const key = (fact: Triple) => + JSON.stringify([fact.id, fact.attribute, fact.value, Option.isSome(fact.retractedAt)]); + +/** Reads every page of one history read and returns the facts in page order. */ +const readAll = ( + entityId: EntityId, + request: { readonly attribute?: string; readonly limit: number }, + between?: () => Effect.Effect, +) => + Effect.gen(function* () { + const t = yield* Triples; + const facts: Triple[] = []; + let cursor: string | undefined; + let pages = 0; + let snapshot: number | undefined; + do { + const page = yield* t.historyPage(entityId, { + ...request, + ...(cursor === undefined ? {} : { cursor }), + }); + yield* check(page.facts.length <= request.limit, "a history page exceeds its limit"); + yield* check( + snapshot === undefined || page.snapshotPosition === snapshot, + "a continuation changed the snapshot position", + ); + snapshot = page.snapshotPosition; + facts.push(...page.facts); + cursor = page.nextCursor; + pages++; + if (between !== undefined && cursor !== undefined) yield* between(); + } while (cursor !== undefined); + return { facts, pages }; + }); + +/** Bounded history pages, checked against the unpaged `history` oracle. */ +export const historyPageConformanceCases = [ + { + name: "historyPage pages equal unpaged history, pinned against writes between pages", + run: Effect.gen(function* () { + const t = yield* Triples; + const entityId = EntityId.make("conf:history-page"); + const other = EntityId.make("conf:history-page:other"); + const ids: Triple["id"][] = []; + for (let step = 0; step < 23; step++) { + const attribute = step % 3 === 0 ? ":hp/b" : ":hp/a"; + // Several facts per transaction share a position, so the ID breaks ties. + const result = yield* t.transact([ + { op: "assert", entityId, attribute, value: string(`v${step}`), validFrom: 0 }, + { op: "assert", entityId, attribute: ":hp/a", value: string(`w${step}`), validFrom: 0 }, + { op: "assert", entityId: other, attribute, value: string(`o${step}`), validFrom: 0 }, + ]); + ids.push(result.triples[0]!.id, result.triples[1]!.id); + if (step % 4 === 1) yield* t.retract(ids[step]!); + } + const oracle = yield* t.history(entityId); + for (const attribute of [undefined, ":hp/a", ":hp/b"]) { + const expected = oracle + .filter((fact) => attribute === undefined || fact.attribute === attribute) + .map(key) + .sort(); + for (const limit of [1, 2, 7, 1000]) { + const { facts, pages } = yield* readAll(entityId, { + limit, + ...(attribute === undefined ? {} : { attribute }), + }); + const label = `attribute ${attribute ?? "*"} limit ${limit}`; + yield* check( + JSON.stringify(facts.map(key).sort()) === JSON.stringify(expected), + `${label}: pages differ from history`, + ); + yield* check( + new Set(facts.map((fact) => fact.id)).size === facts.length, + `${label}: a fact repeated across pages`, + ); + yield* check( + facts.every((fact, i) => i === 0 || facts[i - 1]!.recordedAt <= fact.recordedAt), + `${label}: pages are not in commit order`, + ); + // limit + 1: an exactly full last page has no continuation. + yield* check( + pages === Math.max(1, Math.ceil(expected.length / limit)), + `${label}: ${pages} pages for ${expected.length} facts`, + ); + } + } + + // Writes between pages: later assertions are not shown, and a later retraction + // shows the fact as live, as it was at the first page's position. + const before = yield* t.history(entityId); + const live = before.filter((fact) => Option.isNone(fact.retractedAt)); + let retracted = 0; + const { facts } = yield* readAll(entityId, { limit: 3 }, () => + Effect.gen(function* () { + yield* t.assert({ entityId, attribute: ":hp/a", value: string("late"), validFrom: 0 }); + const target = live[retracted++]; + if (target !== undefined) yield* t.retract(target.id); + }), + ); + yield* check( + JSON.stringify(facts.map(key).sort()) === JSON.stringify(before.map(key).sort()), + "writes between pages changed the pinned history", + ); + + // An explicit recorded position reads history as recorded there. + const first = yield* t.historyPage(entityId, { recordedPosition: 0 }); + yield* check(first.facts.length === 0 && first.nextCursor === undefined, "position zero"); + + // Cursors are bound to their entity, attribute and position. + const page = yield* t.historyPage(entityId, { limit: 1 }); + const reason = (request: Parameters[1], id = entityId) => + t.historyPage(id, request).pipe( + Effect.match({ + onFailure: (error) => (error instanceof PaginationCursorError ? error.reason : "other"), + onSuccess: () => "accepted", + }), + ); + yield* check( + (yield* reason({ cursor: page.nextCursor! }, other)) === "query_mismatch", + "a cursor was accepted for another entity", + ); + yield* check( + (yield* reason({ cursor: page.nextCursor!, attribute: ":hp/a" })) === "query_mismatch", + "a cursor was accepted for another attribute", + ); + yield* check( + (yield* reason({ cursor: page.nextCursor!, recordedPosition: 1 })) === "basis_mismatch", + "a cursor was accepted at another position", + ); + for (const cursor of ["not a cursor", "eyJ2ZXJzaW9uIjoxfQ"]) + yield* check( + (yield* reason({ cursor })) === "malformed", + `a malformed cursor was accepted: ${cursor}`, + ); + for (const limit of [0, 1001, 1.5]) + yield* check((yield* reason({ limit })) === "malformed", `limit ${limit} was accepted`); + }), + }, +] as const; diff --git a/packages/testkit/src/index.ts b/packages/testkit/src/index.ts index 2fa1d806..bddcf299 100644 --- a/packages/testkit/src/index.ts +++ b/packages/testkit/src/index.ts @@ -1,4 +1,5 @@ import { datalogTimeConformanceCases } from "./datalogTimeCases.js"; +import { historyPageConformanceCases } from "./historyPageCases.js"; /** * Shared backend test helpers for Triplex. * @@ -125,6 +126,7 @@ export const triplesConformanceCases: readonly ConformanceCase[] = [ }, ...commitValidatorConformanceCases, ...datalogTimeConformanceCases, + ...historyPageConformanceCases, { name: "entity-prefix feeds are the ordered complete subsequence of the global journal", run: Effect.gen(function* () {