Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,11 @@ This project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

Packages without a separate changelog are covered by the cross-package notes below.

## [Unreleased]
## [Unreleased - Patch]

### Fixed

- Channel and DM message history (`GET /v1/channels/:name/messages`, `GET /v1/dm/:conversation_id/messages` and the workspace-key `GET /v1/dm/conversations/:conversation_id/messages`) now lists messages newest first in true numeric id order, and `before`/`after` cursors no longer skip messages. Message ids grew from 17 to 18 digits in October 2025, and history spanning that change was misordered.

## [12.13.1] - 2026-11-20

Expand Down
6 changes: 5 additions & 1 deletion packages/engine/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@ See the [root changelog](../../CHANGELOG.md) for cross-package release highlight
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
## [Unreleased - Patch]

### Fixed

- `getMessages`, `getDmMessages` and `getDmMessagesForWorkspace` order and compare message ids numerically, by `(length(id), id)`, instead of as text. Snowflake ids are decimal TEXT that widened from 17 to 18 digits in October 2025, so history spanning that change was misordered and `before`/`after` cursors skipped messages across it. A `before` page is read in at most two index seeks. The new `idx_messages_channel_length_id` index `(channel_id, length(id), id)` replaces `idx_messages_channel_time` (SQLite migration 0087, Postgres 0008, where the id is indexed `COLLATE "C"`).

## [12.13.1] - 2026-11-20

Expand Down
122 changes: 122 additions & 0 deletions packages/engine/src/__tests__/conformance/mixedWidthHistory.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
import { eq } from 'drizzle-orm';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { dmConversations, messages } from '../../db/tables.js';
import { createWorkspace, makeNodeStack, registerAgent, type TestStack } from './harness.js';
import { conformancePgDb } from './pgMode.js';

/**
* Snowflake ids are decimal TEXT that widened from 17 to 18 digits in 2025-10.
* History spanning that change must page in numeric order: bare text order
* puts every `9…` 17-digit id after (newer than) any `1…` 18-digit id, and
* text `before`/`after` cursors skip across the boundary.
*/
const NARROW = ['98000000000000001', '98000000000000002', '98000000000000003']; // 17 digits, older
const WIDE = ['100000000000000001', '100000000000000002', '100000000000000003']; // 18 digits, newer

describe('message history across the snowflake width change', () => {
let stack: TestStack;
beforeEach(() => { stack = makeNodeStack(); });
afterEach(async () => { await stack.close(); });

const get = async (token: string, path: string) => {
const res = await stack.app.request(path, { headers: { authorization: `Bearer ${token}` } });
expect(res.status).toBe(200);
return ((await res.json()).data as Array<{ id: string }>).map((m) => m.id);
};

async function seed(workspaceId: string, channelId: string, agentId: string) {
const ids = [...NARROW, ...WIDE];
await stack.runtime.deps.db.insert(messages).values(ids.map((id, i) => ({
id,
workspaceId,
channelId,
agentId,
body: `m-${id}`,
createdAt: new Date(Date.UTC(2025, 9, 1, 0, 0, i)),
})));
}

/** Every cursor shape a client uses, against one history endpoint. */
async function expectNumericPaging(token: string, path: string, newest: string[]) {
const all = [...newest, ...[...WIDE].reverse(), ...[...NARROW].reverse()];
const q = (params: string) => get(token, `${path}?${params}`);

expect(await q('limit=100')).toEqual(all);
// `before` crossing the width boundary, entirely and mid-page.
expect(await q(`limit=100&before=${WIDE[0]}`)).toEqual([...NARROW].reverse());
expect(await q(`limit=2&before=${WIDE[1]}`)).toEqual([WIDE[0], NARROW[2]]);
// `after` crossing it (newest-first page of everything newer).
expect(await q(`limit=100&after=${NARROW[2]}`)).toEqual([...newest, ...[...WIDE].reverse()]);
expect(await q(`limit=100&after=${NARROW[0]}&before=${WIDE[1]}`)).toEqual([WIDE[0], NARROW[2], NARROW[1]]);
// Equal-width cursors are unchanged.
expect(await q(`limit=1&before=${WIDE[2]}`)).toEqual([WIDE[1]]);
expect(await q(`limit=100&before=${NARROW[2]}`)).toEqual([NARROW[1], NARROW[0]]);
expect(await q(`limit=100&after=${WIDE[0]}`)).toEqual([...newest, WIDE[2], WIDE[1]]);

// Walking `before` pages visits every message exactly once, in order.
const walked: string[] = [];
let page = await q('limit=2');
while (page.length > 0) {
walked.push(...page);
page = await q(`limit=2&before=${page[page.length - 1]}`);
}
expect(walked).toEqual(all);
}

it('pages channel history numerically through the channel length-id index', async () => {
const ws = await createWorkspace(stack.app, 'mixed-width-channel');
const agent = await registerAgent(stack.app, ws.workspaceKey, 'author');
const created = await stack.app.request('/v1/channels', {
method: 'POST',
headers: { authorization: `Bearer ${agent.token}`, 'content-type': 'application/json' },
body: JSON.stringify({ name: 'history' }),
});
expect(created.status).toBe(201);
const channelId = (await created.json()).data.id as string;
await seed(ws.workspaceId, channelId, agent.agentId);

await expectNumericPaging(agent.token, '/v1/channels/history/messages', []);

if (conformancePgDb()) return;
// Both page shapes seek the channel index without a sort.
const sqlite = stack.runtime.handle.sqlite;
const plan = (where: string, order: string) => (sqlite.prepare(`
EXPLAIN QUERY PLAN SELECT messages.id FROM messages INDEXED BY idx_messages_channel_length_id
WHERE messages.channel_id = ? AND messages.workspace_id = ? AND messages.thread_id IS NULL AND ${where}
ORDER BY ${order} LIMIT 50
`).all(channelId, ws.workspaceId) as Array<{ detail: string }>).map((s) => s.detail).join('\n');
const sameWidth = plan(`length(messages.id) = 18 AND messages.id < '${WIDE[1]}'`, 'messages.id DESC');
expect(sameWidth).toContain('idx_messages_channel_length_id (channel_id=? AND <expr>=? AND id<?)');
expect(sameWidth).not.toContain('TEMP B-TREE');
const narrower = plan('length(messages.id) < 18', 'length(messages.id) DESC, messages.id DESC');
expect(narrower).toContain('idx_messages_channel_length_id (channel_id=? AND <expr><?)');
expect(narrower).not.toContain('TEMP B-TREE');
});

it('pages DM history numerically on the agent and workspace-key routes', async () => {
const ws = await createWorkspace(stack.app, 'mixed-width-dm');
const alice = await registerAgent(stack.app, ws.workspaceKey, 'alice');
await registerAgent(stack.app, ws.workspaceKey, 'bob');
const sent = await stack.app.request('/v1/dm', {
method: 'POST',
headers: { authorization: `Bearer ${alice.token}`, 'content-type': 'application/json' },
body: JSON.stringify({ to: 'bob', text: 'opener' }),
});
expect(sent.status).toBe(201);
const opener = (await sent.json()).data as { id: string; conversation_id: string };
// The opener has a real, current (18+ digit) id: newest of all.
expect(opener.id.length).toBeGreaterThanOrEqual(18);
const [conv] = await stack.runtime.deps.db
.select({ channelId: dmConversations.channelId })
.from(dmConversations)
.where(eq(dmConversations.id, opener.conversation_id));
await seed(ws.workspaceId, conv.channelId, alice.agentId);

await expectNumericPaging(alice.token, `/v1/dm/${opener.conversation_id}/messages`, [opener.id]);
await expectNumericPaging(
ws.workspaceKey,
`/v1/dm/conversations/${opener.conversation_id}/messages`,
[opener.id],
);
});
});
7 changes: 7 additions & 0 deletions packages/engine/src/adapters/actor/migrations.generated.ts
Original file line number Diff line number Diff line change
Expand Up @@ -880,6 +880,13 @@ export const ACTOR_MIGRATIONS: readonly (readonly [string, readonly string[]])[]
[
"ALTER TABLE workspaces\n ADD COLUMN leaders text"
]
],
[
"0087_messages_channel_length_id_index.sql",
[
"CREATE INDEX IF NOT EXISTS idx_messages_channel_length_id\n ON messages(channel_id, length(id), id)",
"DROP INDEX IF EXISTS idx_messages_channel_time"
]
]
];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ describe("loadBundledPostgresMigrations", () => {
"0005_delivery_confirmation.sql",
"0006_action_invocation_requester.sql",
"0007_workspace_leaders.sql",
"0008_messages_channel_length_id.sql",
]);
expect(bundled.every((m) => m.statements.length > 0)).toBe(true);
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ it('seeds DM read cursors from the newest read receipt, ordered numerically', ()
INSERT INTO read_receipts(message_id,agent_id) VALUES
('${older}','bob'),('${newer}','bob'),('2${'0'.repeat(17)}','alice');`);

expect(runMigrations(handle).applied).toEqual(['0078_dm_participant_read_cursor.sql', '0079_credential_upgrades.sql', '0080_nodes_context_reachable_index.sql', '0081_delivery_confirmation.sql', '0082_pending_events_settled_retention.sql', '0083_strict_spawn_placement_token.sql', '0084_strict_spawn_dispatch_generation.sql', '0085_action_invocation_requester.sql', '0086_workspace_leaders.sql']);
expect(runMigrations(handle).applied).toEqual(['0078_dm_participant_read_cursor.sql', '0079_credential_upgrades.sql', '0080_nodes_context_reachable_index.sql', '0081_delivery_confirmation.sql', '0082_pending_events_settled_retention.sql', '0083_strict_spawn_placement_token.sql', '0084_strict_spawn_dispatch_generation.sql', '0085_action_invocation_requester.sql', '0086_workspace_leaders.sql', '0087_messages_channel_length_id_index.sql']);

const cursors = db.prepare('SELECT agent_id, last_read_id FROM dm_participants ORDER BY agent_id').all();
// A receipt in another channel does not seed a DM cursor.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ describe('registered invocation handler snapshot backfill', () => {
'0084_strict_spawn_dispatch_generation.sql',
'0085_action_invocation_requester.sql',
'0086_workspace_leaders.sql',
'0087_messages_channel_length_id_index.sql',
]);
expect(handle.sqlite.prepare(`
SELECT id, handler_agent_id, handler_node_id, spawn_dispatch_generation
Expand Down
1 change: 1 addition & 0 deletions packages/engine/src/db/__tests__/taskMigration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ it('upgrades existing actions and results additively, preserving rows and every
'0084_strict_spawn_dispatch_generation.sql',
'0085_action_invocation_requester.sql',
'0086_workspace_leaders.sql',
'0087_messages_channel_length_id_index.sql',
]);
for (const table of before) {
const columns = db.pragma(`table_info(${table.name})`) as { name: string }[];
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
DROP INDEX "idx_messages_channel_time";--> statement-breakpoint
CREATE INDEX "idx_messages_channel_length_id" ON "messages" USING btree ("channel_id",length("id"),"id" COLLATE "C");
Loading
Loading