diff --git a/CHANGELOG.md b/CHANGELOG.md index 48f71cb91..79aecb964 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Packages without a separate changelog are covered by the cross-package notes bel ### Fixed - DM and group DM participant rules now apply consistently for agent tokens across the channel, message, search and console routes, matching the `/v1/dm/…` routes. Workspace keys and observer tokens are unchanged. +- Relayfile inbound messages no longer store an oversized provider record in message metadata. A record whose JSON is over 8 KiB is stored as `record: null` with `record_omitted` (`reason`, `bytes`, `limit`); the full record stays in relayfile at `relayfile.path` and `relayfile.revision`, and the message text is unchanged. ## [12.13.0] - 2026-11-17 diff --git a/docs/repository-guide.md b/docs/repository-guide.md index 03d2883c5..13ac052ea 100644 --- a/docs/repository-guide.md +++ b/docs/repository-guide.md @@ -1663,7 +1663,7 @@ Explicit `target_node` on built-in spawn honors fleet placement even when a legacy workspace-global node action named `spawn` exists; its caller allowlist continues to apply. A dispatch acknowledgement does not establish harness readiness. -Verified spawn (`verify_ready: true`) fails with `spawn_target_unavailable` when the selected provider lacks a live handler heartbeat or connected socket. Unverified legacy requests retain queue behavior. Relayfile ingress preserves authenticated `providerEventType` and `resourceRef` as `provider_event_type` and `resource_ref` in message metadata, alongside the provider record. Generic file events do not imply PR, CI, or review semantics. Cloud sync envelopes are unwrapped to expose their provider payload in message metadata and formatting; this does not promote payload fields into authenticated event semantics. +Verified spawn (`verify_ready: true`) fails with `spawn_target_unavailable` when the selected provider lacks a live handler heartbeat or connected socket. Unverified legacy requests retain queue behavior. Relayfile ingress preserves authenticated `providerEventType` and `resourceRef` as `provider_event_type` and `resource_ref` in message metadata, alongside the provider record. A provider record whose JSON exceeds 8 KiB (UTF-8; engine config `relayfileRecordMetadataMaxBytes`) is stored as `record: null` with `record_omitted: { reason: 'too_large', bytes, limit }`; fetch it from relayfile by `relayfile.path` and `relayfile.revision`. Generic file events do not imply PR, CI, or review semantics. Cloud sync envelopes are unwrapped to expose their provider payload in message metadata and formatting; this does not promote payload fields into authenticated event semantics. Readiness requests do not reroute registered global spawn handlers. Only a nonempty explicit `target_node` selects fleet placement around a legacy node alias; handlers that receive `verify_ready` must honor its completion contract. A deleted recipient leaves its old route memberless so a replacement cannot inherit delivery. Retire or replace the inventoried producer binding and webhook when deleting a subscriber; channel history remains for audit. diff --git a/openapi.yaml b/openapi.yaml index 6abe4b081..0d02db449 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -7747,6 +7747,11 @@ paths: A deletion without a snapshot retains its authenticated event metadata and path; deleted record content is unavailable and is not reconstructed. The HMAC input is `X-Relay-Timestamp + "." + raw request body`, signed with SHA-256. + The provider record is stored in message metadata as `record` only when its + JSON is at most 8 KiB (UTF-8, deployment-configurable); a larger record is + stored as `record: null` with `record_omitted: { reason: "too_large", bytes, + limit }` and stays retrievable from relayfile by `relayfile.path` and + `relayfile.revision`. The message text is still formatted from the full record. A full recipient mailbox returns 503 mailbox_full with Retry-After: 30; the producer must retry the same event ID. The message and all delivery rows roll back together, so no recipient is silently skipped. diff --git a/packages/engine/CHANGELOG.md b/packages/engine/CHANGELOG.md index ec314558a..762754b19 100644 --- a/packages/engine/CHANGELOG.md +++ b/packages/engine/CHANGELOG.md @@ -12,6 +12,7 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ### Fixed - DM participant rules are applied consistently for agent tokens. For an agent that is not an active participant of a DM or group DM, `/v1/channels/:name` (detail, messages, members, read-status, update, topic, archive, leave, mute, unmute) and `/v1/messages/:id` (get, replies, reactions, readers) answer `404`, as does posting to the channel, including a node token posting as a non-participant `from`. `join` and `invite` are refused on DM-backed channels for every agent; add group DM members through `/v1/dm/:conversation_id/participants`. `GET /v1/search` and `GET /v1/console/messages` drop DM rows the agent does not participate in. +- The relayfile inbound bridge bounds the provider record it stores in message metadata, which is copied into the message row, webhook outbox and stream events. A record whose JSON exceeds 8 KiB (UTF-8) is stored as `record: null` plus `record_omitted: { reason: 'too_large', bytes, limit }`; other metadata, the message text and the idempotency fingerprint are unchanged. Hosts can change the limit with `relayfileRecordMetadataMaxBytes` in the engine config (`0` never stores a record; invalid values use the default). ## [12.13.0] - 2026-11-17 diff --git a/packages/engine/src/ports/index.ts b/packages/engine/src/ports/index.ts index 5ba387f9f..af8cd0f70 100644 --- a/packages/engine/src/ports/index.ts +++ b/packages/engine/src/ports/index.ts @@ -204,6 +204,14 @@ export interface EngineConfig { * to disable the relayfile inbound bridge. */ relayfileInboundSecret?: string; + /** + * Largest provider record, in UTF-8 bytes of its JSON, that the relayfile + * inbound bridge stores in message metadata. A larger record is stored as + * `record: null` with `record_omitted`; it remains retrievable from + * relayfile by path and revision. `0` never stores a record. Omitted, + * negative or non-finite values use 8 KiB. + */ + relayfileRecordMetadataMaxBytes?: number; /** * Deployment secret used to derive credentials for anonymous, keyed * workspace bootstrap creates. Hosts should persist this value so a replay diff --git a/packages/engine/src/routes/__tests__/relayfileInbound.test.ts b/packages/engine/src/routes/__tests__/relayfileInbound.test.ts index 6fd6f47f4..ccc80bff3 100644 --- a/packages/engine/src/routes/__tests__/relayfileInbound.test.ts +++ b/packages/engine/src/routes/__tests__/relayfileInbound.test.ts @@ -4,9 +4,11 @@ import { and, eq } from 'drizzle-orm'; import { createEngine } from '../../engine.js'; import { createNodeRuntime, type NodeRuntime } from '../../adapters/node/index.js'; import { createWorkspace, registerAgent } from '../../__tests__/conformance/harness.js'; -import { deliveries } from '../../db/tables.js'; +import { deliveries, messages as messageRows, pendingEvents } from '../../db/tables.js'; import type { KeyValueStore } from '../../ports/kv.js'; import { + boundRelayfileRecord, + DEFAULT_RELAYFILE_RECORD_METADATA_MAX_BYTES, deriveRelayfileInboundSecret, formatRelayfileEventMessage, verifyRelayfileSignature, @@ -20,7 +22,7 @@ interface Stack { const stacks: Stack[] = []; /** Create an isolated SQLite engine and register its cleanup for the current test. */ -function makeStack(opts: { kv?: KeyValueStore; depthCap?: number } = {}): Stack { +function makeStack(opts: { kv?: KeyValueStore; depthCap?: number; recordMaxBytes?: unknown } = {}): Stack { const runtime = createNodeRuntime({ dbPath: ':memory:', baseUrl: 'http://localhost:0', @@ -29,6 +31,7 @@ function makeStack(opts: { kv?: KeyValueStore; depthCap?: number } = {}): Stack environment: 'test', ...(opts.depthCap ? { mailbox: { depthCap: opts.depthCap } } : {}), relayfileInboundSecret: 'relaycast-master', + ...('recordMaxBytes' in opts ? { relayfileRecordMetadataMaxBytes: opts.recordMaxBytes as number } : {}), }, presence: { sweepIntervalMs: 0 }, eventQueue: { pollIntervalMs: 0 }, @@ -54,6 +57,42 @@ function signedHeaders(secret: string, body: string, timestamp = String(Math.flo }; } +/** UTF-8 bytes of a record's JSON, the measure the metadata cap uses. */ +function recordBytes(record: unknown): number { + return Buffer.byteLength(JSON.stringify(record), 'utf8'); +} + +/** Provision a Linear target on #general and return a poster for signed record events. */ +async function recordTarget(stack: Stack, name: string) { + const ws = await createWorkspace(stack.app, name); + const targetRes = await stack.app.request('/v1/integrations/relayfile/inbound-target', { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${ws.workspaceKey}` }, + body: JSON.stringify({ channel: 'general', provider: 'linear', path_glob: '/linear/issues/**' }), + }); + expect(targetRes.status).toBe(201); + const target = (await targetRes.json() as { data: { url: string; secret: string } }).data; + const post = (eventId: string, record: unknown) => { + const body = JSON.stringify({ + eventId, type: 'file.updated', provider: 'linear', origin: 'provider_sync', + path: `/linear/issues/${eventId}.json`, revision: `rev_${eventId}`, contentHash: `hash_${eventId}`, + snapshot: { contentType: 'application/json', content: JSON.stringify(record) }, + }); + return stack.app.request(target.url, { + method: 'POST', body, headers: { ...signedHeaders(target.secret, body), 'X-Relay-Event-Id': eventId }, + }); + }; + /** Post an event and return the metadata stored on its message row. */ + const storedMetadata = async (eventId: string, record: unknown) => { + const response = await post(eventId, record); + expect(response.status).toBe(201); + const { data } = await response.json() as { data: { message_id: string } }; + const [row] = await stack.runtime.deps.db.select({ metadata: messageRows.metadata }).from(messageRows).where(eq(messageRows.id, data.message_id)); + return row.metadata as Record; + }; + return { ws, post, storedMetadata }; +} + class FailingKeyValueStore implements KeyValueStore { async get(): Promise { throw new Error('kv unavailable'); @@ -711,4 +750,135 @@ describe('relayfile inbound bridge', () => { expect(formatRelayfileEventMessage({ type: 'file.updated', path: '/github/42.json', snapshot: { content: JSON.stringify({ title: 'Ordinary record', payload }) } }, 'github')?.record).toMatchObject({ title: 'Ordinary record', payload }); }); + describe('provider record metadata cap', () => { + /** A `{"pad":"…"}` record whose JSON is exactly `bytes` UTF-8 bytes. */ + const paddedRecord = (bytes: number) => ({ pad: 'a'.repeat(bytes - recordBytes({ pad: '' })) }); + + it('measures UTF-8 bytes of the JSON and never omits null', () => { + expect(DEFAULT_RELAYFILE_RECORD_METADATA_MAX_BYTES).toBe(8192); + const multiByte = { body: 'é'.repeat(100) }; + const bytes = recordBytes(multiByte); + expect(JSON.stringify(multiByte).length).toBeLessThan(bytes); + expect(boundRelayfileRecord(multiByte, bytes)).toEqual({ record: multiByte }); + expect(boundRelayfileRecord(multiByte, JSON.stringify(multiByte).length)) + .toEqual({ record: null, record_omitted: { reason: 'too_large', bytes, limit: JSON.stringify(multiByte).length } }); + expect(boundRelayfileRecord('abc', 5)).toEqual({ record: 'abc' }); + expect(boundRelayfileRecord('abcd', 5)).toEqual({ record: null, record_omitted: { reason: 'too_large', bytes: 6, limit: 5 } }); + expect(boundRelayfileRecord(null, 0)).toEqual({ record: null }); + expect(boundRelayfileRecord({}, 0)).toEqual({ record: null, record_omitted: { reason: 'too_large', bytes: 2, limit: 0 } }); + }); + + it('stores records under and at the default limit unchanged and omits one byte over', async () => { + const stack = makeStack(); + const { storedMetadata } = await recordTarget(stack, 'relayfile-record-boundary'); + const limit = DEFAULT_RELAYFILE_RECORD_METADATA_MAX_BYTES; + for (const [eventId, bytes] of [['under', limit - 1], ['at', limit]] as const) { + const record = paddedRecord(bytes); + expect(recordBytes(record)).toBe(bytes); + const metadata = await storedMetadata(eventId, record); + expect(metadata.record).toEqual(record); + expect(metadata).not.toHaveProperty('record_omitted'); + } + const over = paddedRecord(limit + 1); + expect(await storedMetadata('over', over)).toMatchObject({ + record: null, record_omitted: { reason: 'too_large', bytes: limit + 1, limit }, + }); + }); + + it('counts multi-byte characters as bytes at the configured limit', async () => { + const record = { title: 'Unicode', body: 'é'.repeat(200) }; + const bytes = recordBytes(record); + const chars = JSON.stringify(record).length; + expect(chars).toBeLessThan(bytes); + const atLimit = await recordTarget(makeStack({ recordMaxBytes: bytes }), 'relayfile-record-multibyte-at'); + expect((await atLimit.storedMetadata('mb_at', record)).record).toEqual(record); + // A limit at the character count would admit the record if characters were measured. + const charLimit = await recordTarget(makeStack({ recordMaxBytes: chars }), 'relayfile-record-multibyte-chars'); + expect(await charLimit.storedMetadata('mb_chars', record)).toMatchObject({ + record: null, record_omitted: { reason: 'too_large', bytes, limit: chars }, + }); + }); + + it('stores the bounded form on the message, outbox event and listed message while text uses the full record', async () => { + const stack = makeStack(); + const { ws, post } = await recordTarget(stack, 'relayfile-record-over'); + const subscription = await stack.app.request('/v1/subscriptions', { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${ws.workspaceKey}` }, + body: JSON.stringify({ events: ['message.created'], url: 'http://127.0.0.1:1/hook' }), + }); + expect(subscription.status).toBe(201); + const record = { title: 'T'.repeat(400), description: 'd'.repeat(20_000), user: { name: 'Grace' } }; + const bytes = recordBytes(record); + const bounded = { record: null, record_omitted: { reason: 'too_large', bytes, limit: DEFAULT_RELAYFILE_RECORD_METADATA_MAX_BYTES } }; + const expectedMetadata = { + ...bounded, + provider: 'linear', path: '/linear/issues/big.json', event: 'file.updated', + relayfile: { path: '/linear/issues/big.json', revision: 'rev_big', contentHash: 'hash_big', eventId: 'big', provider: 'linear' }, + }; + + const response = await post('big', record); + expect(response.status).toBe(201); + const { data: accepted } = await response.json() as { data: { replayed: boolean; message_id: string } }; + expect(accepted.replayed).toBe(false); + + const [row] = await stack.runtime.deps.db.select().from(messageRows).where(eq(messageRows.id, accepted.message_id)); + expect(row.metadata).toMatchObject(expectedMetadata); + expect(row.body).toBe(`Linear update\n${'T'.repeat(300)}...\n${'d'.repeat(1200)}...\nRelayfile path: /linear/issues/big.json`); + expect(JSON.stringify(row.metadata)).not.toContain('d'.repeat(100)); + + const outbox = (await stack.runtime.deps.db.select().from(pendingEvents)) + .map((event) => event.payload as { id?: string; from_name?: string; metadata?: Record }) + .filter((payload) => payload.id === accepted.message_id); + expect(outbox).toHaveLength(1); + expect(outbox[0]).toMatchObject({ from_name: 'Grace', metadata: expectedMetadata }); + + const list = await stack.app.request('/v1/channels/general/messages', { headers: { authorization: `Bearer ${ws.workspaceKey}` } }); + const { data: listed } = await list.json() as { data: Array<{ id: string; metadata: unknown }> }; + expect(listed.find((message) => message.id === accepted.message_id)?.metadata).toMatchObject(bounded); + }); + + it('replays an over-limit event with the same id', async () => { + const stack = makeStack(); + const { post } = await recordTarget(stack, 'relayfile-record-replay'); + const record = paddedRecord(DEFAULT_RELAYFILE_RECORD_METADATA_MAX_BYTES * 4); + const first = await post('replayed', record); + expect(first.status).toBe(201); + const { data: accepted } = await first.json() as { data: { message_id: string } }; + const replay = await post('replayed', record); + expect(replay.status).toBe(201); + expect(await replay.json()).toMatchObject({ data: { replayed: true, message_id: accepted.message_id } }); + expect(await stack.runtime.deps.db.select().from(messageRows).where(eq(messageRows.id, accepted.message_id))).toHaveLength(1); + }); + + it('honours a smaller configured limit', async () => { + const { storedMetadata } = await recordTarget(makeStack({ recordMaxBytes: 100 }), 'relayfile-record-small-limit'); + const small = paddedRecord(100); + expect((await storedMetadata('small_at', small)).record).toEqual(small); + expect(await storedMetadata('small_over', paddedRecord(101))).toMatchObject({ + record: null, record_omitted: { reason: 'too_large', bytes: 101, limit: 100 }, + }); + }); + + it('never stores a non-null record with a zero limit', async () => { + const { storedMetadata } = await recordTarget(makeStack({ recordMaxBytes: 0 }), 'relayfile-record-zero-limit'); + const record = { title: 'tiny' }; + expect(await storedMetadata('zero', record)).toMatchObject({ + record: null, record_omitted: { reason: 'too_large', bytes: recordBytes(record), limit: 0 }, + }); + const nullRecord = await storedMetadata('zero_null', null); + expect(nullRecord.record).toBeNull(); + expect(nullRecord).not.toHaveProperty('record_omitted'); + }); + + it.each([-1, Number.NaN, Number.POSITIVE_INFINITY, '4096'])('falls back to the default limit for invalid config %s', async (recordMaxBytes) => { + const { storedMetadata } = await recordTarget(makeStack({ recordMaxBytes }), `relayfile-record-invalid-${String(recordMaxBytes)}`); + const limit = DEFAULT_RELAYFILE_RECORD_METADATA_MAX_BYTES; + const atDefault = paddedRecord(limit); + expect((await storedMetadata('invalid_at', atDefault)).record).toEqual(atDefault); + expect(await storedMetadata('invalid_over', paddedRecord(limit + 1))).toMatchObject({ + record: null, record_omitted: { reason: 'too_large', bytes: limit + 1, limit }, + }); + }); + }); }); diff --git a/packages/engine/src/routes/relayfileInbound.ts b/packages/engine/src/routes/relayfileInbound.ts index 407f3456b..cf11ab021 100644 --- a/packages/engine/src/routes/relayfileInbound.ts +++ b/packages/engine/src/routes/relayfileInbound.ts @@ -26,6 +26,8 @@ const MAX_SKEW_MS = 5 * 60 * 1000; const MAX_RELAYFILE_BODY_BYTES = 1024 * 1024; const SECRET_LABEL = 'relayfile-inbound:v1'; const IDEMPOTENCY_TTL_SECONDS = 60 * 60 * 24 * 30; +/** Default cap on the provider record stored in relayfile message metadata. */ +export const DEFAULT_RELAYFILE_RECORD_METADATA_MAX_BYTES = 8 * 1024; const createTargetSchema = z.object({ channel: z.string().min(1), @@ -221,6 +223,7 @@ async (c) => { } const mailbox = resolveMailboxConfig(c.get('engine').config, workspaceId); + const recordLimit = relayfileRecordMetadataMaxBytes(c.get('engine').config?.relayfileRecordMetadataMaxBytes); try { const result = await runIdempotent({ workspaceId, @@ -253,7 +256,7 @@ async (c) => { event: event.type, ...(event.providerEventType ? { provider_event_type: event.providerEventType } : {}), ...(event.resourceRef ? { resource_ref: event.resourceRef } : {}), - record: message.record, + ...boundRelayfileRecord(message.record, recordLimit), }, }, { mailbox, workspaceDeliveryPolicy: await resolveWorkspaceDeliveryPolicyById(c.get('db'), c.get('engine').config, workspaceId) }), }); @@ -427,6 +430,36 @@ export function formatRelayfileEventMessage(event: RelayfileEventPublic, provide return { text: lines.filter(Boolean).join('\n'), author, record }; } +export interface RelayfileRecordOmitted { + reason: 'too_large'; + bytes: number; + limit: number; +} + +/** + * Bound the provider record stored in message metadata. The metadata is copied + * into the message row, the webhook outbox and stream events, so an unbounded + * record drives storage growth. A record over `limit` UTF-8 bytes of JSON is + * replaced by `null` plus `record_omitted`; the full record stays retrievable + * from relayfile by path and revision. `null` is never omitted. + */ +export function boundRelayfileRecord( + record: unknown, + limit: number, +): { record: unknown; record_omitted?: RelayfileRecordOmitted } { + if (record === null || record === undefined) return { record }; + const bytes = new TextEncoder().encode(JSON.stringify(record)).byteLength; + if (bytes <= limit) return { record }; + return { record: null, record_omitted: { reason: 'too_large', bytes, limit } }; +} + +/** Configured record cap; anything but a finite non-negative number uses the default. */ +function relayfileRecordMetadataMaxBytes(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 + ? Math.floor(value) + : DEFAULT_RELAYFILE_RECORD_METADATA_MAX_BYTES; +} + function parseSnapshotRecord(snapshot: RelayfileSnapshot | undefined): unknown { if (!snapshot?.content) return null; if (snapshot.encoding === 'base64') return null;