From 98ff32d02603bbc5179568bae0ce8edfb48d9749 Mon Sep 17 00:00:00 2001 From: layered-control-agent-host Date: Sun, 22 Nov 2026 06:09:06 +0000 Subject: [PATCH] fix(mcp): bound Relaycast and relaygate calls made under the hosted-MCP owner lock Hosted MCP tool calls verify, register, recover and release their relay participant inside a transaction that holds the user and organization advisory locks plus the per-owner lock. Those remote calls had no deadline, so a stalled upstream kept the locks and blocked the same user's OAuth token refresh and every other relay call that needs them. Calls made while the owner lock is held now share one 10-second budget, counted from when the locks are held. A call started after the budget is spent is refused without being sent. Calls outside the lock (sends, inbox reads, launches) are unchanged. A timed-out write is reported with the existing unknown-outcome wording, and a timed-out leave keeps the participant so a retry completes it. A Relaycast answer whose body is lost mid-read is now an unknown outcome instead of an unlabelled error. Co-Authored-By: Claude --- CHANGELOG.md | 1 + src/mcp/owner-lock-deadline.ts | 37 +++++++++ src/mcp/relay-participants.ts | 9 ++- src/mcp/routes.ts | 7 +- src/messaging/relaycast.ts | 8 +- test/mcp-oauth-admission.test.ts | 128 ++++++++++++++++++++++++++++++- 6 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 src/mcp/owner-lock-deadline.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d111695..d73c1f3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ assigned at release. ## Unreleased +- Fixed: hosted MCP Relaycast and relaygate calls made while the per-owner lock transaction is open (participant verification, registration, token recovery, and the releases when leaving or linking a Desktop identity) now share a 10-second budget per lock, counted from when the locks are held; a call started after the budget is spent is refused without being sent. A stalled upstream no longer holds the owner's locks, which blocked that user's OAuth token refresh and other relay calls. A timed-out write is reported with the existing wording: unknown outcome for joins and sends, and a retryable failure for `leave_relay` (the participant is kept and a retry completes the leave). Sends, inbox reads and launches run after the lock and are unchanged. A Relaycast answer whose body is lost mid-read is now reported as an unknown outcome instead of an unlabelled error. No migration. - Added: optional `SELFHOST_ROUTED_SITE_URL` (an https origin) for a deployment reached through an edge router on another site origin. When a request carries exactly one `x-original-host` header naming that origin's host, the Desktop/CLI sign-in answers (browser handoff `api_url`, desktop session, workspace switch, device start `verification_uri`, device and enrollment token `api_url`, and refresh `apiUrl`) name it instead of the Host-selected origin, so a Desktop that signed in through that site accepts them. The header only selects the configured value and is never echoed; emailed links, OIDC redirect URIs, the MCP endpoint URL, cookies and browser origin checks are unchanged. The router must overwrite `x-original-host`. Unset: unchanged. No migration. - Added: a session bridge for organizations moved from Agent Relay Cloud (migration 0083; docs/identity.md). Off by default (`SELFHOST_CLOUD_BRIDGE_ENABLED`); while off its routes answer 404. Cloud signs a short-lived, single-use Ed25519 assertion for a browser with a live Cloud session and posts it to `/cloud/api/v1/auth/cloud-bridge/consume`; this deployment verifies it against `SELFHOST_CLOUD_BRIDGE_PUBLIC_KEYS` and issues its own browser session for the existing user (matched by id, never email) in an organization the operator listed with a time-boxed window (`/api/v1/installation/cloud-bridge/organizations`, bootstrap bearer only). It creates nothing, accepts no Cloud token, and is refused after the person signs out here. - Fixed: MCP repository-filtered session summaries apply the first 200 characters of a repo-only filter in the session directory before pagination, while still accepting a 512-character repo value and verifying the full returned repository URL. This requires a sessions read API whose session directory list applies `projectContains` as a case-insensitive substring of project ID, catalog project key or recorded repository URL (directory migration `20261105034000` or later). Exact verification can still produce a short or empty page with a cursor. Relay agent listings now link an exact roster agent ID to its already-synced Desktop native session when the agent has no validated Relaycast session declaration. The optional session join is failure-open: the roster still returns a null session ID and attribution with `session_unavailable_reason` set to `enrichment_failed` or `ambiguous_sync`; an independently recorded metadata session kind is preserved. No migration. diff --git a/src/mcp/owner-lock-deadline.ts b/src/mcp/owner-lock-deadline.ts new file mode 100644 index 00000000..412af528 --- /dev/null +++ b/src/mcp/owner-lock-deadline.ts @@ -0,0 +1,37 @@ +import { AsyncLocalStorage } from "node:async_hooks"; + +/** Total time Relaycast and relaygate calls may take while one hosted-MCP owner lock is held. */ +export const OWNER_LOCK_RELAY_BUDGET_MS = 10_000; + +const held = new AsyncLocalStorage<{ since: number }>(); + +/** + * Runs `work` as holding the owner lock: ownerLockFetch bounds its remote + * calls to one shared budget counted from now. Call it only once the locks are + * held, so time spent waiting for them does not count. A nested call keeps the + * outer budget. + */ +export function underOwnerLock(work: () => Promise): Promise { + return held.getStore() ? work() : held.run({ since: performance.now() }, work); +} + +/** + * The hosted-MCP clients' fetch. A call made under the owner lock gets what is + * left of the lock's budget as a deadline, composed with the caller's own + * signal; once the budget is spent it is rejected without being sent. Every + * other call (sends, inbox reads, launches) passes through unchanged. + */ +export function ownerLockFetch(upstream: typeof fetch = fetch, timeoutMs = OWNER_LOCK_RELAY_BUDGET_MS): typeof fetch { + return (input, init) => { + const lock = held.getStore(); + if (!lock) return upstream(input, init); + const remaining = timeoutMs - (performance.now() - lock.since); + if (remaining <= 0) { + return Promise.reject(new DOMException("The owner lock's relay time budget is spent.", "TimeoutError")); + } + const prior = init?.signal ?? (input instanceof Request ? input.signal : null); + // AbortSignal.timeout takes whole milliseconds only. + const timeout = AbortSignal.timeout(Math.ceil(remaining)); + return upstream(input, { ...init, signal: prior ? AbortSignal.any([prior, timeout]) : timeout }); + }; +} diff --git a/src/mcp/relay-participants.ts b/src/mcp/relay-participants.ts index 81528244..1ff0732f 100644 --- a/src/mcp/relay-participants.ts +++ b/src/mcp/relay-participants.ts @@ -23,6 +23,7 @@ import { } from "./cursor-oidc.js"; import { postgresAttestationStore } from "./copilot-attestation.js"; import { enrolHostedAtGate, freshAtGate, HostedGateRefused, type GateEnrolment } from "./hosted-gate.js"; +import { underOwnerLock } from "./owner-lock-deadline.js"; import type { RelayHostedClient } from "./relay-hosted-credentials.js"; /** @@ -448,13 +449,17 @@ function createRelayService(context: McpContext) { } } - /** Serializes join and leave for this owner in Postgres; every store access inside uses `tx`. */ + /** + * Serializes join and leave for this owner in Postgres; every store access inside uses `tx`. + * Once the locks are held, Relaycast and relaygate calls inside share ownerLockFetch's budget, + * so a stalled upstream cannot hold the locks; waiting for the locks does not count. + */ function withOwnerLock(run: (tx: SqlExecutor) => Promise): Promise { return database.transaction(async (tx) => { if (oauth) await oauth.lock(tx); await tx.query("SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))", ["selfhost-hosted-mcp", `${workspaceId}:${userId}`]); - return run(tx); + return underOwnerLock(() => run(tx)); }); } diff --git a/src/mcp/routes.ts b/src/mcp/routes.ts index ae092ad8..2f2aa230 100644 --- a/src/mcp/routes.ts +++ b/src/mcp/routes.ts @@ -34,6 +34,7 @@ import type { McpOAuthSessions } from "./oauth/sessions.js"; import { MCP_RELAY_SCOPE, type McpOAuthResourcePath } from "./oauth/protocol.js"; import { sendCustomerSupport } from "./customer-support.js"; import { customerSupportDesktop } from "./customer-support-notice.js"; +import { ownerLockFetch } from "./owner-lock-deadline.js"; import { controlSessionToken } from "../auth/session-cookie.js"; /** @@ -169,7 +170,9 @@ export function createMcpHandler(input: { const desktopCredentials = new DesktopMcpCredentials(store.database); const staffReadGrants = new StaffReadGrants(store.database, store.tenancy); const attestations = postgresAttestationStore(store.database); - const relaycast = createRelaycastClient({ internalOrigin: config.relaycastUrl, fetchImpl: input.fetchImpl }); + // Calls made under the hosted-MCP owner lock share one time budget; all others are unchanged. + const relayFetch = ownerLockFetch(input.fetchImpl); + const relaycast = createRelaycastClient({ internalOrigin: config.relaycastUrl, fetchImpl: relayFetch }); function reportUnexpected(error: unknown, request: IncomingMessage, response: ServerResponse, path: string): void { let errorPath = path; try { errorPath = new URL(request.url ?? path, config.publicBaseUrl).pathname; } catch { /* Use the matched route. */ } @@ -180,7 +183,7 @@ export function createMcpHandler(input: { } // Gate-only installs only: desktop identities hold rg_ tokens that only relaygate accepts. const gateClient = config.relaygatePublicUrl - ? createRelaycastClient({ internalOrigin: config.relaygatePublicUrl, fetchImpl: input.fetchImpl }) + ? createRelaycastClient({ internalOrigin: config.relaygatePublicUrl, fetchImpl: relayFetch }) : undefined; // Gate-only installs only: relaygate checks every agent launch before it is reserved or sent. const launchCheck = config.relaygatePublicUrl diff --git a/src/messaging/relaycast.ts b/src/messaging/relaycast.ts index 367689eb..a615d288 100644 --- a/src/messaging/relaycast.ts +++ b/src/messaging/relaycast.ts @@ -224,7 +224,13 @@ export function createRelaycastClient(input: { } catch { throw new RelaycastCallError(0, "", true); } - const raw = await response.text(); + let raw: string; + try { + raw = await response.text(); + } catch { + // A body lost after the status (an abort mid-read) leaves the outcome unknown under the same rule as below. + throw new RelaycastCallError(response.status, "", response.ok || response.status >= 500); + } if (allowNotFound && response.status === 404) return null; if (!response.ok) { const unknown = response.status === 0 || response.status >= 500; diff --git a/test/mcp-oauth-admission.test.ts b/test/mcp-oauth-admission.test.ts index a110aca6..c284c0d6 100644 --- a/test/mcp-oauth-admission.test.ts +++ b/test/mcp-oauth-admission.test.ts @@ -10,7 +10,8 @@ import { encryptSecret } from "../src/crypto.js"; import { createRelaycastClient } from "../src/messaging/relaycast.js"; import { historyReader, type McpContext } from "../src/mcp/context.js"; import { desktopCallContext } from "../src/mcp/desktop-call.js"; -import { HOSTED_SOURCE, ownerHash, relayService } from "../src/mcp/relay-participants.js"; +import { HOSTED_SOURCE, ownerHash, relayFailure, relayService } from "../src/mcp/relay-participants.js"; +import { ownerLockFetch } from "../src/mcp/owner-lock-deadline.js"; import { createRelaycastWorld } from "./support/relaycast-world.js"; import { createServer } from "node:http"; import { createApp } from "../src/app.js"; @@ -23,7 +24,7 @@ import { pkceS256 } from "../src/mcp/oauth/protocol.js"; import { McpOAuthStore } from "../src/mcp/oauth/store.js"; import { MCP_OAUTH_TEARDOWN_DECRYPT_MAX_ATTEMPTS,McpOAuthTeardown,mcpOAuthTeardownBackoffMs,mcpOAuthTeardownFetch } from "../src/mcp/oauth/teardown.js"; import { lockOrganization,lockUser } from "../src/tenancy.js"; -import type { RelaycastClient } from "../src/messaging/relaycast.js"; +import { RelaycastCallError, type RelaycastClient } from "../src/messaging/relaycast.js"; import { relaySpawnService, relaySpawnDeps, spawnStore, type SpawnOwner } from "../src/mcp/relay-spawn.js"; import { NO_FEATURES } from "../src/features.js"; @@ -652,6 +653,129 @@ test("teardown commits its claim and holds no lifecycle lock while Relaycast ans assert.equal(f.world.agents.has("approved-agent"),false); }); +/** + * Records each Relaycast call; a matching call answers only when its signal aborts. + * With `applied`, the stalled request takes effect upstream first and only its answer is lost. + */ +function stalling(f:Awaited>,stalls:(method:string,path:string)=>boolean=()=>false,applied=false) { + const seen:{method:string;path:string;signal:boolean}[]=[]; + let entered!:()=>void; + const reached=new Promise(resolve=>{entered=resolve;}); + const fetchImpl:typeof fetch=async(input,init)=>{ + const method=init?.method??"GET",path=new URL(String(input)).pathname; + seen.push({method,path,signal:Boolean(init?.signal)}); + if(!stalls(method,path)) return f.fetchImpl(input,init); + if(applied) await f.fetchImpl(input,init); + entered(); + return new Promise((_resolve,reject)=>{ + // Later than within(): an unbounded call fails the test instead of answering. + const network=setTimeout(()=>reject(new Error("missing abort")),5000); + const signal=init?.signal; + const aborted=()=>{clearTimeout(network);reject(signal!.reason);}; + if(signal?.aborted)aborted(); + else signal?.addEventListener("abort",aborted,{once:true}); + }); + }; + return {fetchImpl,seen,reached}; +} +const ownerLockBounded=(f:Awaited>,fetchImpl:typeof fetch,timeoutMs:number)=>relayService({...f.context, + relay:{...f.context.relay,client:createRelaycastClient({internalOrigin:"http://fake-relay.test",fetchImpl:ownerLockFetch(fetchImpl,timeoutMs)})}}); + +test("a stalled Relaycast write under the OAuth owner lock ends at the deadline as an unknown outcome and frees the lock",async t=>{ + const f=await relayFixture(t);f.world.addAgent("peer"); + const {fetchImpl,seen,reached}=stalling(f,(method,path)=>method==="POST" && path==="/v1/agents"); + const service=ownerLockBounded(f,fetchImpl,200); + // First use registers the approved agent under the owner lock. + const sending=service.send({to:"peer",text:"never registered",idempotency_key:"fake-deadline-send"}).then(()=>null,(error:unknown)=>error); + await within(reached); + // The same user's refresh needs the user and organization locks the stalled transaction holds. + const refresh=f.oauth.refreshMcpOAuthToken({refreshToken:f.pair.refreshToken,clientId:f.clientId,resource:undefined}); + const error=await within(sending); + assert.ok(error instanceof RelaycastCallError); + assert.equal(error.outcomeUnknown,true); + assert.equal(error.operation,"register_hosted"); + const text=relayFailure(error,"send").content[0]!.text; + assert.match(text,/If a send was attempted, its outcome may be unknown/); + assert.match(text,/operation=register_hosted status=none/); + assert.equal((await within(refresh)).ok,true); + assert.equal(seen.find(call=>call.method==="POST" && call.path==="/v1/agents")?.signal,true); + assert.equal(f.world.messages.size,0); + assert.equal((await f.database.query("SELECT id FROM hosted_relay_participants")).rows.length,0); +}); + +test("owner-lock calls that answer in time are unaffected; sends carry no deadline",async t=>{ + const f=await relayFixture(t);f.world.addAgent("peer"); + const {fetchImpl,seen}=stalling(f); + const service=ownerLockBounded(f,fetchImpl,1000); + await service.send({to:"peer",text:"answered in time",idempotency_key:"fake-in-time"}); + assert.equal(f.world.messages.size,1); + assert.equal(f.connection.access.agentRemoteId,f.world.agents.get("approved-agent")!.id); + assert.equal(seen.find(call=>call.method==="POST" && call.path==="/v1/agents")?.signal,true); + // The send itself runs after the lock is released. + assert.equal(seen.find(call=>call.method==="POST" && call.path==="/v1/dm")?.signal,false); + assert.deepEqual(await service.leave(),{joined:false,already_left:false}); + // leave releases under the owner lock, so the release is bounded too. + assert.equal(seen.find(call=>call.path==="/v1/agents/release-exact")?.signal,true); + assert.equal(f.world.agents.has("approved-agent"),false); +}); + +test("a call made after the owner lock's budget is spent is refused without being sent",async t=>{ + const f=await relayFixture(t);f.world.addAgent("peer"); + const {fetchImpl,seen}=stalling(f); + const service=ownerLockBounded(f,fetchImpl,0); + const error=await within(service.send({to:"peer",text:"never sent",idempotency_key:"fake-spent"}).then(()=>null,(caught:unknown)=>caught)); + assert.ok(error instanceof RelaycastCallError); + assert.equal(error.outcomeUnknown,true); + // The registration under the lock never reached the upstream. + assert.equal(seen.some(call=>call.method==="POST" && call.path==="/v1/agents"),false); + assert.equal(f.world.agents.has("approved-agent"),false); + assert.equal(f.world.messages.size,0); + assert.equal((await f.database.query("SELECT id FROM hosted_relay_participants")).rows.length,0); +}); + +for (const applied of [false,true]) { + test(`a stalled leave release ${applied?"that took effect upstream ":""}fails at the deadline with the leave wording and a retry leaves`,async t=>{ + const f=await relayFixture(t);f.world.addAgent("peer"); + let stall=false; + const {fetchImpl,seen}=stalling(f,(_method,path)=>stall && path==="/v1/agents/release-exact",applied); + const service=ownerLockBounded(f,fetchImpl,200); + await service.send({to:"peer",text:"joined",idempotency_key:"fake-leave-stall"}); + stall=true; + const error=await within(service.leave().then(()=>null,(caught:unknown)=>caught)); + assert.ok(error instanceof RelaycastCallError); + assert.equal(error.outcomeUnknown,true); + assert.match(relayFailure(error,"leave").content[0]!.text,/^Leaving the hosted relay participant failed\. It is safe to retry leave_relay\./); + assert.equal(seen.find(call=>call.path==="/v1/agents/release-exact")?.signal,true); + // The transaction rolled back: the participant row is kept for the retry. + assert.equal((await f.database.query("SELECT id FROM hosted_relay_participants")).rows.length,1); + assert.equal(f.world.agents.has("approved-agent"),!applied); + stall=false; + assert.deepEqual(await within(service.leave()),{joined:false,already_left:false}); + assert.equal(f.world.agents.has("approved-agent"),false); + assert.equal((await f.database.query("SELECT id FROM hosted_relay_participants")).rows.length,0); + }); +} + +test("a Relaycast answer whose body is lost mid-read is an unknown outcome",async()=>{ + const fetchImpl:typeof fetch=async(_input,init)=>{ + const signal=init!.signal!; + // Headers arrive; the body stream errors only when the caller's signal aborts. + const body=new ReadableStream({start(controller){ + signal.addEventListener("abort",()=>controller.error(signal.reason),{once:true}); + }}); + return new Response(body,{status:200,headers:{"content-type":"application/json"}}); + }; + const client=createRelaycastClient({internalOrigin:"http://fake-relay.test",fetchImpl}); + const controller=new AbortController(); + const reading=client.authenticatedAgent("at_fake",controller.signal).then(()=>null,(caught:unknown)=>caught); + setTimeout(()=>controller.abort(),10); + const error=await within(reading); + assert.ok(error instanceof RelaycastCallError); + assert.equal(error.status,200); + assert.equal(error.outcomeUnknown,true); + assert.equal(error.operation,"authenticated_agent"); +}); + test("a retryable failure backs off exponentially and retries only once due",async t=>{ assert.deepEqual([1,2,3,6,7,40].map(mcpOAuthTeardownBackoffMs),[60_000,120_000,240_000,1_920_000,3_600_000,3_600_000]); const f=await relayFixture(t);await f.service.join({});