diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d111695..d73c1f3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ assigned at release. ## Unreleased +- Fixed: hosted MCP Relaycast and relaygate calls made while the per-owner lock transaction is open (participant verification, registration, token recovery, and the releases when leaving or linking a Desktop identity) now share a 10-second budget per lock, counted from when the locks are held; a call started after the budget is spent is refused without being sent. A stalled upstream no longer holds the owner's locks, which blocked that user's OAuth token refresh and other relay calls. A timed-out write is reported with the existing wording: unknown outcome for joins and sends, and a retryable failure for `leave_relay` (the participant is kept and a retry completes the leave). Sends, inbox reads and launches run after the lock and are unchanged. A Relaycast answer whose body is lost mid-read is now reported as an unknown outcome instead of an unlabelled error. No migration. - Added: optional `SELFHOST_ROUTED_SITE_URL` (an https origin) for a deployment reached through an edge router on another site origin. When a request carries exactly one `x-original-host` header naming that origin's host, the Desktop/CLI sign-in answers (browser handoff `api_url`, desktop session, workspace switch, device start `verification_uri`, device and enrollment token `api_url`, and refresh `apiUrl`) name it instead of the Host-selected origin, so a Desktop that signed in through that site accepts them. The header only selects the configured value and is never echoed; emailed links, OIDC redirect URIs, the MCP endpoint URL, cookies and browser origin checks are unchanged. The router must overwrite `x-original-host`. Unset: unchanged. No migration. - Added: a session bridge for organizations moved from Agent Relay Cloud (migration 0083; docs/identity.md). Off by default (`SELFHOST_CLOUD_BRIDGE_ENABLED`); while off its routes answer 404. Cloud signs a short-lived, single-use Ed25519 assertion for a browser with a live Cloud session and posts it to `/cloud/api/v1/auth/cloud-bridge/consume`; this deployment verifies it against `SELFHOST_CLOUD_BRIDGE_PUBLIC_KEYS` and issues its own browser session for the existing user (matched by id, never email) in an organization the operator listed with a time-boxed window (`/api/v1/installation/cloud-bridge/organizations`, bootstrap bearer only). It creates nothing, accepts no Cloud token, and is refused after the person signs out here. - Fixed: MCP repository-filtered session summaries apply the first 200 characters of a repo-only filter in the session directory before pagination, while still accepting a 512-character repo value and verifying the full returned repository URL. This requires a sessions read API whose session directory list applies `projectContains` as a case-insensitive substring of project ID, catalog project key or recorded repository URL (directory migration `20261105034000` or later). Exact verification can still produce a short or empty page with a cursor. Relay agent listings now link an exact roster agent ID to its already-synced Desktop native session when the agent has no validated Relaycast session declaration. The optional session join is failure-open: the roster still returns a null session ID and attribution with `session_unavailable_reason` set to `enrichment_failed` or `ambiguous_sync`; an independently recorded metadata session kind is preserved. No migration. diff --git a/src/mcp/owner-lock-deadline.ts b/src/mcp/owner-lock-deadline.ts new file mode 100644 index 00000000..412af528 --- /dev/null +++ b/src/mcp/owner-lock-deadline.ts @@ -0,0 +1,37 @@ +import { AsyncLocalStorage } from "node:async_hooks"; + +/** Total time Relaycast and relaygate calls may take while one hosted-MCP owner lock is held. */ +export const OWNER_LOCK_RELAY_BUDGET_MS = 10_000; + +const held = new AsyncLocalStorage<{ since: number }>(); + +/** + * Runs `work` as holding the owner lock: ownerLockFetch bounds its remote + * calls to one shared budget counted from now. Call it only once the locks are + * held, so time spent waiting for them does not count. A nested call keeps the + * outer budget. + */ +export function underOwnerLock(work: () => Promise): Promise { + return held.getStore() ? work() : held.run({ since: performance.now() }, work); +} + +/** + * The hosted-MCP clients' fetch. A call made under the owner lock gets what is + * left of the lock's budget as a deadline, composed with the caller's own + * signal; once the budget is spent it is rejected without being sent. Every + * other call (sends, inbox reads, launches) passes through unchanged. + */ +export function ownerLockFetch(upstream: typeof fetch = fetch, timeoutMs = OWNER_LOCK_RELAY_BUDGET_MS): typeof fetch { + return (input, init) => { + const lock = held.getStore(); + if (!lock) return upstream(input, init); + const remaining = timeoutMs - (performance.now() - lock.since); + if (remaining <= 0) { + return Promise.reject(new DOMException("The owner lock's relay time budget is spent.", "TimeoutError")); + } + const prior = init?.signal ?? (input instanceof Request ? input.signal : null); + // AbortSignal.timeout takes whole milliseconds only. + const timeout = AbortSignal.timeout(Math.ceil(remaining)); + return upstream(input, { ...init, signal: prior ? AbortSignal.any([prior, timeout]) : timeout }); + }; +} diff --git a/src/mcp/relay-participants.ts b/src/mcp/relay-participants.ts index 81528244..1ff0732f 100644 --- a/src/mcp/relay-participants.ts +++ b/src/mcp/relay-participants.ts @@ -23,6 +23,7 @@ import { } from "./cursor-oidc.js"; import { postgresAttestationStore } from "./copilot-attestation.js"; import { enrolHostedAtGate, freshAtGate, HostedGateRefused, type GateEnrolment } from "./hosted-gate.js"; +import { underOwnerLock } from "./owner-lock-deadline.js"; import type { RelayHostedClient } from "./relay-hosted-credentials.js"; /** @@ -448,13 +449,17 @@ function createRelayService(context: McpContext) { } } - /** Serializes join and leave for this owner in Postgres; every store access inside uses `tx`. */ + /** + * Serializes join and leave for this owner in Postgres; every store access inside uses `tx`. + * Once the locks are held, Relaycast and relaygate calls inside share ownerLockFetch's budget, + * so a stalled upstream cannot hold the locks; waiting for the locks does not count. + */ function withOwnerLock(run: (tx: SqlExecutor) => Promise): Promise { return database.transaction(async (tx) => { if (oauth) await oauth.lock(tx); await tx.query("SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))", ["selfhost-hosted-mcp", `${workspaceId}:${userId}`]); - return run(tx); + return underOwnerLock(() => run(tx)); }); } diff --git a/src/mcp/routes.ts b/src/mcp/routes.ts index ae092ad8..2f2aa230 100644 --- a/src/mcp/routes.ts +++ b/src/mcp/routes.ts @@ -34,6 +34,7 @@ import type { McpOAuthSessions } from "./oauth/sessions.js"; import { MCP_RELAY_SCOPE, type McpOAuthResourcePath } from "./oauth/protocol.js"; import { sendCustomerSupport } from "./customer-support.js"; import { customerSupportDesktop } from "./customer-support-notice.js"; +import { ownerLockFetch } from "./owner-lock-deadline.js"; import { controlSessionToken } from "../auth/session-cookie.js"; /** @@ -169,7 +170,9 @@ export function createMcpHandler(input: { const desktopCredentials = new DesktopMcpCredentials(store.database); const staffReadGrants = new StaffReadGrants(store.database, store.tenancy); const attestations = postgresAttestationStore(store.database); - const relaycast = createRelaycastClient({ internalOrigin: config.relaycastUrl, fetchImpl: input.fetchImpl }); + // Calls made under the hosted-MCP owner lock share one time budget; all others are unchanged. + const relayFetch = ownerLockFetch(input.fetchImpl); + const relaycast = createRelaycastClient({ internalOrigin: config.relaycastUrl, fetchImpl: relayFetch }); function reportUnexpected(error: unknown, request: IncomingMessage, response: ServerResponse, path: string): void { let errorPath = path; try { errorPath = new URL(request.url ?? path, config.publicBaseUrl).pathname; } catch { /* Use the matched route. */ } @@ -180,7 +183,7 @@ export function createMcpHandler(input: { } // Gate-only installs only: desktop identities hold rg_ tokens that only relaygate accepts. const gateClient = config.relaygatePublicUrl - ? createRelaycastClient({ internalOrigin: config.relaygatePublicUrl, fetchImpl: input.fetchImpl }) + ? createRelaycastClient({ internalOrigin: config.relaygatePublicUrl, fetchImpl: relayFetch }) : undefined; // Gate-only installs only: relaygate checks every agent launch before it is reserved or sent. const launchCheck = config.relaygatePublicUrl diff --git a/src/messaging/relaycast.ts b/src/messaging/relaycast.ts index 367689eb..a615d288 100644 --- a/src/messaging/relaycast.ts +++ b/src/messaging/relaycast.ts @@ -224,7 +224,13 @@ export function createRelaycastClient(input: { } catch { throw new RelaycastCallError(0, "", true); } - const raw = await response.text(); + let raw: string; + try { + raw = await response.text(); + } catch { + // A body lost after the status (an abort mid-read) leaves the outcome unknown under the same rule as below. + throw new RelaycastCallError(response.status, "", response.ok || response.status >= 500); + } if (allowNotFound && response.status === 404) return null; if (!response.ok) { const unknown = response.status === 0 || response.status >= 500; diff --git a/test/mcp-oauth-admission.test.ts b/test/mcp-oauth-admission.test.ts index a110aca6..c284c0d6 100644 --- a/test/mcp-oauth-admission.test.ts +++ b/test/mcp-oauth-admission.test.ts @@ -10,7 +10,8 @@ import { encryptSecret } from "../src/crypto.js"; import { createRelaycastClient } from "../src/messaging/relaycast.js"; import { historyReader, type McpContext } from "../src/mcp/context.js"; import { desktopCallContext } from "../src/mcp/desktop-call.js"; -import { HOSTED_SOURCE, ownerHash, relayService } from "../src/mcp/relay-participants.js"; +import { HOSTED_SOURCE, ownerHash, relayFailure, relayService } from "../src/mcp/relay-participants.js"; +import { ownerLockFetch } from "../src/mcp/owner-lock-deadline.js"; import { createRelaycastWorld } from "./support/relaycast-world.js"; import { createServer } from "node:http"; import { createApp } from "../src/app.js"; @@ -23,7 +24,7 @@ import { pkceS256 } from "../src/mcp/oauth/protocol.js"; import { McpOAuthStore } from "../src/mcp/oauth/store.js"; import { MCP_OAUTH_TEARDOWN_DECRYPT_MAX_ATTEMPTS,McpOAuthTeardown,mcpOAuthTeardownBackoffMs,mcpOAuthTeardownFetch } from "../src/mcp/oauth/teardown.js"; import { lockOrganization,lockUser } from "../src/tenancy.js"; -import type { RelaycastClient } from "../src/messaging/relaycast.js"; +import { RelaycastCallError, type RelaycastClient } from "../src/messaging/relaycast.js"; import { relaySpawnService, relaySpawnDeps, spawnStore, type SpawnOwner } from "../src/mcp/relay-spawn.js"; import { NO_FEATURES } from "../src/features.js"; @@ -652,6 +653,129 @@ test("teardown commits its claim and holds no lifecycle lock while Relaycast ans assert.equal(f.world.agents.has("approved-agent"),false); }); +/** + * Records each Relaycast call; a matching call answers only when its signal aborts. + * With `applied`, the stalled request takes effect upstream first and only its answer is lost. + */ +function stalling(f:Awaited>,stalls:(method:string,path:string)=>boolean=()=>false,applied=false) { + const seen:{method:string;path:string;signal:boolean}[]=[]; + let entered!:()=>void; + const reached=new Promise(resolve=>{entered=resolve;}); + const fetchImpl:typeof fetch=async(input,init)=>{ + const method=init?.method??"GET",path=new URL(String(input)).pathname; + seen.push({method,path,signal:Boolean(init?.signal)}); + if(!stalls(method,path)) return f.fetchImpl(input,init); + if(applied) await f.fetchImpl(input,init); + entered(); + return new Promise((_resolve,reject)=>{ + // Later than within(): an unbounded call fails the test instead of answering. + const network=setTimeout(()=>reject(new Error("missing abort")),5000); + const signal=init?.signal; + const aborted=()=>{clearTimeout(network);reject(signal!.reason);}; + if(signal?.aborted)aborted(); + else signal?.addEventListener("abort",aborted,{once:true}); + }); + }; + return {fetchImpl,seen,reached}; +} +const ownerLockBounded=(f:Awaited>,fetchImpl:typeof fetch,timeoutMs:number)=>relayService({...f.context, + relay:{...f.context.relay,client:createRelaycastClient({internalOrigin:"http://fake-relay.test",fetchImpl:ownerLockFetch(fetchImpl,timeoutMs)})}}); + +test("a stalled Relaycast write under the OAuth owner lock ends at the deadline as an unknown outcome and frees the lock",async t=>{ + const f=await relayFixture(t);f.world.addAgent("peer"); + const {fetchImpl,seen,reached}=stalling(f,(method,path)=>method==="POST" && path==="/v1/agents"); + const service=ownerLockBounded(f,fetchImpl,200); + // First use registers the approved agent under the owner lock. + const sending=service.send({to:"peer",text:"never registered",idempotency_key:"fake-deadline-send"}).then(()=>null,(error:unknown)=>error); + await within(reached); + // The same user's refresh needs the user and organization locks the stalled transaction holds. + const refresh=f.oauth.refreshMcpOAuthToken({refreshToken:f.pair.refreshToken,clientId:f.clientId,resource:undefined}); + const error=await within(sending); + assert.ok(error instanceof RelaycastCallError); + assert.equal(error.outcomeUnknown,true); + assert.equal(error.operation,"register_hosted"); + const text=relayFailure(error,"send").content[0]!.text; + assert.match(text,/If a send was attempted, its outcome may be unknown/); + assert.match(text,/operation=register_hosted status=none/); + assert.equal((await within(refresh)).ok,true); + assert.equal(seen.find(call=>call.method==="POST" && call.path==="/v1/agents")?.signal,true); + assert.equal(f.world.messages.size,0); + assert.equal((await f.database.query("SELECT id FROM hosted_relay_participants")).rows.length,0); +}); + +test("owner-lock calls that answer in time are unaffected; sends carry no deadline",async t=>{ + const f=await relayFixture(t);f.world.addAgent("peer"); + const {fetchImpl,seen}=stalling(f); + const service=ownerLockBounded(f,fetchImpl,1000); + await service.send({to:"peer",text:"answered in time",idempotency_key:"fake-in-time"}); + assert.equal(f.world.messages.size,1); + assert.equal(f.connection.access.agentRemoteId,f.world.agents.get("approved-agent")!.id); + assert.equal(seen.find(call=>call.method==="POST" && call.path==="/v1/agents")?.signal,true); + // The send itself runs after the lock is released. + assert.equal(seen.find(call=>call.method==="POST" && call.path==="/v1/dm")?.signal,false); + assert.deepEqual(await service.leave(),{joined:false,already_left:false}); + // leave releases under the owner lock, so the release is bounded too. + assert.equal(seen.find(call=>call.path==="/v1/agents/release-exact")?.signal,true); + assert.equal(f.world.agents.has("approved-agent"),false); +}); + +test("a call made after the owner lock's budget is spent is refused without being sent",async t=>{ + const f=await relayFixture(t);f.world.addAgent("peer"); + const {fetchImpl,seen}=stalling(f); + const service=ownerLockBounded(f,fetchImpl,0); + const error=await within(service.send({to:"peer",text:"never sent",idempotency_key:"fake-spent"}).then(()=>null,(caught:unknown)=>caught)); + assert.ok(error instanceof RelaycastCallError); + assert.equal(error.outcomeUnknown,true); + // The registration under the lock never reached the upstream. + assert.equal(seen.some(call=>call.method==="POST" && call.path==="/v1/agents"),false); + assert.equal(f.world.agents.has("approved-agent"),false); + assert.equal(f.world.messages.size,0); + assert.equal((await f.database.query("SELECT id FROM hosted_relay_participants")).rows.length,0); +}); + +for (const applied of [false,true]) { + test(`a stalled leave release ${applied?"that took effect upstream ":""}fails at the deadline with the leave wording and a retry leaves`,async t=>{ + const f=await relayFixture(t);f.world.addAgent("peer"); + let stall=false; + const {fetchImpl,seen}=stalling(f,(_method,path)=>stall && path==="/v1/agents/release-exact",applied); + const service=ownerLockBounded(f,fetchImpl,200); + await service.send({to:"peer",text:"joined",idempotency_key:"fake-leave-stall"}); + stall=true; + const error=await within(service.leave().then(()=>null,(caught:unknown)=>caught)); + assert.ok(error instanceof RelaycastCallError); + assert.equal(error.outcomeUnknown,true); + assert.match(relayFailure(error,"leave").content[0]!.text,/^Leaving the hosted relay participant failed\. It is safe to retry leave_relay\./); + assert.equal(seen.find(call=>call.path==="/v1/agents/release-exact")?.signal,true); + // The transaction rolled back: the participant row is kept for the retry. + assert.equal((await f.database.query("SELECT id FROM hosted_relay_participants")).rows.length,1); + assert.equal(f.world.agents.has("approved-agent"),!applied); + stall=false; + assert.deepEqual(await within(service.leave()),{joined:false,already_left:false}); + assert.equal(f.world.agents.has("approved-agent"),false); + assert.equal((await f.database.query("SELECT id FROM hosted_relay_participants")).rows.length,0); + }); +} + +test("a Relaycast answer whose body is lost mid-read is an unknown outcome",async()=>{ + const fetchImpl:typeof fetch=async(_input,init)=>{ + const signal=init!.signal!; + // Headers arrive; the body stream errors only when the caller's signal aborts. + const body=new ReadableStream({start(controller){ + signal.addEventListener("abort",()=>controller.error(signal.reason),{once:true}); + }}); + return new Response(body,{status:200,headers:{"content-type":"application/json"}}); + }; + const client=createRelaycastClient({internalOrigin:"http://fake-relay.test",fetchImpl}); + const controller=new AbortController(); + const reading=client.authenticatedAgent("at_fake",controller.signal).then(()=>null,(caught:unknown)=>caught); + setTimeout(()=>controller.abort(),10); + const error=await within(reading); + assert.ok(error instanceof RelaycastCallError); + assert.equal(error.status,200); + assert.equal(error.outcomeUnknown,true); + assert.equal(error.operation,"authenticated_agent"); +}); + test("a retryable failure backs off exponentially and retries only once due",async t=>{ assert.deepEqual([1,2,3,6,7,40].map(mcpOAuthTeardownBackoffMs),[60_000,120_000,240_000,1_920_000,3_600_000,3_600_000]); const f=await relayFixture(t);await f.service.join({});