diff --git a/docs/repository-guide.md b/docs/repository-guide.md index 727a4af1..69c5348d 100644 --- a/docs/repository-guide.md +++ b/docs/repository-guide.md @@ -975,3 +975,13 @@ no Agent Relay house key ("Agent Relay API key", "OpenCode managed"), keys are not probed at save (use Check token). Members are added directly (there is no email invite flow), and workspaces cannot be deleted from the API. + +Connection management listings verify authorization with read-only provider +identity requests. `authorizationStatus` (`active`, `inactive`, `unknown`), +`authorizationReason` and `authorizationCheckedAt` describe that check separately +from the stored connection status. A saved credential alone is not verification. +Refresh retries unknown checks; rate limits, insufficient verification scopes, +unsupported providers and outages remain unknown rather than appearing revoked. +The provider checks and bounded scheduler come from `@relayfile/connection-health`. + +Authorization checks cache completed proofs for 30 seconds, keyed by workspace and credential revision. Explicit Refresh (`refreshAuthorization=true` on management GETs) bypasses completed proofs; concurrent reads still share in-flight work. Up to eight checks run at once, with at most five seconds queued and five seconds active (ten seconds total for authorization checks). Overflow returns unknown/checks_busy. Unknown results are not cached and emit a sanitized, per-provider/reason warning at most once a minute. Runtime/environment listings keep their existing readiness contract. diff --git a/package-lock.json b/package-lock.json index 174d4d03..295f09f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,8 @@ "pg": "^8.16.3", "ws": "^8.23.0", "yaml": "^2.9.1", - "zod": "^4.6.7" + "zod": "^4.6.7", + "@relayfile/connection-health": "0.1.1" }, "bin": { "agent-relay-analytics-collector": "dist/collector-cli.js", @@ -2270,6 +2271,12 @@ "peerDependencies": { "zod": "^3.25.28 || ^4" } + }, + "node_modules/@relayfile/connection-health": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@relayfile/connection-health/-/connection-health-0.1.1.tgz", + "integrity": "sha512-4HHrCZM/BtV80sdzoP+rKB4CFf6Wmf4WyEZUyVulVql9G5kjGK+iO9Qhlwjcjd+paRn3gdC4pmwtgGIeDHZOjA==", + "license": "MIT" } } } diff --git a/package.json b/package.json index 3ce9d54e..fdd5914f 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,8 @@ "pg": "^8.16.3", "ws": "^8.23.0", "yaml": "^2.9.1", - "zod": "^4.6.7" + "zod": "^4.6.7", + "@relayfile/connection-health": "0.1.1" }, "devDependencies": { "@electric-sql/pglite": "^0.3.16", diff --git a/src/admin/connections.ts b/src/admin/connections.ts index 4939de4f..46461822 100644 --- a/src/admin/connections.ts +++ b/src/admin/connections.ts @@ -1,3 +1,4 @@ +import { AuthorizationProbeQueue, credentialTransport, verifyProviderAuthorization, type AuthorizationHealth } from "@relayfile/connection-health"; import { createHash, randomBytes, randomUUID } from "node:crypto"; import type { Database, SqlExecutor } from "../database.js"; import { @@ -52,7 +53,7 @@ export type OAuthAppRecord = { }; /** Public connection record; never carries tokens or API keys. */ -export type ConnectionRecord = { +export type ConnectionRecord = Partial & { connectionId: string; version: string; provider: string; @@ -78,6 +79,8 @@ export type ConnectionRecord = { export type RuntimeToken = { connectionId: string; + /** Exact encrypted credential snapshot returned atomically with this token. */ + credentialVersion: string; provider: string; authMode: "oauth2" | "api_key" | "github_app"; installationId: string | null; @@ -838,6 +841,74 @@ export class ConnectionStore { return result.rows.map((row) => this.toRecord(row)); } + private readonly authorizationQueue = new AuthorizationProbeQueue({concurrency:8, networkMs:5_000, queueMs:5_000, maxPending:64}); + private readonly authorizationCache = new Map(); + private readonly authorizationWarnings = new Map(); + + /** Live proof for management listings. Runtime/env callers keep the plain list contract. */ + async listHealth(workspaceId: string, actor: AuditActor, options: {refreshAuthorization?:boolean} = {}): Promise { + return Promise.all((await this.list(workspaceId)).map(async record => { + const cacheKey = JSON.stringify([workspaceId, record.connectionId, record.version]); + const cached = this.authorizationCache.get(cacheKey); + if (!options.refreshAuthorization && cached && cached.expires > Date.now()) return {...record,...cached.health,authMode:record.authMode}; + const started = Date.now(); + let checkedVersion = record.version; + const health = await this.authorizationQueue.probe( + JSON.stringify([workspaceId, record.connectionId, record.version]), + async signal => { + try { + const token = await this.runtimeToken({workspaceId, provider:record.provider, actor}); + if (token.connectionId !== record.connectionId) throw new Error("connection_changed"); + checkedVersion = token.credentialVersion; + const verified = await verifyProviderAuthorization({ + provider:record.provider, + authMode:record.authMode === "github_app" ? "APP" : record.authMode, + transport:credentialTransport({accessToken:token.accessToken,tokenType:token.tokenType,signal, + fetch:(url,init)=>this.fetchImpl(String(url),init ?? {})}), + }); + const current = await this.get(workspaceId, record.provider); + if (!current || current.connectionId !== record.connectionId || current.version !== token.credentialVersion) throw new Error("connection_changed"); + checkedVersion = current.version; + // Presence at the native credential authority is separate from authorization. + return {...verified,upstreamConnected:true}; + } catch (error) { + const failed = await this.get(workspaceId, record.provider).catch(() => null); + const sameBinding = failed?.connectionId === record.connectionId; + const expired = failed?.authMode === "oauth2" && !failed.refreshable && + failed.accessTokenExpiresAt !== null && Date.parse(failed.accessTokenExpiresAt) <= Date.now(); + const inactive = error instanceof ConnectionError && sameBinding && ( + (error.code === "token_expired" && expired) || + (error.code === "refresh_failed" && failed?.lastError === "refresh_failed: invalid_grant") + ); + // runtimeToken commits its own error marker. That marker changes the + // edit version, but it must not erase a definitive authorization failure. + if (inactive && failed) checkedVersion = failed.version; + return {authorizationStatus:inactive ? "inactive" as const : "unknown" as const, + authorizationReason:inactive ? "credentials_rejected" : "credentials_unavailable", + authorizationCheckedAt:new Date().toISOString(),upstreamConnected:inactive ? true : null}; + } + }, + ); + const current = await this.get(workspaceId, record.provider).catch(() => null); + if (!current || current.connectionId !== record.connectionId || current.version !== checkedVersion) { + return {...(current ?? record),authorizationStatus:"unknown" as const, + authorizationReason:"connection_changed",authorizationCheckedAt:new Date().toISOString(),upstreamConnected:null}; + } + if (health.authorizationStatus !== "unknown") { + if (this.authorizationCache.size >= 512) this.authorizationCache.delete(this.authorizationCache.keys().next().value!); + this.authorizationCache.set(JSON.stringify([workspaceId,current.connectionId,current.version]), {expires:Date.now()+30_000,health}); + } else { + const key = JSON.stringify([record.provider,health.authorizationReason]); + if ((this.authorizationWarnings.get(key) ?? 0) <= Date.now()) { + if (this.authorizationWarnings.size >= 128) this.authorizationWarnings.delete(this.authorizationWarnings.keys().next().value!); + this.authorizationWarnings.set(key,Date.now()+60_000); + console.warn("[connection-authorization]",{backend:"native",provider:record.provider,reason:health.authorizationReason,durationMs:Date.now()-started}); + } + } + return {...current,...health,authMode:current.authMode}; + })); + } + async get(workspaceId: string, providerId: string): Promise { const provider = this.provider(providerId); const result = await this.database.query( @@ -1139,6 +1210,7 @@ export class ConnectionStore { delete metadata.config; return { connectionId: row.id, + credentialVersion: connectionVersion(row), provider: row.provider, authMode: row.auth_mode, installationId: row.installation_id, diff --git a/src/admin/routes.ts b/src/admin/routes.ts index 43e71fce..93f0b0b8 100644 --- a/src/admin/routes.ts +++ b/src/admin/routes.ts @@ -142,7 +142,12 @@ function listEntry( provider: connection.provider, providerConfigKey: provider?.configKey ?? connection.provider, status: connection.status, - upstreamConnected: connection.status === "ready", + upstreamConnected: connection.authorizationStatus === undefined + ? connection.status === "ready" // Existing internal runtime readiness contract. + : connection.upstreamConnected ?? null, + authorizationStatus: connection.authorizationStatus ?? "unknown", + authorizationReason: connection.authorizationReason ?? "not_checked", + authorizationCheckedAt: connection.authorizationCheckedAt, deprecated: false, backend: "selfhost", }; @@ -513,7 +518,7 @@ export function createAdminModule(input: { // Provider connections. if (rest === "integrations" && method === "GET") { const access = await helpers.workspaceAccess(request, workspaceId, false); - const list = await connections.list(access.workspace.id); + const list = await connections.listHealth(access.workspace.id, userActor(access), {refreshAuthorization:url.searchParams.get("refreshAuthorization") === "true"}); // Bare array, as hosted (relayfile CLI `integration list` parses this). json(response, 200, list.map((connection) => listEntry(providers.get(connection.provider), connection) @@ -525,7 +530,7 @@ export function createAdminModule(input: { const access = await helpers.workspaceAccess(request, workspaceId, false); const [apps, list] = await Promise.all([ connections.listOAuthApps(access.workspace.id), - connections.list(access.workspace.id), + connections.listHealth(access.workspace.id, userActor(access), {refreshAuthorization:url.searchParams.get("refreshAuthorization") === "true"}), ]); json(response, 200, { redirectUri: config.oauthRedirectUrl, diff --git a/test/admin.test.ts b/test/admin.test.ts index fd991436..f01e094b 100644 --- a/test/admin.test.ts +++ b/test/admin.test.ts @@ -13,7 +13,8 @@ import { import { maskValue, envKeyReachesRuntime } from "../src/admin/env-policy.js"; import type { SelfhostConfig } from "../src/config.js"; import { loadConfig, loadOperatorOAuthApps } from "../src/config.js"; -import { decryptSecret } from "../src/crypto.js"; +import { decryptSecret, encryptSecret } from "../src/crypto.js"; +import type { FetchLike } from "../src/admin/connections.js"; import { migrate } from "../src/migrations.js"; import { ControlPlaneStore } from "../src/store.js"; import { close, listen, PGliteDatabase, request } from "./support/harness.js"; @@ -151,7 +152,7 @@ async function deviceToken( return issued.body.access_token as string; } -async function setup(t: test.TestContext, extra: Partial = {}) { +async function setup(t: test.TestContext, extra: Partial = {}, fetchImpl?: FetchLike) { const mock = await startMockOAuth(); const database = new PGliteDatabase(); await migrate(database); @@ -177,7 +178,7 @@ async function setup(t: test.TestContext, extra: Partial = {}) { ...extra, }; const store = new ControlPlaneStore(database, KEYRING, 0); - const admin = createAdminModule({ config, database, keyring: KEYRING, providers }); + const admin = createAdminModule({ config, database, keyring: KEYRING, providers, ...(fetchImpl ? {fetchImpl} : {}) }); control.on("request", createApp({ config, store, admin })); t.after(async () => { await close(control); @@ -210,6 +211,7 @@ async function setup(t: test.TestContext, extra: Partial = {}) { const workspaceId = workspaces[0]!; return { database, + connections: admin.connections, mock, origin: controlOrigin, organizationId: org.body.id as string, @@ -953,3 +955,77 @@ test("operator OAuth apps load from env and mounted Secret files", async () => { await rm(dir, { recursive: true, force: true }); } }); + +test("connection health: catalog and listing verify API keys, preserve refusal vs outage, and retry", async t => { + let providerStatus=200; + const requests: Array<{url:string;authorization:string|null}> = []; + const ctx=await setup(t,{},async(url,init)=>{ + requests.push({url,authorization:new Headers(init.headers).get("authorization")}); + return Response.json({id:"neon-user",email:"bot@example.test"}, {status:providerStatus}); + }); + const ws=`/api/v1/workspaces/${ctx.workspaceId}`; + const saved=await request(ctx.origin,`${ws}/integrations/neon`,{method:"POST",token:ctx.owner,body:{credentials:{apiKey:"health-secret-sentinel"}}}); + assert.equal(saved.status,201); + const catalog=await request(ctx.origin,`${ws}/integrations/providers`,{token:ctx.member}); + const connection=catalog.body.providers.find((p:any)=>p.id === "neon").connection; + assert.equal(connection.authorizationStatus,"active"); + assert.equal(connection.authorizationReason,"provider_verified"); + assert.equal(requests[0]?.url,"https://console.neon.tech/api/v2/users/me"); + assert.equal(requests[0]?.authorization,"Bearer health-secret-sentinel"); + assert(!catalog.raw.includes("health-secret-sentinel")); + await request(ctx.origin,`${ws}/integrations`,{token:ctx.member}); + assert.equal(requests.length,1,"repeated listings reuse the revision-bound proof"); + const internal=await request(ctx.origin,`/internal/v1/workspaces/${ctx.workspaceId}/integrations`,{token:SERVICE}); + assert.equal(internal.status,200); + assert.equal(internal.body.integrations[0].upstreamConnected,true,"internal local readiness contract is unchanged"); + providerStatus=429; + const limited=await request(ctx.origin,`${ws}/integrations?refreshAuthorization=true`,{token:ctx.member}); + assert.equal(limited.body[0].authorizationStatus,"unknown"); + assert.equal(limited.body[0].authorizationReason,"rate_limited"); + assert.equal(limited.body[0].upstreamConnected,true); + providerStatus=401; + const rejected=await request(ctx.origin,`${ws}/integrations?refreshAuthorization=true`,{token:ctx.member}); + assert.equal(rejected.body[0].authorizationStatus,"inactive"); + providerStatus=200; + const retried=await request(ctx.origin,`${ws}/integrations?refreshAuthorization=true`,{token:ctx.member}); + assert.equal(retried.body[0].authorizationStatus,"active"); + const other=await request(ctx.origin,`/api/v1/workspaces/${ctx.otherWorkspaceId}/integrations`,{token:ctx.owner}); + assert.notEqual(other.status,200); +}); + +test("expired unrefreshable credentials stay inactive despite the committed error-marker version", async t => { + const ctx=await setup(t); + await ctx.database.query(`INSERT INTO integration_connections + (id,workspace_id,provider,auth_mode,status,credentials_ciphertext,token_type,access_token_expires_at) + VALUES (gen_random_uuid(),$1,'mockhub','oauth2','ready',$2,'bearer',now()-interval '1 hour')`, + [ctx.workspaceId,encryptSecret(JSON.stringify({accessToken:"expired-secret",refreshToken:null}),KEYRING.active)]); + const ws=`/api/v1/workspaces/${ctx.workspaceId}`; + for(let attempt=0;attempt<2;attempt++){ + const listed=await request(ctx.origin,`${ws}/integrations`,{token:ctx.member}); + assert.equal(listed.status,200); + assert.equal(listed.body[0].authorizationStatus,"inactive"); + assert.equal(listed.body[0].authorizationReason,"credentials_rejected"); + assert(!listed.raw.includes("expired-secret")); + } +}); + +test("authorization proof is bound atomically to the token version across same-id reconnects", async t => { + const ctx=await setup(t,{},async()=>Response.json({id:"neon-user",email:"bot@example.test"})); + const ws=`/api/v1/workspaces/${ctx.workspaceId}`; + const saved=await request(ctx.origin,`${ws}/integrations/neon`,{method:"POST",token:ctx.owner,body:{credentials:{apiKey:"old-key"}}}); + assert.equal(saved.status,201); + const original=ctx.connections.runtimeToken.bind(ctx.connections); + ctx.connections.runtimeToken=async input=>{ + const token=await original(input); + await ctx.database.query(`UPDATE integration_connections SET credentials_ciphertext=$2,updated_at=now() + WHERE workspace_id=$1 AND provider='neon'`, + [ctx.workspaceId,encryptSecret(JSON.stringify({fields:{apiKey:"replacement-key"}}),KEYRING.active)]); + return token; + }; + const listed=await request(ctx.origin,`${ws}/integrations`,{token:ctx.member}); + assert.equal(listed.status,200); + assert.equal(listed.body[0].connectionId,saved.body.connectionId,"reconnect retained the binding id"); + assert.equal(listed.body[0].authorizationStatus,"unknown","old credentials cannot verify a replacement"); + assert.equal(listed.body[0].authorizationReason,"connection_changed"); + assert(!listed.raw.includes("old-key") && !listed.raw.includes("replacement-key")); +});