Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions docs/repository-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -975,3 +975,13 @@ no Agent Relay house key ("Agent Relay API key", "OpenCode managed"), keys are
not probed at save (use Check token). Members are
added directly (there is no email invite flow), and workspaces cannot be
deleted from the API.

Connection management listings verify authorization with read-only provider
identity requests. `authorizationStatus` (`active`, `inactive`, `unknown`),
`authorizationReason` and `authorizationCheckedAt` describe that check separately
from the stored connection status. A saved credential alone is not verification.
Refresh retries unknown checks; rate limits, insufficient verification scopes,
unsupported providers and outages remain unknown rather than appearing revoked.
The provider checks and bounded scheduler come from `@relayfile/connection-health`.

Authorization checks cache completed proofs for 30 seconds, keyed by workspace and credential revision. Explicit Refresh (`refreshAuthorization=true` on management GETs) bypasses completed proofs; concurrent reads still share in-flight work. Up to eight checks run at once, with at most five seconds queued and five seconds active (ten seconds total for authorization checks). Overflow returns unknown/checks_busy. Unknown results are not cached and emit a sanitized, per-provider/reason warning at most once a minute. Runtime/environment listings keep their existing readiness contract.
9 changes: 8 additions & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@
"pg": "^8.16.3",
"ws": "^8.23.0",
"yaml": "^2.9.1",
"zod": "^4.6.7"
"zod": "^4.6.7",
"@relayfile/connection-health": "0.1.1"
},
"devDependencies": {
"@electric-sql/pglite": "^0.3.16",
Expand Down
74 changes: 73 additions & 1 deletion src/admin/connections.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { AuthorizationProbeQueue, credentialTransport, verifyProviderAuthorization, type AuthorizationHealth } from "@relayfile/connection-health";
import { createHash, randomBytes, randomUUID } from "node:crypto";
import type { Database, SqlExecutor } from "../database.js";
import {
Expand Down Expand Up @@ -52,7 +53,7 @@ export type OAuthAppRecord = {
};

/** Public connection record; never carries tokens or API keys. */
export type ConnectionRecord = {
export type ConnectionRecord = Partial<AuthorizationHealth> & {
connectionId: string;
version: string;
provider: string;
Expand All @@ -78,6 +79,8 @@ export type ConnectionRecord = {

export type RuntimeToken = {
connectionId: string;
/** Exact encrypted credential snapshot returned atomically with this token. */
credentialVersion: string;
provider: string;
authMode: "oauth2" | "api_key" | "github_app";
installationId: string | null;
Expand Down Expand Up @@ -838,6 +841,74 @@ export class ConnectionStore {
return result.rows.map((row) => this.toRecord(row));
}

private readonly authorizationQueue = new AuthorizationProbeQueue({concurrency:8, networkMs:5_000, queueMs:5_000, maxPending:64});
private readonly authorizationCache = new Map<string, {expires:number; health:AuthorizationHealth}>();
private readonly authorizationWarnings = new Map<string, number>();

/** Live proof for management listings. Runtime/env callers keep the plain list contract. */
async listHealth(workspaceId: string, actor: AuditActor, options: {refreshAuthorization?:boolean} = {}): Promise<ConnectionRecord[]> {
return Promise.all((await this.list(workspaceId)).map(async record => {
const cacheKey = JSON.stringify([workspaceId, record.connectionId, record.version]);
const cached = this.authorizationCache.get(cacheKey);
if (!options.refreshAuthorization && cached && cached.expires > Date.now()) return {...record,...cached.health,authMode:record.authMode};
const started = Date.now();
let checkedVersion = record.version;
const health = await this.authorizationQueue.probe(
JSON.stringify([workspaceId, record.connectionId, record.version]),
async signal => {
try {
const token = await this.runtimeToken({workspaceId, provider:record.provider, actor});
if (token.connectionId !== record.connectionId) throw new Error("connection_changed");
checkedVersion = token.credentialVersion;
const verified = await verifyProviderAuthorization({
provider:record.provider,
authMode:record.authMode === "github_app" ? "APP" : record.authMode,
transport:credentialTransport({accessToken:token.accessToken,tokenType:token.tokenType,signal,
fetch:(url,init)=>this.fetchImpl(String(url),init ?? {})}),
});
const current = await this.get(workspaceId, record.provider);
if (!current || current.connectionId !== record.connectionId || current.version !== token.credentialVersion) throw new Error("connection_changed");
checkedVersion = current.version;
// Presence at the native credential authority is separate from authorization.
return {...verified,upstreamConnected:true};
Comment thread
khaliqgant marked this conversation as resolved.
Resolved
} catch (error) {
const failed = await this.get(workspaceId, record.provider).catch(() => null);
const sameBinding = failed?.connectionId === record.connectionId;
const expired = failed?.authMode === "oauth2" && !failed.refreshable &&
failed.accessTokenExpiresAt !== null && Date.parse(failed.accessTokenExpiresAt) <= Date.now();
const inactive = error instanceof ConnectionError && sameBinding && (
(error.code === "token_expired" && expired) ||
(error.code === "refresh_failed" && failed?.lastError === "refresh_failed: invalid_grant")
);
// runtimeToken commits its own error marker. That marker changes the
// edit version, but it must not erase a definitive authorization failure.
if (inactive && failed) checkedVersion = failed.version;
return {authorizationStatus:inactive ? "inactive" as const : "unknown" as const,
authorizationReason:inactive ? "credentials_rejected" : "credentials_unavailable",
authorizationCheckedAt:new Date().toISOString(),upstreamConnected:inactive ? true : null};
}
},
);
const current = await this.get(workspaceId, record.provider).catch(() => null);
if (!current || current.connectionId !== record.connectionId || current.version !== checkedVersion) {
return {...(current ?? record),authorizationStatus:"unknown" as const,
authorizationReason:"connection_changed",authorizationCheckedAt:new Date().toISOString(),upstreamConnected:null};
}
if (health.authorizationStatus !== "unknown") {
if (this.authorizationCache.size >= 512) this.authorizationCache.delete(this.authorizationCache.keys().next().value!);
this.authorizationCache.set(JSON.stringify([workspaceId,current.connectionId,current.version]), {expires:Date.now()+30_000,health});
} else {
const key = JSON.stringify([record.provider,health.authorizationReason]);
if ((this.authorizationWarnings.get(key) ?? 0) <= Date.now()) {
if (this.authorizationWarnings.size >= 128) this.authorizationWarnings.delete(this.authorizationWarnings.keys().next().value!);
this.authorizationWarnings.set(key,Date.now()+60_000);
console.warn("[connection-authorization]",{backend:"native",provider:record.provider,reason:health.authorizationReason,durationMs:Date.now()-started});
}
}
return {...current,...health,authMode:current.authMode};
}));
}

async get(workspaceId: string, providerId: string): Promise<ConnectionRecord | null> {
const provider = this.provider(providerId);
const result = await this.database.query<ConnectionRow>(
Expand Down Expand Up @@ -1139,6 +1210,7 @@ export class ConnectionStore {
delete metadata.config;
return {
connectionId: row.id,
credentialVersion: connectionVersion(row),
provider: row.provider,
authMode: row.auth_mode,
installationId: row.installation_id,
Expand Down
11 changes: 8 additions & 3 deletions src/admin/routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,12 @@ function listEntry(
provider: connection.provider,
providerConfigKey: provider?.configKey ?? connection.provider,
status: connection.status,
upstreamConnected: connection.status === "ready",
upstreamConnected: connection.authorizationStatus === undefined
? connection.status === "ready" // Existing internal runtime readiness contract.
: connection.upstreamConnected ?? null,
authorizationStatus: connection.authorizationStatus ?? "unknown",
authorizationReason: connection.authorizationReason ?? "not_checked",
authorizationCheckedAt: connection.authorizationCheckedAt,
deprecated: false,
backend: "selfhost",
};
Expand Down Expand Up @@ -513,7 +518,7 @@ export function createAdminModule(input: {
// Provider connections.
if (rest === "integrations" && method === "GET") {
const access = await helpers.workspaceAccess(request, workspaceId, false);
const list = await connections.list(access.workspace.id);
const list = await connections.listHealth(access.workspace.id, userActor(access), {refreshAuthorization:url.searchParams.get("refreshAuthorization") === "true"});
// Bare array, as hosted (relayfile CLI `integration list` parses this).
json(response, 200, list.map((connection) =>
listEntry(providers.get(connection.provider), connection)
Expand All @@ -525,7 +530,7 @@ export function createAdminModule(input: {
const access = await helpers.workspaceAccess(request, workspaceId, false);
const [apps, list] = await Promise.all([
connections.listOAuthApps(access.workspace.id),
connections.list(access.workspace.id),
connections.listHealth(access.workspace.id, userActor(access), {refreshAuthorization:url.searchParams.get("refreshAuthorization") === "true"}),
]);
json(response, 200, {
redirectUri: config.oauthRedirectUrl,
Expand Down
82 changes: 79 additions & 3 deletions test/admin.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ import {
import { maskValue, envKeyReachesRuntime } from "../src/admin/env-policy.js";
import type { SelfhostConfig } from "../src/config.js";
import { loadConfig, loadOperatorOAuthApps } from "../src/config.js";
import { decryptSecret } from "../src/crypto.js";
import { decryptSecret, encryptSecret } from "../src/crypto.js";
import type { FetchLike } from "../src/admin/connections.js";
import { migrate } from "../src/migrations.js";
import { ControlPlaneStore } from "../src/store.js";
import { close, listen, PGliteDatabase, request } from "./support/harness.js";
Expand Down Expand Up @@ -151,7 +152,7 @@ async function deviceToken(
return issued.body.access_token as string;
}

async function setup(t: test.TestContext, extra: Partial<SelfhostConfig> = {}) {
async function setup(t: test.TestContext, extra: Partial<SelfhostConfig> = {}, fetchImpl?: FetchLike) {
const mock = await startMockOAuth();
const database = new PGliteDatabase();
await migrate(database);
Expand All @@ -177,7 +178,7 @@ async function setup(t: test.TestContext, extra: Partial<SelfhostConfig> = {}) {
...extra,
};
const store = new ControlPlaneStore(database, KEYRING, 0);
const admin = createAdminModule({ config, database, keyring: KEYRING, providers });
const admin = createAdminModule({ config, database, keyring: KEYRING, providers, ...(fetchImpl ? {fetchImpl} : {}) });
control.on("request", createApp({ config, store, admin }));
t.after(async () => {
await close(control);
Expand Down Expand Up @@ -210,6 +211,7 @@ async function setup(t: test.TestContext, extra: Partial<SelfhostConfig> = {}) {
const workspaceId = workspaces[0]!;
return {
database,
connections: admin.connections,
mock,
origin: controlOrigin,
organizationId: org.body.id as string,
Expand Down Expand Up @@ -953,3 +955,77 @@ test("operator OAuth apps load from env and mounted Secret files", async () => {
await rm(dir, { recursive: true, force: true });
}
});

test("connection health: catalog and listing verify API keys, preserve refusal vs outage, and retry", async t => {
let providerStatus=200;
const requests: Array<{url:string;authorization:string|null}> = [];
const ctx=await setup(t,{},async(url,init)=>{
requests.push({url,authorization:new Headers(init.headers).get("authorization")});
return Response.json({id:"neon-user",email:"bot@example.test"}, {status:providerStatus});
});
const ws=`/api/v1/workspaces/${ctx.workspaceId}`;
const saved=await request(ctx.origin,`${ws}/integrations/neon`,{method:"POST",token:ctx.owner,body:{credentials:{apiKey:"health-secret-sentinel"}}});
assert.equal(saved.status,201);
const catalog=await request(ctx.origin,`${ws}/integrations/providers`,{token:ctx.member});
const connection=catalog.body.providers.find((p:any)=>p.id === "neon").connection;
assert.equal(connection.authorizationStatus,"active");
assert.equal(connection.authorizationReason,"provider_verified");
assert.equal(requests[0]?.url,"https://console.neon.tech/api/v2/users/me");
assert.equal(requests[0]?.authorization,"Bearer health-secret-sentinel");
assert(!catalog.raw.includes("health-secret-sentinel"));
await request(ctx.origin,`${ws}/integrations`,{token:ctx.member});
assert.equal(requests.length,1,"repeated listings reuse the revision-bound proof");
const internal=await request(ctx.origin,`/internal/v1/workspaces/${ctx.workspaceId}/integrations`,{token:SERVICE});
assert.equal(internal.status,200);
assert.equal(internal.body.integrations[0].upstreamConnected,true,"internal local readiness contract is unchanged");
providerStatus=429;
const limited=await request(ctx.origin,`${ws}/integrations?refreshAuthorization=true`,{token:ctx.member});
assert.equal(limited.body[0].authorizationStatus,"unknown");
assert.equal(limited.body[0].authorizationReason,"rate_limited");
assert.equal(limited.body[0].upstreamConnected,true);
providerStatus=401;
const rejected=await request(ctx.origin,`${ws}/integrations?refreshAuthorization=true`,{token:ctx.member});
assert.equal(rejected.body[0].authorizationStatus,"inactive");
providerStatus=200;
const retried=await request(ctx.origin,`${ws}/integrations?refreshAuthorization=true`,{token:ctx.member});
assert.equal(retried.body[0].authorizationStatus,"active");
const other=await request(ctx.origin,`/api/v1/workspaces/${ctx.otherWorkspaceId}/integrations`,{token:ctx.owner});
assert.notEqual(other.status,200);
});

test("expired unrefreshable credentials stay inactive despite the committed error-marker version", async t => {
const ctx=await setup(t);
await ctx.database.query(`INSERT INTO integration_connections
(id,workspace_id,provider,auth_mode,status,credentials_ciphertext,token_type,access_token_expires_at)
VALUES (gen_random_uuid(),$1,'mockhub','oauth2','ready',$2,'bearer',now()-interval '1 hour')`,
[ctx.workspaceId,encryptSecret(JSON.stringify({accessToken:"expired-secret",refreshToken:null}),KEYRING.active)]);
const ws=`/api/v1/workspaces/${ctx.workspaceId}`;
for(let attempt=0;attempt<2;attempt++){
const listed=await request(ctx.origin,`${ws}/integrations`,{token:ctx.member});
assert.equal(listed.status,200);
assert.equal(listed.body[0].authorizationStatus,"inactive");
assert.equal(listed.body[0].authorizationReason,"credentials_rejected");
assert(!listed.raw.includes("expired-secret"));
}
});

test("authorization proof is bound atomically to the token version across same-id reconnects", async t => {
const ctx=await setup(t,{},async()=>Response.json({id:"neon-user",email:"bot@example.test"}));
const ws=`/api/v1/workspaces/${ctx.workspaceId}`;
const saved=await request(ctx.origin,`${ws}/integrations/neon`,{method:"POST",token:ctx.owner,body:{credentials:{apiKey:"old-key"}}});
assert.equal(saved.status,201);
const original=ctx.connections.runtimeToken.bind(ctx.connections);
ctx.connections.runtimeToken=async input=>{
const token=await original(input);
await ctx.database.query(`UPDATE integration_connections SET credentials_ciphertext=$2,updated_at=now()
WHERE workspace_id=$1 AND provider='neon'`,
[ctx.workspaceId,encryptSecret(JSON.stringify({fields:{apiKey:"replacement-key"}}),KEYRING.active)]);
return token;
};
const listed=await request(ctx.origin,`${ws}/integrations`,{token:ctx.member});
assert.equal(listed.status,200);
assert.equal(listed.body[0].connectionId,saved.body.connectionId,"reconnect retained the binding id");
assert.equal(listed.body[0].authorizationStatus,"unknown","old credentials cannot verify a replacement");
assert.equal(listed.body[0].authorizationReason,"connection_changed");
assert(!listed.raw.includes("old-key") && !listed.raw.includes("replacement-key"));
});
Loading