diff --git a/.agents/skills/add-block/SKILL.md b/.agents/skills/add-block/SKILL.md index 02584416cef..c74e459a06f 100644 --- a/.agents/skills/add-block/SKILL.md +++ b/.agents/skills/add-block/SKILL.md @@ -163,6 +163,32 @@ export const {ServiceName}Block: BlockConfig = { Optional companions: `credentialLabels` (override the picker's section/connect-row copy) and `allowServiceAccounts: true` (trigger-mode only — list service accounts, which triggers otherwise exclude; set only when the trigger's polling path can resolve a service-account token). The connect modal, provider families (Google JSON key, Atlassian token, token-paste, client-credential, Slack bot), and the preview gate are all resolved from `serviceAccountProviderId` — you don't wire them per block. +### OAuth deployment availability (required for integration blocks) + +A visible tools-category block with OAuth is deployment-gated. Its `oauth-input.serviceId` is +projected into `apps/sim/lib/integrations/integrations.json`, then resolved through +`resolveOAuthClientCapabilityId()` in `apps/sim/lib/core/config/env-capabilities.ts`. + +When adding or changing an OAuth integration block: + +1. Keep exactly one distinct OAuth `serviceId` across the block's `oauth-input` subBlocks. +2. Confirm that service ID resolves to an entry in `OAUTH_CLIENT_CAPABILITIES`. Google and + Microsoft service IDs intentionally share their provider-level capability; do not add duplicate + entries for those aliases. +3. For a new capability, add its required client fields to `OAUTH_CLIENT_CAPABILITIES` and ensure + every referenced field exists in the env schema in `apps/sim/lib/core/config/env.ts`. Then add + the matching `text` or `secret` input modes to `OAUTH_CLIENT_SETUP_FIELDS` in + `scripts/setup/capability-config.ts`. The CLI catalog is exhaustively typed and checked against + the runtime field list; do not infer secrecy from the field name. +4. If the canonical OAuth service declares `serviceAccountProviderId`, keep + `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` in + `apps/sim/lib/integrations/service-account-metadata.ts` aligned. Set + `deploymentRequirement` only when the service-account path is preview-gated or depends on the + OAuth client fields; otherwise omit it. + +Missing capability metadata is a runtime configuration error, not a reason to make the integration +silently available. + ### Selectors (with dynamic options) ```typescript // Channel selector (Slack, Discord, etc.) @@ -919,12 +945,25 @@ Derive templates from the service's real use cases. Each prompt should name a co - **Ground every skill in operations the block actually exposes** — cross-check each skill's steps against `tools.access`. Never describe an action the integration cannot perform. - **Derive skills from real, popular use cases found online — never invent them.** Web-search the service's documented use cases (vendor use-case/solutions pages, official docs describing the workflow, reputable "top automations for X" articles) and only add a skill you can source as something people genuinely do with the service. Do not hallucinate skills. -## Generated tool metadata +## Generated artifacts -Adding a block on its own needs **no** regeneration — a block references existing tool IDs through `tools.access` and does not change any tool's shape. +Adding a block on its own needs no **tool metadata** regeneration — a block references existing +tool IDs through `tools.access` and does not change any tool's shape. But if the same change also adds, edits **or removes** a tool, run `bun run tool-metadata:generate` and commit the result, or CI fails on stale artifacts. That matters here because a block's `outputs` are authored to match its tools' outputs, and the UI now reads those from the generated metadata rather than the executable registry — an unregenerated tool change makes the block's outputs disagree with what the panel renders. See `.agents/skills/tool-registry-boundary/SKILL.md`. +A visible integration block does require the generated integration catalog and docs to be refreshed. +After adding or changing one, run: + +```bash +bun run scripts/generate-docs.ts +bun run integration-catalog:check +``` + +The catalog check independently derives deployment metadata from the executable block registry and +compares it with the committed `apps/sim/lib/integrations/integrations.json`. Review the generated +diff and keep only intentional changes. + ## Checklist Before Finishing - [ ] `integrationType` is set to the correct `IntegrationType` enum value @@ -934,12 +973,17 @@ But if the same change also adds, edits **or removes** a tool, run `bun run tool - [ ] DependsOn set for fields that need other values - [ ] Required fields marked correctly (boolean or condition) - [ ] OAuth inputs have correct `serviceId` and `requiredScopes: getScopesForService(serviceId)` +- [ ] Every OAuth `serviceId` resolves through `resolveOAuthClientCapabilityId()` to the correct `OAUTH_CLIENT_CAPABILITIES` entry +- [ ] Any new OAuth capability fields exist in `apps/sim/lib/core/config/env.ts` +- [ ] If the OAuth service supports service accounts, `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` matches its canonical `serviceAccountProviderId` and deployment requirement - [ ] Scope descriptions added to `SCOPE_DESCRIPTIONS` in `lib/oauth/utils.ts` for any new scopes - [ ] Tools.access lists all tool IDs (snake_case) - [ ] Tools.config.tool returns correct tool ID (snake_case) - [ ] Outputs match tool outputs - [ ] Block + meta registered in registry-maps.ts (`BLOCK_REGISTRY` / `BLOCK_META_REGISTRY`) - [ ] If any tool was added, changed or removed alongside the block: ran `bun run tool-metadata:generate` and committed the artifacts +- [ ] Ran `bun run scripts/generate-docs.ts`, reviewed the generated diff, and committed the integration catalog changes +- [ ] `bun run integration-catalog:check` passes - [ ] If icon missing: asked user to provide SVG - [ ] If triggers exist: `triggers` config set, trigger subBlocks spread - [ ] Optional/rarely-used fields set to `mode: 'advanced'` diff --git a/.agents/skills/add-integration/SKILL.md b/.agents/skills/add-integration/SKILL.md index da7eccf4cd5..58b1f32f800 100644 --- a/.agents/skills/add-integration/SKILL.md +++ b/.agents/skills/add-integration/SKILL.md @@ -17,7 +17,8 @@ Adding an integration involves these steps in order: 4. **Add Icon** - Add the service's brand icon 5. **Create Triggers** (optional) - If the service supports webhooks 6. **Register** - Register tools, block, and triggers in their registries -7. **Generate Docs** - Run the docs generation script +7. **Configure Deployment Availability** - Wire OAuth client and service-account metadata +8. **Generate and Validate the Catalog** - Regenerate docs/catalog artifacts and run drift checks ## Step 1: Research the API @@ -465,15 +466,48 @@ export const TRIGGER_REGISTRY: TriggerRegistry = { } ``` -## Step 7: Generate Docs +## Step 7: Configure Deployment Availability + +Do this for every visible OAuth integration. API-key and unauthenticated integrations do not need +an OAuth client capability. + +The block's `oauth-input.serviceId` is the canonical link between the generated integration catalog, +the OAuth service configuration, deployment availability, and the setup CLI. + +1. Ensure the block has exactly one distinct OAuth `serviceId` and that it matches the canonical + service entry in `apps/sim/lib/oauth/oauth.ts`. +2. Confirm `resolveOAuthClientCapabilityId(serviceId)` resolves to the intended provider entry in + `OAUTH_CLIENT_CAPABILITIES` in `apps/sim/lib/core/config/env-capabilities.ts`. Google and + Microsoft service IDs deliberately share provider-level capabilities. +3. For a new OAuth provider, add the required client fields to `OAUTH_CLIENT_CAPABILITIES`, add + every referenced field to the env schema in `apps/sim/lib/core/config/env.ts`, and add the + matching `text` or `secret` entries to `OAUTH_CLIENT_SETUP_FIELDS` in + `scripts/setup/capability-config.ts`. Do not create integration-specific setup logic or infer + secret fields from naming; the CLI mapping is exhaustively checked against the runtime fields. +4. If the canonical OAuth service has `serviceAccountProviderId`, add the matching projection to + `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` in + `apps/sim/lib/integrations/service-account-metadata.ts`. Use: + - no `deploymentRequirement` when the service-account path works independently of OAuth client fields; + - `'oauth-client'` when it requires the same deployment OAuth client fields; + - `'preview-gated'` when availability is controlled by the service-account preview block. + +Never add a permissive fallback for missing capability metadata. A visible OAuth integration without +a resolvable capability must fail validation. + +## Step 8: Generate and Validate the Catalog Run the documentation generator: ```bash bun run scripts/generate-docs.ts +bun run integration-catalog:check ``` This creates `apps/docs/content/docs/en/integrations/{service}.mdx` — one page per service carrying the block's Actions and, if it has one, its Triggers section. Never hand-edit generated pages; the only editable region is the `{/* MANUAL-CONTENT */}` block (see `scripts/README.md`). +The same generator refreshes `apps/sim/lib/integrations/integrations.json`. The catalog check then +derives the deployment-relevant fields from the executable block registry and compares them with the +committed projection. Review the generated diff and keep only intentional changes. + ## V2 Integration Pattern If creating V2 versions (API-aligned outputs): @@ -524,6 +558,13 @@ If creating V2 versions (API-aligned outputs): - [ ] Used `getCanonicalScopesForProvider()` in `auth.ts` (never hardcode) - [ ] Used `getScopesForService()` in block `requiredScopes` (never hardcode) +### Deployment Availability (if OAuth service) +- [ ] Block declares exactly one distinct `oauth-input.serviceId` +- [ ] `resolveOAuthClientCapabilityId(serviceId)` resolves to the intended `OAUTH_CLIENT_CAPABILITIES` entry +- [ ] Every new OAuth capability field exists in `apps/sim/lib/core/config/env.ts` +- [ ] Runtime OAuth fields live in `OAUTH_CLIENT_CAPABILITIES`; matching CLI input modes live in the exhaustively checked `OAUTH_CLIENT_SETUP_FIELDS` +- [ ] If `serviceAccountProviderId` is configured, `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` has the matching projection and deployment requirement + ### Icon - [ ] Asked user to provide SVG - [ ] Added icon to `components/icons.tsx` @@ -542,6 +583,8 @@ If creating V2 versions (API-aligned outputs): ### Docs - [ ] Ran `bun run scripts/generate-docs.ts` - [ ] Verified docs file created +- [ ] Reviewed and committed the generated `apps/sim/lib/integrations/integrations.json` change +- [ ] `bun run integration-catalog:check` passes ### Final Validation (Required) - [ ] Read every tool file and cross-referenced inputs/outputs against the API docs @@ -886,3 +929,5 @@ requiredScopes: getScopesForService('{service}'), 10. **Complex inputs need wandConfig** - Timestamps, JSON arrays, and other hard-to-type values should have `wandConfig` enabled 11. **Never hardcode scopes** - Use `getScopesForService()` in blocks and `getCanonicalScopesForProvider()` in auth.ts 12. **Always add scope descriptions** - New scopes must have entries in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` +13. **OAuth service IDs need deployment capabilities** - Every visible OAuth integration must resolve through `OAUTH_CLIENT_CAPABILITIES`; shared Google/Microsoft aliases map to their provider capability +14. **Keep runtime and presentation separate** - Runtime OAuth fields live in `env-capabilities.ts`; CLI input modes live in the exhaustively checked `scripts/setup/capability-config.ts` mapping diff --git a/.agents/skills/ship/SKILL.md b/.agents/skills/ship/SKILL.md index 8797df795b3..d1bfd8b4a0e 100644 --- a/.agents/skills/ship/SKILL.md +++ b/.agents/skills/ship/SKILL.md @@ -54,17 +54,23 @@ When the user runs `/ship`: ``` Then `git status --short` to see what regenerated — those files must be staged in step 7 alongside your own changes. - **Do NOT blanket-run the domain generators here.** `mship:generate` (`generate-mship-contracts.ts`) is an **umbrella** that drives all nine mothership contract generators (`mship-contracts`, `billing-protocol-contract`, `mship-tools`, the four `trace-*`, `metrics-contract`, `vfs-snapshot-contract`) and biome-formats `apps/sim/lib/copilot/generated/` — never run it *and* its constituents (they write the same files and corrupt each other in parallel), and never run it on an ordinary ship: it reads an **external** copilot-contract source that isn't checked out in most worktrees, so it hard-fails with `ENOENT` and would abort ship for an unrelated reason. `generate:pi-model-catalog` (under `apps/sim`) likewise regenerates from the installed Pi package, not repo source. Only when **this PR's diff actually touches** a domain generator's input do you regenerate it deliberately and run its matching `:check` (`bun run mship:check` / the individual `*:check`) — with the external source present. + **Do NOT blanket-run the domain generators here.** `mship:generate` (`generate-mship-contracts.ts`) is an **umbrella** that drives all nine mothership contract generators (`mship-contracts`, `billing-protocol-contract`, `mship-tools`, the four `trace-*`, `metrics-contract`, `vfs-snapshot-contract`) and biome-formats `apps/sim/lib/copilot/generated/` — never run it *and* its constituents (they write the same files and corrupt each other in parallel), and never run it on an ordinary ship: it reads an **external** copilot-contract source that isn't checked out in most worktrees, so it hard-fails with `ENOENT` and would abort ship for an unrelated reason. `generate:pi-model-catalog` (under `apps/sim`) likewise regenerates from the installed Pi package, not repo source. `scripts/generate-docs.ts` rewrites the integration docs and client-safe catalog; run it when this PR changes their block/icon/landing-content inputs or when `integration-catalog:check` reports drift, then review its broad generated diff. Only when **this PR's diff actually touches** a domain generator's input do you regenerate it deliberately and run its matching `:check` (`bun run mship:check` / the individual `*:check`) — with the external source present. - **Phase B — run lint + every audit CI enforces, in parallel, and abort ship if any fails.** `bun run lint` first (it autofixes formatting and mutates files, so don't parallelize it with the read-only audits), then fan the rest out and collect exit codes. This is exactly the read-only audit set from CI's `Lint and Test` job (all in-repo, runnable in any worktree): + **Phase B — run lint + every audit CI enforces, in parallel, and abort ship if any fails.** Before running the commands, compare this list with `.github/workflows/test-build.yml`; when CI adds an audit, run it and update this skill instead of trusting a stale snapshot. The env-flag audit is currently an inline workflow block rather than a package script: when `apps/sim/lib/core/config/env-flags.ts` changed, run that current workflow block verbatim instead of copying a second version into this skill. Run `bun run lint` first (it autofixes formatting and mutates files, so don't parallelize it with the read-only audits), then run the base-sensitive block-registry check, then fan the independent audits out and collect exit codes: ```bash # autofix formatting first (mutating; not parallel-safe with the audits). Gate its exit too — # a non-zero lint (unfixable errors) must abort before the audits run, not be ignored. bun run lint || { echo "❌ lint failed — do not ship"; exit 1; } + bun run apps/sim/scripts/check-block-registry.ts origin/staging || { + echo "❌ block registry audit failed — do not ship" + exit 1 + } rm -f /tmp/ship-audit-results - for s in check:boundaries check:api-validation:strict check:utils check:zustand-v5 \ + for s in check:boundaries check:api-validation:strict check:desktop-bridge check:desktop-ipc \ + check:utils check:zustand-v5 \ check:react-query check:client-boundary check:bare-icons check:icon-paths \ - check:realtime-prune skills:check agent-stream-docs:check; do + check:realtime-prune check:tool-registry-boundary tool-metadata:check \ + integration-catalog:check skills:check agent-stream-docs:check; do ( bun run "$s" >"/tmp/ship-audit-${s//:/-}.log" 2>&1; echo "$? $s" >>/tmp/ship-audit-results ) & done wait @@ -150,4 +156,3 @@ gh pr create --base staging --title "COMMIT_MESSAGE" --body "PR_BODY" - "Tested manually" is acceptable for testing section; include lint, boundary validation, and (when migrations changed) `check:migrations` results when run - Checkboxes filled in appropriately - No screenshots section unless UI changes - diff --git a/.agents/skills/validate-integration/SKILL.md b/.agents/skills/validate-integration/SKILL.md index a0b6dc198c6..fb503813eba 100644 --- a/.agents/skills/validate-integration/SKILL.md +++ b/.agents/skills/validate-integration/SKILL.md @@ -30,6 +30,11 @@ apps/sim/components/icons.tsx # Icon definition apps/sim/lib/auth/auth.ts # OAuth config — should use getCanonicalScopesForProvider() apps/sim/lib/oauth/oauth.ts # OAuth provider config — single source of truth for scopes apps/sim/lib/oauth/utils.ts # Scope utilities, SCOPE_DESCRIPTIONS for modal UI +apps/sim/lib/core/config/env-capabilities.ts # OAuth client runtime capability source of truth +apps/sim/lib/core/config/env.ts # Runtime env schema for capability fields +scripts/setup/capability-config.ts # Exhaustive CLI input-mode mapping for OAuth fields +apps/sim/lib/integrations/integrations.json # Generated client-safe integration catalog +apps/sim/lib/integrations/service-account-metadata.ts # Lightweight service-account projection ``` ## Step 2: Pull API Documentation @@ -233,7 +238,28 @@ Scopes are centralized — the single source of truth is `OAUTH_PROVIDERS` in `l - [ ] Each scope has a human-readable description in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` - [ ] No excess scopes that aren't needed by any tool -## Step 6: Validate Pagination Consistency +## Step 6: Validate Deployment Availability (if OAuth service) + +The deployment UI and setup CLI do not infer OAuth client fields from scopes. They resolve the +block's generated `oauthServiceId` through the application-owned capability catalog. + +- [ ] The visible integration block has exactly one distinct `oauth-input.serviceId` +- [ ] `resolveOAuthClientCapabilityId(serviceId)` returns the intended provider capability +- [ ] The resolved provider exists in `OAUTH_CLIENT_CAPABILITIES` +- [ ] Every field listed by that capability exists in `apps/sim/lib/core/config/env.ts` +- [ ] Every capability field has the correct `text` or `secret` entry in `OAUTH_CLIENT_SETUP_FIELDS`; no CLI naming heuristic is required +- [ ] Shared Google/Microsoft service IDs resolve to their provider capability rather than duplicate entries +- [ ] `bun run setup integration ` is the command emitted by availability; the CLI has only the exhaustive input-mode projection, not a second runtime provider definition +- [ ] If the canonical OAuth service declares `serviceAccountProviderId`, + `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID[serviceId]` has the same provider ID +- [ ] The service-account `deploymentRequirement` matches how that credential actually works: + omitted for an independent path, `'oauth-client'` when it needs the OAuth client fields, or + `'preview-gated'` when controlled by a preview block + +Treat a missing capability as **critical**: runtime availability intentionally throws instead of +silently exposing an unusable integration. + +## Step 7: Validate Pagination Consistency If any tools support pagination: - [ ] Pagination param names match the API docs (e.g., `pagination_token` vs `next_token` vs `cursor`) @@ -241,7 +267,7 @@ If any tools support pagination: - [ ] Pagination response fields (`nextToken`, `cursor`, etc.) are included in tool outputs - [ ] Pagination subBlocks are set to `mode: 'advanced'` -## Step 7: Validate Memory Load Safety +## Step 8: Validate Memory Load Safety If any tool lists, searches, exports, imports, downloads, uploads, paginates, batches, transforms arrays, or reads file/HTTP bodies, read `.agents/skills/memory-load-check/SKILL.md` and apply it to the integration. @@ -251,13 +277,13 @@ If any tool lists, searches, exports, imports, downloads, uploads, paginates, ba - [ ] Large result payloads are summarized, paginated, referenced, or capped rather than raw-dumped - [ ] Pagination and download tests cover caps, early stop behavior, or partial-result preservation when relevant -## Step 8: Validate Error Handling +## Step 9: Validate Error Handling - [ ] `transformResponse` checks for error conditions before accessing data - [ ] Error responses include meaningful messages (not just generic "failed") - [ ] HTTP error status codes are handled (check `response.ok` or status codes) -## Step 9: Report and Fix +## Step 10: Report and Fix ### Report Format @@ -270,6 +296,9 @@ Group findings by severity: - Missing error handling that would cause crashes - Tool ID mismatch between tool file, registry, and block `tools.access` - OAuth scopes missing in `auth.ts` that tools need +- OAuth integration `serviceId` missing from the deployment capability catalog +- Capability references an env field absent from the runtime env schema +- Service-account metadata disagrees with the canonical OAuth service configuration - `tools.config.tool` returning wrong tool ID for an operation - Type coercions in `tools.config.tool` instead of `tools.config.params` @@ -301,11 +330,15 @@ Several files are generated from tool and block definitions. Editing a tool or b ```bash bun run tool-metadata:generate # repo root — apps/sim/tools/generated/* -cd apps/sim && bun run generate-docs # docs .mdx + lib/integrations/integrations.json + docs icons +bun run scripts/generate-docs.ts # docs .mdx + lib/integrations/integrations.json + docs icons +bun run integration-catalog:check # registry ↔ committed deployment metadata drift ``` - **`tool-metadata:generate`** — required whenever a tool's `outputs`, `params`, or descriptions change. CI enforces this with `bun run tool-metadata:check`, which fails with *"Generated tool metadata is stale"*. This is the easiest gate to miss, because nothing in the tool file hints that a generated artifact mirrors it. - **`generate-docs`** — required whenever block metadata changes (`bgColor`, `name`, `description`, operations, outputs). Regenerates the integration `.mdx`, `integrations.json`, and the docs copy of `components/icons.tsx`. +- **`integration-catalog:check`** — loads the executable block registry, derives visible integration + deployment fields, and compares them with the committed catalog. It catches missing/unexpected + entries and stale auth/service IDs without loading the executable registry in client code. **Always diff the regen output before committing.** These generators rewrite every file they own, so they will also sweep in unrelated drift that accumulated on the base branch — pages losing sections, unrelated icons appearing. Keep only the hunks belonging to the integration under validation and `git checkout --` the rest, otherwise an unrelated doc regression rides along in the PR. Verify no page was silently dropped by comparing the directory listing before and after. @@ -318,8 +351,10 @@ After fixing, confirm: 2. TypeScript compiles clean (no type errors) — check the error list is empty for the files you touched; pre-existing unrelated errors in a worktree usually mean workspace packages resolve to the main checkout 3. The integration's tests pass, and any test you added actually fails without its fix (revert it once and watch it go red) 4. Derived artifacts regenerated and their diffs reviewed (see above) -5. Re-read all modified files to verify fixes are correct -6. Any remaining unknown response schemas were explicitly reported to the user instead of guessed +5. `bun run integration-catalog:check` passes +6. For OAuth or service-account changes, `bun test apps/sim/lib/integrations/availability.server.test.ts` passes +7. Re-read all modified files to verify fixes are correct +8. Any remaining unknown response schemas were explicitly reported to the user instead of guessed ## Checklist Summary @@ -333,6 +368,9 @@ After fixing, confirm: - [ ] Validated block outputs match what tools return, with typed JSON where possible - [ ] Validated OAuth scopes use centralized utilities (getScopesForService, getCanonicalScopesForProvider) — no hardcoded arrays - [ ] Validated scope descriptions exist in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` for all scopes +- [ ] Validated OAuth `serviceId` resolves to the intended `OAUTH_CLIENT_CAPABILITIES` entry and all capability fields exist in the env schema +- [ ] Validated service-account projection and deployment requirement against the canonical OAuth service config +- [ ] Regenerated `integrations.json` when block metadata changed and ran `bun run integration-catalog:check` - [ ] Validated pagination consistency across tools and block - [ ] Validated memory load safety using `.agents/skills/memory-load-check/SKILL.md` when tools list/search/download/import/export/batch data - [ ] Validated error handling (error checks, meaningful messages) diff --git a/.claude/commands/add-block.md b/.claude/commands/add-block.md index bcc57d1ef48..57df2b4dd1b 100644 --- a/.claude/commands/add-block.md +++ b/.claude/commands/add-block.md @@ -162,6 +162,32 @@ export const {ServiceName}Block: BlockConfig = { Optional companions: `credentialLabels` (override the picker's section/connect-row copy) and `allowServiceAccounts: true` (trigger-mode only — list service accounts, which triggers otherwise exclude; set only when the trigger's polling path can resolve a service-account token). The connect modal, provider families (Google JSON key, Atlassian token, token-paste, client-credential, Slack bot), and the preview gate are all resolved from `serviceAccountProviderId` — you don't wire them per block. +### OAuth deployment availability (required for integration blocks) + +A visible tools-category block with OAuth is deployment-gated. Its `oauth-input.serviceId` is +projected into `apps/sim/lib/integrations/integrations.json`, then resolved through +`resolveOAuthClientCapabilityId()` in `apps/sim/lib/core/config/env-capabilities.ts`. + +When adding or changing an OAuth integration block: + +1. Keep exactly one distinct OAuth `serviceId` across the block's `oauth-input` subBlocks. +2. Confirm that service ID resolves to an entry in `OAUTH_CLIENT_CAPABILITIES`. Google and + Microsoft service IDs intentionally share their provider-level capability; do not add duplicate + entries for those aliases. +3. For a new capability, add its required client fields to `OAUTH_CLIENT_CAPABILITIES` and ensure + every referenced field exists in the env schema in `apps/sim/lib/core/config/env.ts`. Then add + the matching `text` or `secret` input modes to `OAUTH_CLIENT_SETUP_FIELDS` in + `scripts/setup/capability-config.ts`. The CLI catalog is exhaustively typed and checked against + the runtime field list; do not infer secrecy from the field name. +4. If the canonical OAuth service declares `serviceAccountProviderId`, keep + `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` in + `apps/sim/lib/integrations/service-account-metadata.ts` aligned. Set + `deploymentRequirement` only when the service-account path is preview-gated or depends on the + OAuth client fields; otherwise omit it. + +Missing capability metadata is a runtime configuration error, not a reason to make the integration +silently available. + ### Selectors (with dynamic options) ```typescript // Channel selector (Slack, Discord, etc.) @@ -918,12 +944,25 @@ Derive templates from the service's real use cases. Each prompt should name a co - **Ground every skill in operations the block actually exposes** — cross-check each skill's steps against `tools.access`. Never describe an action the integration cannot perform. - **Derive skills from real, popular use cases found online — never invent them.** Web-search the service's documented use cases (vendor use-case/solutions pages, official docs describing the workflow, reputable "top automations for X" articles) and only add a skill you can source as something people genuinely do with the service. Do not hallucinate skills. -## Generated tool metadata +## Generated artifacts -Adding a block on its own needs **no** regeneration — a block references existing tool IDs through `tools.access` and does not change any tool's shape. +Adding a block on its own needs no **tool metadata** regeneration — a block references existing +tool IDs through `tools.access` and does not change any tool's shape. But if the same change also adds, edits **or removes** a tool, run `bun run tool-metadata:generate` and commit the result, or CI fails on stale artifacts. That matters here because a block's `outputs` are authored to match its tools' outputs, and the UI now reads those from the generated metadata rather than the executable registry — an unregenerated tool change makes the block's outputs disagree with what the panel renders. See `.agents/skills/tool-registry-boundary/SKILL.md`. +A visible integration block does require the generated integration catalog and docs to be refreshed. +After adding or changing one, run: + +```bash +bun run scripts/generate-docs.ts +bun run integration-catalog:check +``` + +The catalog check independently derives deployment metadata from the executable block registry and +compares it with the committed `apps/sim/lib/integrations/integrations.json`. Review the generated +diff and keep only intentional changes. + ## Checklist Before Finishing - [ ] `integrationType` is set to the correct `IntegrationType` enum value @@ -933,12 +972,17 @@ But if the same change also adds, edits **or removes** a tool, run `bun run tool - [ ] DependsOn set for fields that need other values - [ ] Required fields marked correctly (boolean or condition) - [ ] OAuth inputs have correct `serviceId` and `requiredScopes: getScopesForService(serviceId)` +- [ ] Every OAuth `serviceId` resolves through `resolveOAuthClientCapabilityId()` to the correct `OAUTH_CLIENT_CAPABILITIES` entry +- [ ] Any new OAuth capability fields exist in `apps/sim/lib/core/config/env.ts` +- [ ] If the OAuth service supports service accounts, `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` matches its canonical `serviceAccountProviderId` and deployment requirement - [ ] Scope descriptions added to `SCOPE_DESCRIPTIONS` in `lib/oauth/utils.ts` for any new scopes - [ ] Tools.access lists all tool IDs (snake_case) - [ ] Tools.config.tool returns correct tool ID (snake_case) - [ ] Outputs match tool outputs - [ ] Block + meta registered in registry-maps.ts (`BLOCK_REGISTRY` / `BLOCK_META_REGISTRY`) - [ ] If any tool was added, changed or removed alongside the block: ran `bun run tool-metadata:generate` and committed the artifacts +- [ ] Ran `bun run scripts/generate-docs.ts`, reviewed the generated diff, and committed the integration catalog changes +- [ ] `bun run integration-catalog:check` passes - [ ] If icon missing: asked user to provide SVG - [ ] If triggers exist: `triggers` config set, trigger subBlocks spread - [ ] Optional/rarely-used fields set to `mode: 'advanced'` diff --git a/.claude/commands/add-integration.md b/.claude/commands/add-integration.md index 02aeb75c545..8df06ac1771 100644 --- a/.claude/commands/add-integration.md +++ b/.claude/commands/add-integration.md @@ -16,7 +16,8 @@ Adding an integration involves these steps in order: 4. **Add Icon** - Add the service's brand icon 5. **Create Triggers** (optional) - If the service supports webhooks 6. **Register** - Register tools, block, and triggers in their registries -7. **Generate Docs** - Run the docs generation script +7. **Configure Deployment Availability** - Wire OAuth client and service-account metadata +8. **Generate and Validate the Catalog** - Regenerate docs/catalog artifacts and run drift checks ## Step 1: Research the API @@ -464,15 +465,48 @@ export const TRIGGER_REGISTRY: TriggerRegistry = { } ``` -## Step 7: Generate Docs +## Step 7: Configure Deployment Availability + +Do this for every visible OAuth integration. API-key and unauthenticated integrations do not need +an OAuth client capability. + +The block's `oauth-input.serviceId` is the canonical link between the generated integration catalog, +the OAuth service configuration, deployment availability, and the setup CLI. + +1. Ensure the block has exactly one distinct OAuth `serviceId` and that it matches the canonical + service entry in `apps/sim/lib/oauth/oauth.ts`. +2. Confirm `resolveOAuthClientCapabilityId(serviceId)` resolves to the intended provider entry in + `OAUTH_CLIENT_CAPABILITIES` in `apps/sim/lib/core/config/env-capabilities.ts`. Google and + Microsoft service IDs deliberately share provider-level capabilities. +3. For a new OAuth provider, add the required client fields to `OAUTH_CLIENT_CAPABILITIES`, add + every referenced field to the env schema in `apps/sim/lib/core/config/env.ts`, and add the + matching `text` or `secret` entries to `OAUTH_CLIENT_SETUP_FIELDS` in + `scripts/setup/capability-config.ts`. Do not create integration-specific setup logic or infer + secret fields from naming; the CLI mapping is exhaustively checked against the runtime fields. +4. If the canonical OAuth service has `serviceAccountProviderId`, add the matching projection to + `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` in + `apps/sim/lib/integrations/service-account-metadata.ts`. Use: + - no `deploymentRequirement` when the service-account path works independently of OAuth client fields; + - `'oauth-client'` when it requires the same deployment OAuth client fields; + - `'preview-gated'` when availability is controlled by the service-account preview block. + +Never add a permissive fallback for missing capability metadata. A visible OAuth integration without +a resolvable capability must fail validation. + +## Step 8: Generate and Validate the Catalog Run the documentation generator: ```bash bun run scripts/generate-docs.ts +bun run integration-catalog:check ``` This creates `apps/docs/content/docs/en/integrations/{service}.mdx` — one page per service carrying the block's Actions and, if it has one, its Triggers section. Never hand-edit generated pages; the only editable region is the `{/* MANUAL-CONTENT */}` block (see `scripts/README.md`). +The same generator refreshes `apps/sim/lib/integrations/integrations.json`. The catalog check then +derives the deployment-relevant fields from the executable block registry and compares them with the +committed projection. Review the generated diff and keep only intentional changes. + ## V2 Integration Pattern If creating V2 versions (API-aligned outputs): @@ -523,6 +557,13 @@ If creating V2 versions (API-aligned outputs): - [ ] Used `getCanonicalScopesForProvider()` in `auth.ts` (never hardcode) - [ ] Used `getScopesForService()` in block `requiredScopes` (never hardcode) +### Deployment Availability (if OAuth service) +- [ ] Block declares exactly one distinct `oauth-input.serviceId` +- [ ] `resolveOAuthClientCapabilityId(serviceId)` resolves to the intended `OAUTH_CLIENT_CAPABILITIES` entry +- [ ] Every new OAuth capability field exists in `apps/sim/lib/core/config/env.ts` +- [ ] Runtime OAuth fields live in `OAUTH_CLIENT_CAPABILITIES`; matching CLI input modes live in the exhaustively checked `OAUTH_CLIENT_SETUP_FIELDS` +- [ ] If `serviceAccountProviderId` is configured, `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` has the matching projection and deployment requirement + ### Icon - [ ] Asked user to provide SVG - [ ] Added icon to `components/icons.tsx` @@ -541,6 +582,8 @@ If creating V2 versions (API-aligned outputs): ### Docs - [ ] Ran `bun run scripts/generate-docs.ts` - [ ] Verified docs file created +- [ ] Reviewed and committed the generated `apps/sim/lib/integrations/integrations.json` change +- [ ] `bun run integration-catalog:check` passes ### Final Validation (Required) - [ ] Read every tool file and cross-referenced inputs/outputs against the API docs @@ -885,3 +928,5 @@ requiredScopes: getScopesForService('{service}'), 10. **Complex inputs need wandConfig** - Timestamps, JSON arrays, and other hard-to-type values should have `wandConfig` enabled 11. **Never hardcode scopes** - Use `getScopesForService()` in blocks and `getCanonicalScopesForProvider()` in auth.ts 12. **Always add scope descriptions** - New scopes must have entries in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` +13. **OAuth service IDs need deployment capabilities** - Every visible OAuth integration must resolve through `OAUTH_CLIENT_CAPABILITIES`; shared Google/Microsoft aliases map to their provider capability +14. **Keep runtime and presentation separate** - Runtime OAuth fields live in `env-capabilities.ts`; CLI input modes live in the exhaustively checked `scripts/setup/capability-config.ts` mapping diff --git a/.claude/commands/ship.md b/.claude/commands/ship.md index fdd40c011e6..326abe9fcb3 100644 --- a/.claude/commands/ship.md +++ b/.claude/commands/ship.md @@ -53,17 +53,23 @@ When the user runs `/ship`: ``` Then `git status --short` to see what regenerated — those files must be staged in step 7 alongside your own changes. - **Do NOT blanket-run the domain generators here.** `mship:generate` (`generate-mship-contracts.ts`) is an **umbrella** that drives all nine mothership contract generators (`mship-contracts`, `billing-protocol-contract`, `mship-tools`, the four `trace-*`, `metrics-contract`, `vfs-snapshot-contract`) and biome-formats `apps/sim/lib/copilot/generated/` — never run it *and* its constituents (they write the same files and corrupt each other in parallel), and never run it on an ordinary ship: it reads an **external** copilot-contract source that isn't checked out in most worktrees, so it hard-fails with `ENOENT` and would abort ship for an unrelated reason. `generate:pi-model-catalog` (under `apps/sim`) likewise regenerates from the installed Pi package, not repo source. Only when **this PR's diff actually touches** a domain generator's input do you regenerate it deliberately and run its matching `:check` (`bun run mship:check` / the individual `*:check`) — with the external source present. + **Do NOT blanket-run the domain generators here.** `mship:generate` (`generate-mship-contracts.ts`) is an **umbrella** that drives all nine mothership contract generators (`mship-contracts`, `billing-protocol-contract`, `mship-tools`, the four `trace-*`, `metrics-contract`, `vfs-snapshot-contract`) and biome-formats `apps/sim/lib/copilot/generated/` — never run it *and* its constituents (they write the same files and corrupt each other in parallel), and never run it on an ordinary ship: it reads an **external** copilot-contract source that isn't checked out in most worktrees, so it hard-fails with `ENOENT` and would abort ship for an unrelated reason. `generate:pi-model-catalog` (under `apps/sim`) likewise regenerates from the installed Pi package, not repo source. `scripts/generate-docs.ts` rewrites the integration docs and client-safe catalog; run it when this PR changes their block/icon/landing-content inputs or when `integration-catalog:check` reports drift, then review its broad generated diff. Only when **this PR's diff actually touches** a domain generator's input do you regenerate it deliberately and run its matching `:check` (`bun run mship:check` / the individual `*:check`) — with the external source present. - **Phase B — run lint + every audit CI enforces, in parallel, and abort ship if any fails.** `bun run lint` first (it autofixes formatting and mutates files, so don't parallelize it with the read-only audits), then fan the rest out and collect exit codes. This is exactly the read-only audit set from CI's `Lint and Test` job (all in-repo, runnable in any worktree): + **Phase B — run lint + every audit CI enforces, in parallel, and abort ship if any fails.** Before running the commands, compare this list with `.github/workflows/test-build.yml`; when CI adds an audit, run it and update this skill instead of trusting a stale snapshot. The env-flag audit is currently an inline workflow block rather than a package script: when `apps/sim/lib/core/config/env-flags.ts` changed, run that current workflow block verbatim instead of copying a second version into this skill. Run `bun run lint` first (it autofixes formatting and mutates files, so don't parallelize it with the read-only audits), then run the base-sensitive block-registry check, then fan the independent audits out and collect exit codes: ```bash # autofix formatting first (mutating; not parallel-safe with the audits). Gate its exit too — # a non-zero lint (unfixable errors) must abort before the audits run, not be ignored. bun run lint || { echo "❌ lint failed — do not ship"; exit 1; } + bun run apps/sim/scripts/check-block-registry.ts origin/staging || { + echo "❌ block registry audit failed — do not ship" + exit 1 + } rm -f /tmp/ship-audit-results - for s in check:boundaries check:api-validation:strict check:utils check:zustand-v5 \ + for s in check:boundaries check:api-validation:strict check:desktop-bridge check:desktop-ipc \ + check:utils check:zustand-v5 \ check:react-query check:client-boundary check:bare-icons check:icon-paths \ - check:realtime-prune skills:check agent-stream-docs:check; do + check:realtime-prune check:tool-registry-boundary tool-metadata:check \ + integration-catalog:check skills:check agent-stream-docs:check; do ( bun run "$s" >"/tmp/ship-audit-${s//:/-}.log" 2>&1; echo "$? $s" >>/tmp/ship-audit-results ) & done wait diff --git a/.claude/commands/validate-integration.md b/.claude/commands/validate-integration.md index b243e5c1963..540da61cd4b 100644 --- a/.claude/commands/validate-integration.md +++ b/.claude/commands/validate-integration.md @@ -29,6 +29,11 @@ apps/sim/components/icons.tsx # Icon definition apps/sim/lib/auth/auth.ts # OAuth config — should use getCanonicalScopesForProvider() apps/sim/lib/oauth/oauth.ts # OAuth provider config — single source of truth for scopes apps/sim/lib/oauth/utils.ts # Scope utilities, SCOPE_DESCRIPTIONS for modal UI +apps/sim/lib/core/config/env-capabilities.ts # OAuth client runtime capability source of truth +apps/sim/lib/core/config/env.ts # Runtime env schema for capability fields +scripts/setup/capability-config.ts # Exhaustive CLI input-mode mapping for OAuth fields +apps/sim/lib/integrations/integrations.json # Generated client-safe integration catalog +apps/sim/lib/integrations/service-account-metadata.ts # Lightweight service-account projection ``` ## Step 2: Pull API Documentation @@ -232,7 +237,28 @@ Scopes are centralized — the single source of truth is `OAUTH_PROVIDERS` in `l - [ ] Each scope has a human-readable description in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` - [ ] No excess scopes that aren't needed by any tool -## Step 6: Validate Pagination Consistency +## Step 6: Validate Deployment Availability (if OAuth service) + +The deployment UI and setup CLI do not infer OAuth client fields from scopes. They resolve the +block's generated `oauthServiceId` through the application-owned capability catalog. + +- [ ] The visible integration block has exactly one distinct `oauth-input.serviceId` +- [ ] `resolveOAuthClientCapabilityId(serviceId)` returns the intended provider capability +- [ ] The resolved provider exists in `OAUTH_CLIENT_CAPABILITIES` +- [ ] Every field listed by that capability exists in `apps/sim/lib/core/config/env.ts` +- [ ] Every capability field has the correct `text` or `secret` entry in `OAUTH_CLIENT_SETUP_FIELDS`; no CLI naming heuristic is required +- [ ] Shared Google/Microsoft service IDs resolve to their provider capability rather than duplicate entries +- [ ] `bun run setup integration ` is the command emitted by availability; the CLI has only the exhaustive input-mode projection, not a second runtime provider definition +- [ ] If the canonical OAuth service declares `serviceAccountProviderId`, + `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID[serviceId]` has the same provider ID +- [ ] The service-account `deploymentRequirement` matches how that credential actually works: + omitted for an independent path, `'oauth-client'` when it needs the OAuth client fields, or + `'preview-gated'` when controlled by a preview block + +Treat a missing capability as **critical**: runtime availability intentionally throws instead of +silently exposing an unusable integration. + +## Step 7: Validate Pagination Consistency If any tools support pagination: - [ ] Pagination param names match the API docs (e.g., `pagination_token` vs `next_token` vs `cursor`) @@ -240,7 +266,7 @@ If any tools support pagination: - [ ] Pagination response fields (`nextToken`, `cursor`, etc.) are included in tool outputs - [ ] Pagination subBlocks are set to `mode: 'advanced'` -## Step 7: Validate Memory Load Safety +## Step 8: Validate Memory Load Safety If any tool lists, searches, exports, imports, downloads, uploads, paginates, batches, transforms arrays, or reads file/HTTP bodies, read `.agents/skills/memory-load-check/SKILL.md` and apply it to the integration. @@ -250,13 +276,13 @@ If any tool lists, searches, exports, imports, downloads, uploads, paginates, ba - [ ] Large result payloads are summarized, paginated, referenced, or capped rather than raw-dumped - [ ] Pagination and download tests cover caps, early stop behavior, or partial-result preservation when relevant -## Step 8: Validate Error Handling +## Step 9: Validate Error Handling - [ ] `transformResponse` checks for error conditions before accessing data - [ ] Error responses include meaningful messages (not just generic "failed") - [ ] HTTP error status codes are handled (check `response.ok` or status codes) -## Step 9: Report and Fix +## Step 10: Report and Fix ### Report Format @@ -269,6 +295,9 @@ Group findings by severity: - Missing error handling that would cause crashes - Tool ID mismatch between tool file, registry, and block `tools.access` - OAuth scopes missing in `auth.ts` that tools need +- OAuth integration `serviceId` missing from the deployment capability catalog +- Capability references an env field absent from the runtime env schema +- Service-account metadata disagrees with the canonical OAuth service configuration - `tools.config.tool` returning wrong tool ID for an operation - Type coercions in `tools.config.tool` instead of `tools.config.params` @@ -300,11 +329,15 @@ Several files are generated from tool and block definitions. Editing a tool or b ```bash bun run tool-metadata:generate # repo root — apps/sim/tools/generated/* -cd apps/sim && bun run generate-docs # docs .mdx + lib/integrations/integrations.json + docs icons +bun run scripts/generate-docs.ts # docs .mdx + lib/integrations/integrations.json + docs icons +bun run integration-catalog:check # registry ↔ committed deployment metadata drift ``` - **`tool-metadata:generate`** — required whenever a tool's `outputs`, `params`, or descriptions change. CI enforces this with `bun run tool-metadata:check`, which fails with *"Generated tool metadata is stale"*. This is the easiest gate to miss, because nothing in the tool file hints that a generated artifact mirrors it. - **`generate-docs`** — required whenever block metadata changes (`bgColor`, `name`, `description`, operations, outputs). Regenerates the integration `.mdx`, `integrations.json`, and the docs copy of `components/icons.tsx`. +- **`integration-catalog:check`** — loads the executable block registry, derives visible integration + deployment fields, and compares them with the committed catalog. It catches missing/unexpected + entries and stale auth/service IDs without loading the executable registry in client code. **Always diff the regen output before committing.** These generators rewrite every file they own, so they will also sweep in unrelated drift that accumulated on the base branch — pages losing sections, unrelated icons appearing. Keep only the hunks belonging to the integration under validation and `git checkout --` the rest, otherwise an unrelated doc regression rides along in the PR. Verify no page was silently dropped by comparing the directory listing before and after. @@ -317,8 +350,10 @@ After fixing, confirm: 2. TypeScript compiles clean (no type errors) — check the error list is empty for the files you touched; pre-existing unrelated errors in a worktree usually mean workspace packages resolve to the main checkout 3. The integration's tests pass, and any test you added actually fails without its fix (revert it once and watch it go red) 4. Derived artifacts regenerated and their diffs reviewed (see above) -5. Re-read all modified files to verify fixes are correct -6. Any remaining unknown response schemas were explicitly reported to the user instead of guessed +5. `bun run integration-catalog:check` passes +6. For OAuth or service-account changes, `bun test apps/sim/lib/integrations/availability.server.test.ts` passes +7. Re-read all modified files to verify fixes are correct +8. Any remaining unknown response schemas were explicitly reported to the user instead of guessed ## Checklist Summary @@ -332,6 +367,9 @@ After fixing, confirm: - [ ] Validated block outputs match what tools return, with typed JSON where possible - [ ] Validated OAuth scopes use centralized utilities (getScopesForService, getCanonicalScopesForProvider) — no hardcoded arrays - [ ] Validated scope descriptions exist in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` for all scopes +- [ ] Validated OAuth `serviceId` resolves to the intended `OAUTH_CLIENT_CAPABILITIES` entry and all capability fields exist in the env schema +- [ ] Validated service-account projection and deployment requirement against the canonical OAuth service config +- [ ] Regenerated `integrations.json` when block metadata changed and ran `bun run integration-catalog:check` - [ ] Validated pagination consistency across tools and block - [ ] Validated memory load safety using `.agents/skills/memory-load-check/SKILL.md` when tools list/search/download/import/export/batch data - [ ] Validated error handling (error checks, meaningful messages) diff --git a/.cursor/commands/add-block.md b/.cursor/commands/add-block.md index e4776c63147..1ac378de897 100644 --- a/.cursor/commands/add-block.md +++ b/.cursor/commands/add-block.md @@ -157,6 +157,32 @@ export const {ServiceName}Block: BlockConfig = { Optional companions: `credentialLabels` (override the picker's section/connect-row copy) and `allowServiceAccounts: true` (trigger-mode only — list service accounts, which triggers otherwise exclude; set only when the trigger's polling path can resolve a service-account token). The connect modal, provider families (Google JSON key, Atlassian token, token-paste, client-credential, Slack bot), and the preview gate are all resolved from `serviceAccountProviderId` — you don't wire them per block. +### OAuth deployment availability (required for integration blocks) + +A visible tools-category block with OAuth is deployment-gated. Its `oauth-input.serviceId` is +projected into `apps/sim/lib/integrations/integrations.json`, then resolved through +`resolveOAuthClientCapabilityId()` in `apps/sim/lib/core/config/env-capabilities.ts`. + +When adding or changing an OAuth integration block: + +1. Keep exactly one distinct OAuth `serviceId` across the block's `oauth-input` subBlocks. +2. Confirm that service ID resolves to an entry in `OAUTH_CLIENT_CAPABILITIES`. Google and + Microsoft service IDs intentionally share their provider-level capability; do not add duplicate + entries for those aliases. +3. For a new capability, add its required client fields to `OAUTH_CLIENT_CAPABILITIES` and ensure + every referenced field exists in the env schema in `apps/sim/lib/core/config/env.ts`. Then add + the matching `text` or `secret` input modes to `OAUTH_CLIENT_SETUP_FIELDS` in + `scripts/setup/capability-config.ts`. The CLI catalog is exhaustively typed and checked against + the runtime field list; do not infer secrecy from the field name. +4. If the canonical OAuth service declares `serviceAccountProviderId`, keep + `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` in + `apps/sim/lib/integrations/service-account-metadata.ts` aligned. Set + `deploymentRequirement` only when the service-account path is preview-gated or depends on the + OAuth client fields; otherwise omit it. + +Missing capability metadata is a runtime configuration error, not a reason to make the integration +silently available. + ### Selectors (with dynamic options) ```typescript // Channel selector (Slack, Discord, etc.) @@ -913,12 +939,25 @@ Derive templates from the service's real use cases. Each prompt should name a co - **Ground every skill in operations the block actually exposes** — cross-check each skill's steps against `tools.access`. Never describe an action the integration cannot perform. - **Derive skills from real, popular use cases found online — never invent them.** Web-search the service's documented use cases (vendor use-case/solutions pages, official docs describing the workflow, reputable "top automations for X" articles) and only add a skill you can source as something people genuinely do with the service. Do not hallucinate skills. -## Generated tool metadata +## Generated artifacts -Adding a block on its own needs **no** regeneration — a block references existing tool IDs through `tools.access` and does not change any tool's shape. +Adding a block on its own needs no **tool metadata** regeneration — a block references existing +tool IDs through `tools.access` and does not change any tool's shape. But if the same change also adds, edits **or removes** a tool, run `bun run tool-metadata:generate` and commit the result, or CI fails on stale artifacts. That matters here because a block's `outputs` are authored to match its tools' outputs, and the UI now reads those from the generated metadata rather than the executable registry — an unregenerated tool change makes the block's outputs disagree with what the panel renders. See `.agents/skills/tool-registry-boundary/SKILL.md`. +A visible integration block does require the generated integration catalog and docs to be refreshed. +After adding or changing one, run: + +```bash +bun run scripts/generate-docs.ts +bun run integration-catalog:check +``` + +The catalog check independently derives deployment metadata from the executable block registry and +compares it with the committed `apps/sim/lib/integrations/integrations.json`. Review the generated +diff and keep only intentional changes. + ## Checklist Before Finishing - [ ] `integrationType` is set to the correct `IntegrationType` enum value @@ -928,12 +967,17 @@ But if the same change also adds, edits **or removes** a tool, run `bun run tool - [ ] DependsOn set for fields that need other values - [ ] Required fields marked correctly (boolean or condition) - [ ] OAuth inputs have correct `serviceId` and `requiredScopes: getScopesForService(serviceId)` +- [ ] Every OAuth `serviceId` resolves through `resolveOAuthClientCapabilityId()` to the correct `OAUTH_CLIENT_CAPABILITIES` entry +- [ ] Any new OAuth capability fields exist in `apps/sim/lib/core/config/env.ts` +- [ ] If the OAuth service supports service accounts, `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` matches its canonical `serviceAccountProviderId` and deployment requirement - [ ] Scope descriptions added to `SCOPE_DESCRIPTIONS` in `lib/oauth/utils.ts` for any new scopes - [ ] Tools.access lists all tool IDs (snake_case) - [ ] Tools.config.tool returns correct tool ID (snake_case) - [ ] Outputs match tool outputs - [ ] Block + meta registered in registry-maps.ts (`BLOCK_REGISTRY` / `BLOCK_META_REGISTRY`) - [ ] If any tool was added, changed or removed alongside the block: ran `bun run tool-metadata:generate` and committed the artifacts +- [ ] Ran `bun run scripts/generate-docs.ts`, reviewed the generated diff, and committed the integration catalog changes +- [ ] `bun run integration-catalog:check` passes - [ ] If icon missing: asked user to provide SVG - [ ] If triggers exist: `triggers` config set, trigger subBlocks spread - [ ] Optional/rarely-used fields set to `mode: 'advanced'` diff --git a/.cursor/commands/add-integration.md b/.cursor/commands/add-integration.md index d81aa6e2fd0..47c3ae3f0d0 100644 --- a/.cursor/commands/add-integration.md +++ b/.cursor/commands/add-integration.md @@ -11,7 +11,8 @@ Adding an integration involves these steps in order: 4. **Add Icon** - Add the service's brand icon 5. **Create Triggers** (optional) - If the service supports webhooks 6. **Register** - Register tools, block, and triggers in their registries -7. **Generate Docs** - Run the docs generation script +7. **Configure Deployment Availability** - Wire OAuth client and service-account metadata +8. **Generate and Validate the Catalog** - Regenerate docs/catalog artifacts and run drift checks ## Step 1: Research the API @@ -459,15 +460,48 @@ export const TRIGGER_REGISTRY: TriggerRegistry = { } ``` -## Step 7: Generate Docs +## Step 7: Configure Deployment Availability + +Do this for every visible OAuth integration. API-key and unauthenticated integrations do not need +an OAuth client capability. + +The block's `oauth-input.serviceId` is the canonical link between the generated integration catalog, +the OAuth service configuration, deployment availability, and the setup CLI. + +1. Ensure the block has exactly one distinct OAuth `serviceId` and that it matches the canonical + service entry in `apps/sim/lib/oauth/oauth.ts`. +2. Confirm `resolveOAuthClientCapabilityId(serviceId)` resolves to the intended provider entry in + `OAUTH_CLIENT_CAPABILITIES` in `apps/sim/lib/core/config/env-capabilities.ts`. Google and + Microsoft service IDs deliberately share provider-level capabilities. +3. For a new OAuth provider, add the required client fields to `OAUTH_CLIENT_CAPABILITIES`, add + every referenced field to the env schema in `apps/sim/lib/core/config/env.ts`, and add the + matching `text` or `secret` entries to `OAUTH_CLIENT_SETUP_FIELDS` in + `scripts/setup/capability-config.ts`. Do not create integration-specific setup logic or infer + secret fields from naming; the CLI mapping is exhaustively checked against the runtime fields. +4. If the canonical OAuth service has `serviceAccountProviderId`, add the matching projection to + `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` in + `apps/sim/lib/integrations/service-account-metadata.ts`. Use: + - no `deploymentRequirement` when the service-account path works independently of OAuth client fields; + - `'oauth-client'` when it requires the same deployment OAuth client fields; + - `'preview-gated'` when availability is controlled by the service-account preview block. + +Never add a permissive fallback for missing capability metadata. A visible OAuth integration without +a resolvable capability must fail validation. + +## Step 8: Generate and Validate the Catalog Run the documentation generator: ```bash bun run scripts/generate-docs.ts +bun run integration-catalog:check ``` This creates `apps/docs/content/docs/en/integrations/{service}.mdx` — one page per service carrying the block's Actions and, if it has one, its Triggers section. Never hand-edit generated pages; the only editable region is the `{/* MANUAL-CONTENT */}` block (see `scripts/README.md`). +The same generator refreshes `apps/sim/lib/integrations/integrations.json`. The catalog check then +derives the deployment-relevant fields from the executable block registry and compares them with the +committed projection. Review the generated diff and keep only intentional changes. + ## V2 Integration Pattern If creating V2 versions (API-aligned outputs): @@ -518,6 +552,13 @@ If creating V2 versions (API-aligned outputs): - [ ] Used `getCanonicalScopesForProvider()` in `auth.ts` (never hardcode) - [ ] Used `getScopesForService()` in block `requiredScopes` (never hardcode) +### Deployment Availability (if OAuth service) +- [ ] Block declares exactly one distinct `oauth-input.serviceId` +- [ ] `resolveOAuthClientCapabilityId(serviceId)` resolves to the intended `OAUTH_CLIENT_CAPABILITIES` entry +- [ ] Every new OAuth capability field exists in `apps/sim/lib/core/config/env.ts` +- [ ] Runtime OAuth fields live in `OAUTH_CLIENT_CAPABILITIES`; matching CLI input modes live in the exhaustively checked `OAUTH_CLIENT_SETUP_FIELDS` +- [ ] If `serviceAccountProviderId` is configured, `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID` has the matching projection and deployment requirement + ### Icon - [ ] Asked user to provide SVG - [ ] Added icon to `components/icons.tsx` @@ -536,6 +577,8 @@ If creating V2 versions (API-aligned outputs): ### Docs - [ ] Ran `bun run scripts/generate-docs.ts` - [ ] Verified docs file created +- [ ] Reviewed and committed the generated `apps/sim/lib/integrations/integrations.json` change +- [ ] `bun run integration-catalog:check` passes ### Final Validation (Required) - [ ] Read every tool file and cross-referenced inputs/outputs against the API docs @@ -880,3 +923,5 @@ requiredScopes: getScopesForService('{service}'), 10. **Complex inputs need wandConfig** - Timestamps, JSON arrays, and other hard-to-type values should have `wandConfig` enabled 11. **Never hardcode scopes** - Use `getScopesForService()` in blocks and `getCanonicalScopesForProvider()` in auth.ts 12. **Always add scope descriptions** - New scopes must have entries in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` +13. **OAuth service IDs need deployment capabilities** - Every visible OAuth integration must resolve through `OAUTH_CLIENT_CAPABILITIES`; shared Google/Microsoft aliases map to their provider capability +14. **Keep runtime and presentation separate** - Runtime OAuth fields live in `env-capabilities.ts`; CLI input modes live in the exhaustively checked `scripts/setup/capability-config.ts` mapping diff --git a/.cursor/commands/ship.md b/.cursor/commands/ship.md index 3049709300f..58ceae2ccae 100644 --- a/.cursor/commands/ship.md +++ b/.cursor/commands/ship.md @@ -48,17 +48,23 @@ When the user runs `/ship`: ``` Then `git status --short` to see what regenerated — those files must be staged in step 7 alongside your own changes. - **Do NOT blanket-run the domain generators here.** `mship:generate` (`generate-mship-contracts.ts`) is an **umbrella** that drives all nine mothership contract generators (`mship-contracts`, `billing-protocol-contract`, `mship-tools`, the four `trace-*`, `metrics-contract`, `vfs-snapshot-contract`) and biome-formats `apps/sim/lib/copilot/generated/` — never run it *and* its constituents (they write the same files and corrupt each other in parallel), and never run it on an ordinary ship: it reads an **external** copilot-contract source that isn't checked out in most worktrees, so it hard-fails with `ENOENT` and would abort ship for an unrelated reason. `generate:pi-model-catalog` (under `apps/sim`) likewise regenerates from the installed Pi package, not repo source. Only when **this PR's diff actually touches** a domain generator's input do you regenerate it deliberately and run its matching `:check` (`bun run mship:check` / the individual `*:check`) — with the external source present. + **Do NOT blanket-run the domain generators here.** `mship:generate` (`generate-mship-contracts.ts`) is an **umbrella** that drives all nine mothership contract generators (`mship-contracts`, `billing-protocol-contract`, `mship-tools`, the four `trace-*`, `metrics-contract`, `vfs-snapshot-contract`) and biome-formats `apps/sim/lib/copilot/generated/` — never run it *and* its constituents (they write the same files and corrupt each other in parallel), and never run it on an ordinary ship: it reads an **external** copilot-contract source that isn't checked out in most worktrees, so it hard-fails with `ENOENT` and would abort ship for an unrelated reason. `generate:pi-model-catalog` (under `apps/sim`) likewise regenerates from the installed Pi package, not repo source. `scripts/generate-docs.ts` rewrites the integration docs and client-safe catalog; run it when this PR changes their block/icon/landing-content inputs or when `integration-catalog:check` reports drift, then review its broad generated diff. Only when **this PR's diff actually touches** a domain generator's input do you regenerate it deliberately and run its matching `:check` (`bun run mship:check` / the individual `*:check`) — with the external source present. - **Phase B — run lint + every audit CI enforces, in parallel, and abort ship if any fails.** `bun run lint` first (it autofixes formatting and mutates files, so don't parallelize it with the read-only audits), then fan the rest out and collect exit codes. This is exactly the read-only audit set from CI's `Lint and Test` job (all in-repo, runnable in any worktree): + **Phase B — run lint + every audit CI enforces, in parallel, and abort ship if any fails.** Before running the commands, compare this list with `.github/workflows/test-build.yml`; when CI adds an audit, run it and update this skill instead of trusting a stale snapshot. The env-flag audit is currently an inline workflow block rather than a package script: when `apps/sim/lib/core/config/env-flags.ts` changed, run that current workflow block verbatim instead of copying a second version into this skill. Run `bun run lint` first (it autofixes formatting and mutates files, so don't parallelize it with the read-only audits), then run the base-sensitive block-registry check, then fan the independent audits out and collect exit codes: ```bash # autofix formatting first (mutating; not parallel-safe with the audits). Gate its exit too — # a non-zero lint (unfixable errors) must abort before the audits run, not be ignored. bun run lint || { echo "❌ lint failed — do not ship"; exit 1; } + bun run apps/sim/scripts/check-block-registry.ts origin/staging || { + echo "❌ block registry audit failed — do not ship" + exit 1 + } rm -f /tmp/ship-audit-results - for s in check:boundaries check:api-validation:strict check:utils check:zustand-v5 \ + for s in check:boundaries check:api-validation:strict check:desktop-bridge check:desktop-ipc \ + check:utils check:zustand-v5 \ check:react-query check:client-boundary check:bare-icons check:icon-paths \ - check:realtime-prune skills:check agent-stream-docs:check; do + check:realtime-prune check:tool-registry-boundary tool-metadata:check \ + integration-catalog:check skills:check agent-stream-docs:check; do ( bun run "$s" >"/tmp/ship-audit-${s//:/-}.log" 2>&1; echo "$? $s" >>/tmp/ship-audit-results ) & done wait diff --git a/.cursor/commands/validate-integration.md b/.cursor/commands/validate-integration.md index 45af009bc1d..223b76c44e1 100644 --- a/.cursor/commands/validate-integration.md +++ b/.cursor/commands/validate-integration.md @@ -24,6 +24,11 @@ apps/sim/components/icons.tsx # Icon definition apps/sim/lib/auth/auth.ts # OAuth config — should use getCanonicalScopesForProvider() apps/sim/lib/oauth/oauth.ts # OAuth provider config — single source of truth for scopes apps/sim/lib/oauth/utils.ts # Scope utilities, SCOPE_DESCRIPTIONS for modal UI +apps/sim/lib/core/config/env-capabilities.ts # OAuth client runtime capability source of truth +apps/sim/lib/core/config/env.ts # Runtime env schema for capability fields +scripts/setup/capability-config.ts # Exhaustive CLI input-mode mapping for OAuth fields +apps/sim/lib/integrations/integrations.json # Generated client-safe integration catalog +apps/sim/lib/integrations/service-account-metadata.ts # Lightweight service-account projection ``` ## Step 2: Pull API Documentation @@ -227,7 +232,28 @@ Scopes are centralized — the single source of truth is `OAUTH_PROVIDERS` in `l - [ ] Each scope has a human-readable description in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` - [ ] No excess scopes that aren't needed by any tool -## Step 6: Validate Pagination Consistency +## Step 6: Validate Deployment Availability (if OAuth service) + +The deployment UI and setup CLI do not infer OAuth client fields from scopes. They resolve the +block's generated `oauthServiceId` through the application-owned capability catalog. + +- [ ] The visible integration block has exactly one distinct `oauth-input.serviceId` +- [ ] `resolveOAuthClientCapabilityId(serviceId)` returns the intended provider capability +- [ ] The resolved provider exists in `OAUTH_CLIENT_CAPABILITIES` +- [ ] Every field listed by that capability exists in `apps/sim/lib/core/config/env.ts` +- [ ] Every capability field has the correct `text` or `secret` entry in `OAUTH_CLIENT_SETUP_FIELDS`; no CLI naming heuristic is required +- [ ] Shared Google/Microsoft service IDs resolve to their provider capability rather than duplicate entries +- [ ] `bun run setup integration ` is the command emitted by availability; the CLI has only the exhaustive input-mode projection, not a second runtime provider definition +- [ ] If the canonical OAuth service declares `serviceAccountProviderId`, + `SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID[serviceId]` has the same provider ID +- [ ] The service-account `deploymentRequirement` matches how that credential actually works: + omitted for an independent path, `'oauth-client'` when it needs the OAuth client fields, or + `'preview-gated'` when controlled by a preview block + +Treat a missing capability as **critical**: runtime availability intentionally throws instead of +silently exposing an unusable integration. + +## Step 7: Validate Pagination Consistency If any tools support pagination: - [ ] Pagination param names match the API docs (e.g., `pagination_token` vs `next_token` vs `cursor`) @@ -235,7 +261,7 @@ If any tools support pagination: - [ ] Pagination response fields (`nextToken`, `cursor`, etc.) are included in tool outputs - [ ] Pagination subBlocks are set to `mode: 'advanced'` -## Step 7: Validate Memory Load Safety +## Step 8: Validate Memory Load Safety If any tool lists, searches, exports, imports, downloads, uploads, paginates, batches, transforms arrays, or reads file/HTTP bodies, read `.agents/skills/memory-load-check/SKILL.md` and apply it to the integration. @@ -245,13 +271,13 @@ If any tool lists, searches, exports, imports, downloads, uploads, paginates, ba - [ ] Large result payloads are summarized, paginated, referenced, or capped rather than raw-dumped - [ ] Pagination and download tests cover caps, early stop behavior, or partial-result preservation when relevant -## Step 8: Validate Error Handling +## Step 9: Validate Error Handling - [ ] `transformResponse` checks for error conditions before accessing data - [ ] Error responses include meaningful messages (not just generic "failed") - [ ] HTTP error status codes are handled (check `response.ok` or status codes) -## Step 9: Report and Fix +## Step 10: Report and Fix ### Report Format @@ -264,6 +290,9 @@ Group findings by severity: - Missing error handling that would cause crashes - Tool ID mismatch between tool file, registry, and block `tools.access` - OAuth scopes missing in `auth.ts` that tools need +- OAuth integration `serviceId` missing from the deployment capability catalog +- Capability references an env field absent from the runtime env schema +- Service-account metadata disagrees with the canonical OAuth service configuration - `tools.config.tool` returning wrong tool ID for an operation - Type coercions in `tools.config.tool` instead of `tools.config.params` @@ -295,11 +324,15 @@ Several files are generated from tool and block definitions. Editing a tool or b ```bash bun run tool-metadata:generate # repo root — apps/sim/tools/generated/* -cd apps/sim && bun run generate-docs # docs .mdx + lib/integrations/integrations.json + docs icons +bun run scripts/generate-docs.ts # docs .mdx + lib/integrations/integrations.json + docs icons +bun run integration-catalog:check # registry ↔ committed deployment metadata drift ``` - **`tool-metadata:generate`** — required whenever a tool's `outputs`, `params`, or descriptions change. CI enforces this with `bun run tool-metadata:check`, which fails with *"Generated tool metadata is stale"*. This is the easiest gate to miss, because nothing in the tool file hints that a generated artifact mirrors it. - **`generate-docs`** — required whenever block metadata changes (`bgColor`, `name`, `description`, operations, outputs). Regenerates the integration `.mdx`, `integrations.json`, and the docs copy of `components/icons.tsx`. +- **`integration-catalog:check`** — loads the executable block registry, derives visible integration + deployment fields, and compares them with the committed catalog. It catches missing/unexpected + entries and stale auth/service IDs without loading the executable registry in client code. **Always diff the regen output before committing.** These generators rewrite every file they own, so they will also sweep in unrelated drift that accumulated on the base branch — pages losing sections, unrelated icons appearing. Keep only the hunks belonging to the integration under validation and `git checkout --` the rest, otherwise an unrelated doc regression rides along in the PR. Verify no page was silently dropped by comparing the directory listing before and after. @@ -312,8 +345,10 @@ After fixing, confirm: 2. TypeScript compiles clean (no type errors) — check the error list is empty for the files you touched; pre-existing unrelated errors in a worktree usually mean workspace packages resolve to the main checkout 3. The integration's tests pass, and any test you added actually fails without its fix (revert it once and watch it go red) 4. Derived artifacts regenerated and their diffs reviewed (see above) -5. Re-read all modified files to verify fixes are correct -6. Any remaining unknown response schemas were explicitly reported to the user instead of guessed +5. `bun run integration-catalog:check` passes +6. For OAuth or service-account changes, `bun test apps/sim/lib/integrations/availability.server.test.ts` passes +7. Re-read all modified files to verify fixes are correct +8. Any remaining unknown response schemas were explicitly reported to the user instead of guessed ## Checklist Summary @@ -327,6 +362,9 @@ After fixing, confirm: - [ ] Validated block outputs match what tools return, with typed JSON where possible - [ ] Validated OAuth scopes use centralized utilities (getScopesForService, getCanonicalScopesForProvider) — no hardcoded arrays - [ ] Validated scope descriptions exist in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` for all scopes +- [ ] Validated OAuth `serviceId` resolves to the intended `OAUTH_CLIENT_CAPABILITIES` entry and all capability fields exist in the env schema +- [ ] Validated service-account projection and deployment requirement against the canonical OAuth service config +- [ ] Regenerated `integrations.json` when block metadata changed and ran `bun run integration-catalog:check` - [ ] Validated pagination consistency across tools and block - [ ] Validated memory load safety using `.agents/skills/memory-load-check/SKILL.md` when tools list/search/download/import/export/batch data - [ ] Validated error handling (error checks, meaningful messages) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 988256441c9..a5e85b5753b 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -316,7 +316,7 @@ If you prefer not to use Docker or Dev Containers. **All commands run from the r ```bash bun run type-check # TypeScript across every workspace bun run lint:check # Biome lint across every workspace - bun run test # Vitest across every workspace + bun run test # Setup CLI Bun tests, then Vitest across every workspace ``` ### Email Template Development diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 4b5a0fb1404..5d971ab15fc 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -159,6 +159,9 @@ jobs: - name: Verify generated tool metadata is in sync run: bun run tool-metadata:check + - name: Verify integration deployment metadata is in sync + run: bun run integration-catalog:check + - name: Verify skill projections are in sync run: bun run skills:check @@ -183,8 +186,8 @@ jobs: - name: Install ripgrep run: command -v rg || (sudo apt-get update && sudo apt-get install -y ripgrep) - # Named for what it does: `bun run test` is `vitest run`, with no - # `--coverage`. See the Codecov note below. + # Runs the setup CLI's Bun tests plus each workspace's Vitest suite, + # without `--coverage`. See the Codecov note below. - name: Run tests env: NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' diff --git a/README.md b/README.md index d1635b95cbd..03fc399473e 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ ```bash git clone https://github.com/simstudioai/sim.git && cd sim +bun install bun run setup ``` @@ -81,6 +82,25 @@ Open [http://localhost:3000](http://localhost:3000) When it finishes, open [http://localhost:3000](http://localhost:3000). +Reconfigure an optional capability without rerunning the full wizard: + +```bash +bun run setup status +bun run setup email +bun run setup storage +bun run setup sandbox +bun run setup jobs +bun run setup cache +bun run setup knowledge +bun run setup llm +bun run setup integration slack +``` + +`bun run setup status` detects the effective local-dev, Docker Compose, or current-context +Helm configuration and reports configured, missing, or invalid capabilities and OAuth +integrations without printing credential values. This is separate from `bun run sim status`, +which reports whether installed services are running and healthy. + Manage your install with `bun run sim`: ```bash diff --git a/apps/docs/content/docs/en/platform/self-hosting/object-storage.mdx b/apps/docs/content/docs/en/platform/self-hosting/object-storage.mdx index 64716f9f0f3..0cfe03f1723 100644 --- a/apps/docs/content/docs/en/platform/self-hosting/object-storage.mdx +++ b/apps/docs/content/docs/en/platform/self-hosting/object-storage.mdx @@ -23,14 +23,14 @@ Sim stores every uploaded file — knowledge base documents, chat attachments, e ## How the backend is selected -Sim picks the backend automatically from environment variables — there is no explicit "provider" flag. The logic, in order of precedence: +Set `STORAGE_PROVIDER` to `local`, `s3`, `azure`, or `gcs` to select a backend explicitly. When it is unset, Sim infers the backend from the configured environment variables in this order: 1. **Azure Blob** — used if `AZURE_STORAGE_CONTAINER_NAME` is set **and** either (`AZURE_ACCOUNT_NAME` + `AZURE_ACCOUNT_KEY`) or `AZURE_CONNECTION_STRING` is set. 2. **AWS S3** — used if `S3_BUCKET_NAME` **and** `AWS_REGION` are set (and Azure is not configured). 3. **Google Cloud Storage** — used if `GCS_BUCKET_NAME` is set (and neither Azure nor S3 is configured). 4. **Local disk** — the fallback when none is configured. -If more than one backend is configured, the first match in that order wins. Set only the variables for the backend you intend to use. +If `STORAGE_PROVIDER` is unset, Sim skips incomplete backends and uses the first ready match in that order. A higher-priority backend with its required fields present but invalid supplied values fails fast instead of silently falling through. An explicit `STORAGE_PROVIDER` takes precedence and must be valid and complete. ## Set up AWS S3 diff --git a/apps/sim/.env.example b/apps/sim/.env.example index acb633776bf..fe6daf70fed 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -32,9 +32,9 @@ API_ENCRYPTION_KEY=your_api_encryption_key # Use `openssl rand -hex 32` to gener CRON_SECRET=your_cron_secret # Use `openssl rand -hex 32` to generate. Authenticates the scheduler against the background job endpoints (scheduled workflows, polling triggers, connector syncs) # Email Provider (Optional) -# Configure ONE provider — the mailer auto-detects in priority order: -# Resend → AWS SES → SMTP → Azure Communication Services → Gmail. If none -# are configured, emails are logged to console instead. +# Configure one or more providers. Every configured provider stays active and is +# tried in order: Resend → AWS SES → SMTP → Azure Communication Services → Gmail. +# If none are configured, emails are logged to console instead. # # Resend # RESEND_API_KEY= # API key from https://resend.com @@ -96,7 +96,11 @@ CRON_SECRET=your_cron_secret # Use `openssl rand -hex 32` to generate. Authentic # CONTEXT_DEV_API_KEY_1= # Context.dev API key #1 # CONTEXT_DEV_API_KEY_2= # Context.dev API key #2 +# PDF OCR provider (Optional - defaults to local; legacy installs infer Mistral from configured credentials) +# OCR_PROVIDER=local # One of: local, mistral, azure-mistral + # File Storage (Optional - defaults to local disk; use S3, Azure Blob, or Google Cloud Storage for production) +# STORAGE_PROVIDER=local # Optional override: local, s3, azure, or gcs. Unset preserves Azure → S3 → GCS → local precedence # AWS_REGION=us-east-1 # Required with S3_BUCKET_NAME to enable S3. Use "auto" for Cloudflare R2 # AWS_ACCESS_KEY_ID= # Omit to use the instance/IRSA credential chain # AWS_SECRET_ACCESS_KEY= # Omit to use the instance/IRSA credential chain @@ -121,7 +125,7 @@ CRON_SECRET=your_cron_secret # Use `openssl rand -hex 32` to generate. Authentic # TIKTOK_CLIENT_ID= # TIKTOK_CLIENT_SECRET= -# Azure Blob Storage takes precedence over S3 if both are configured +# Azure Blob Storage # AZURE_ACCOUNT_NAME= # Azure storage account name # AZURE_ACCOUNT_KEY= # Azure storage account key # AZURE_CONNECTION_STRING= # Alternative to account name/key @@ -134,7 +138,7 @@ CRON_SECRET=your_cron_secret # Use `openssl rand -hex 32` to generate. Authentic # AZURE_STORAGE_OG_IMAGES_CONTAINER_NAME= # OpenGraph preview images (falls back to AZURE_STORAGE_CONTAINER_NAME) # AZURE_STORAGE_WORKSPACE_LOGOS_CONTAINER_NAME= # Workspace logos (falls back to AZURE_STORAGE_CONTAINER_NAME) -# Google Cloud Storage (used when neither Azure Blob nor S3 is configured) +# Google Cloud Storage # GCS_PROJECT_ID= # GCP project ID (optional — inferred from credentials/ADC when unset) # GCS_CREDENTIALS_JSON= # Inline service-account JSON. Omit to use Application Default Credentials (Workload Identity, GOOGLE_APPLICATION_CREDENTIALS) # GCS_BUCKET_NAME= # General workspace files bucket (enables GCS; all other buckets fall back to it) diff --git a/apps/sim/app/api/auth/instagram/authorize/route.ts b/apps/sim/app/api/auth/instagram/authorize/route.ts index 1b509122aa1..b33a0c0c510 100644 --- a/apps/sim/app/api/auth/instagram/authorize/route.ts +++ b/apps/sim/app/api/auth/instagram/authorize/route.ts @@ -4,7 +4,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { authorizeInstagramContract } from '@/lib/api/contracts/oauth-connections' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' -import { env } from '@/lib/core/config/env' +import { requireConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' import { getBaseUrl } from '@/lib/core/utils/urls' import { isSameOrigin } from '@/lib/core/utils/validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' @@ -28,11 +28,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } - const clientId = env.INSTAGRAM_CLIENT_ID - if (!clientId) { - logger.error('INSTAGRAM_CLIENT_ID not configured') - return NextResponse.json({ error: 'Instagram client ID not configured' }, { status: 500 }) - } + const { + values: { INSTAGRAM_CLIENT_ID: clientId }, + } = requireConfiguredOAuthClient('instagram') const parsed = await parseRequest(authorizeInstagramContract, request, {}) if (!parsed.success) return parsed.response diff --git a/apps/sim/app/api/auth/oauth2/authorize/route.test.ts b/apps/sim/app/api/auth/oauth2/authorize/route.test.ts index 8b4abab40b0..69f7b3a3a60 100644 --- a/apps/sim/app/api/auth/oauth2/authorize/route.test.ts +++ b/apps/sim/app/api/auth/oauth2/authorize/route.test.ts @@ -81,7 +81,11 @@ describe('OAuth2 authorize route', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - setEnv({ NEXT_PUBLIC_APP_URL: BASE_URL }) + setEnv({ + NEXT_PUBLIC_APP_URL: BASE_URL, + GOOGLE_CLIENT_ID: 'google-client', + GOOGLE_CLIENT_SECRET: 'google-secret', + }) mockGetSession.mockResolvedValue({ user: { id: USER_ID } }) mockCheckWorkspaceAccess.mockResolvedValue({ hasAccess: true, @@ -139,6 +143,18 @@ describe('OAuth2 authorize route', () => { expect(set).toHaveProperty('credentialId', null) }) + it('rejects an OAuth client that is not configured for the deployment', async () => { + setEnv({ GOOGLE_CLIENT_ID: undefined, GOOGLE_CLIENT_SECRET: undefined }) + + const response = await GET( + authorizeRequest({ providerId: 'google-email', workspaceId: WORKSPACE_ID }) + ) + + expect(response.headers.get('location')).toBe(`${BASE_URL}/workspace?error=oauth_link_failed`) + expect(dbChainMockFns.values).not.toHaveBeenCalled() + expect(mockOAuth2LinkAccount).not.toHaveBeenCalled() + }) + it('redirects to login when unauthenticated', async () => { mockGetSession.mockResolvedValue(null) diff --git a/apps/sim/app/api/auth/oauth2/authorize/route.ts b/apps/sim/app/api/auth/oauth2/authorize/route.ts index 2eb916fa53a..063de2ca015 100644 --- a/apps/sim/app/api/auth/oauth2/authorize/route.ts +++ b/apps/sim/app/api/auth/oauth2/authorize/route.ts @@ -3,6 +3,7 @@ import { type NextRequest, NextResponse } from 'next/server' import { authorizeOAuth2Contract } from '@/lib/api/contracts/oauth-connections' import { parseRequest } from '@/lib/api/server' import { auth, getSession } from '@/lib/auth/auth' +import { requireConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' import { getBaseUrl } from '@/lib/core/utils/urls' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getCredentialActorContext } from '@/lib/credentials/access' @@ -100,6 +101,8 @@ export const GET = withRouteHandler(async (request: NextRequest) => { reconnectDisplayName = actor.credential.displayName } + requireConfiguredOAuthClient(providerId) + // Create the draft before initiating the link so it is guaranteed to exist // (and freshly clocked) when the OAuth callback's `account.create.after` // hook runs. If this throws, we never start the OAuth flow. diff --git a/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts b/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts index c016a87c49c..6b6b9b76a7b 100644 --- a/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts +++ b/apps/sim/app/api/auth/oauth2/callback/instagram/route.ts @@ -7,7 +7,8 @@ import { type NextRequest, NextResponse } from 'next/server' import { instagramCallbackContract } from '@/lib/api/contracts/oauth-connections' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' -import { env } from '@/lib/core/config/env' +import { EnvCapabilityConfigurationError } from '@/lib/core/config/env-capabilities' +import { requireConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -76,14 +77,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => { ) } - const clientId = env.INSTAGRAM_CLIENT_ID - const clientSecret = env.INSTAGRAM_CLIENT_SECRET - if (!clientId || !clientSecret) { - logger.error('Instagram credentials not configured') - return clearOAuthCookies( - NextResponse.redirect(`${baseUrl}/workspace?error=instagram_config_error`) - ) - } + const { + values: { INSTAGRAM_CLIENT_ID: clientId, INSTAGRAM_CLIENT_SECRET: clientSecret }, + } = requireConfiguredOAuthClient('instagram') if (!code) { logger.error('No authorization code received from Instagram') @@ -318,8 +314,10 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return clearOAuthCookies(NextResponse.redirect(finalUrl.toString())) } catch (error) { logger.error('Error in Instagram OAuth callback', { error }) - return clearOAuthCookies( - NextResponse.redirect(`${baseUrl}/workspace?error=instagram_callback_error`) - ) + const errorCode = + error instanceof EnvCapabilityConfigurationError && error.capabilityId === 'oauth' + ? 'instagram_config_error' + : 'instagram_callback_error' + return clearOAuthCookies(NextResponse.redirect(`${baseUrl}/workspace?error=${errorCode}`)) } }) diff --git a/apps/sim/app/api/auth/oauth2/callback/shopify/route.ts b/apps/sim/app/api/auth/oauth2/callback/shopify/route.ts index a3ab5ac06df..2292a76a9a6 100644 --- a/apps/sim/app/api/auth/oauth2/callback/shopify/route.ts +++ b/apps/sim/app/api/auth/oauth2/callback/shopify/route.ts @@ -7,7 +7,8 @@ import { shopifyShopDomainSchema, } from '@/lib/api/contracts/oauth-connections' import { getSession } from '@/lib/auth' -import { env } from '@/lib/core/config/env' +import { EnvCapabilityConfigurationError } from '@/lib/core/config/env-capabilities' +import { requireConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' import { getBaseUrl } from '@/lib/core/utils/urls' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' @@ -61,13 +62,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const storedState = request.cookies.get('shopify_oauth_state')?.value const storedShop = request.cookies.get('shopify_shop_domain')?.value - const clientId = env.SHOPIFY_CLIENT_ID - const clientSecret = env.SHOPIFY_CLIENT_SECRET - - if (!clientId || !clientSecret) { - logger.error('Shopify credentials not configured') - return NextResponse.redirect(`${baseUrl}/workspace?error=shopify_config_error`) - } + const { + values: { SHOPIFY_CLIENT_ID: clientId, SHOPIFY_CLIENT_SECRET: clientSecret }, + } = requireConfiguredOAuthClient('shopify') if (!validateHmac(searchParams, clientSecret)) { logger.error('HMAC validation failed in Shopify OAuth callback') @@ -164,6 +161,10 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return response } catch (error) { logger.error('Error in Shopify OAuth callback:', error) - return NextResponse.redirect(`${baseUrl}/workspace?error=shopify_callback_error`) + const errorCode = + error instanceof EnvCapabilityConfigurationError && error.capabilityId === 'oauth' + ? 'shopify_config_error' + : 'shopify_callback_error' + return NextResponse.redirect(`${baseUrl}/workspace?error=${errorCode}`) } }) diff --git a/apps/sim/app/api/auth/shopify/authorize/route.ts b/apps/sim/app/api/auth/shopify/authorize/route.ts index 43be71dfd17..d2d3a5401e4 100644 --- a/apps/sim/app/api/auth/shopify/authorize/route.ts +++ b/apps/sim/app/api/auth/shopify/authorize/route.ts @@ -6,7 +6,7 @@ import { shopifyShopDomainSchema, } from '@/lib/api/contracts/oauth-connections' import { getSession } from '@/lib/auth' -import { env } from '@/lib/core/config/env' +import { requireConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' import { getBaseUrl } from '@/lib/core/utils/urls' import { isSameOrigin } from '@/lib/core/utils/validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' @@ -25,12 +25,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } - const clientId = env.SHOPIFY_CLIENT_ID - - if (!clientId) { - logger.error('SHOPIFY_CLIENT_ID not configured') - return NextResponse.json({ error: 'Shopify client ID not configured' }, { status: 500 }) - } + const { + values: { SHOPIFY_CLIENT_ID: clientId }, + } = requireConfiguredOAuthClient('shopify') const query = shopifyAuthorizeQuerySchema.parse({ shop: request.nextUrl.searchParams.get('shop') || undefined, diff --git a/apps/sim/app/api/settings/allowed-integrations/route.ts b/apps/sim/app/api/settings/allowed-integrations/route.ts index 19da51361ca..c5acc2582ca 100644 --- a/apps/sim/app/api/settings/allowed-integrations/route.ts +++ b/apps/sim/app/api/settings/allowed-integrations/route.ts @@ -2,6 +2,7 @@ import { NextResponse } from 'next/server' import { getSession } from '@/lib/auth' import { getAllowedIntegrationsFromEnv } from '@/lib/core/config/env-flags' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { getIntegrationAvailability } from '@/lib/integrations/availability.server' export const GET = withRouteHandler(async () => { const session = await getSession() @@ -11,5 +12,8 @@ export const GET = withRouteHandler(async () => { return NextResponse.json({ allowedIntegrations: getAllowedIntegrationsFromEnv(), + integrationAvailability: getIntegrationAvailability().map( + ({ type, state, oauthAvailable }) => ({ type, state, oauthAvailable }) + ), }) }) diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx index dcb21376344..cfc63c06bba 100644 --- a/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx +++ b/apps/sim/app/workspace/[workspaceId]/integrations/[block]/integration-block-detail.tsx @@ -24,7 +24,10 @@ import { } from '@/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal' import { IntegrationSection } from '@/app/workspace/[workspaceId]/integrations/components/integration-section' import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' -import { CONNECT_MODE } from '@/app/workspace/[workspaceId]/integrations/connect-route' +import { + CONNECT_MODE, + resolveAvailableConnectMode, +} from '@/app/workspace/[workspaceId]/integrations/connect-route' import { useScrollRestoration } from '@/app/workspace/[workspaceId]/integrations/hooks/use-scroll-restoration' import { RESOURCE_LIST_STACK, @@ -40,6 +43,7 @@ import { } from '@/blocks/registry' import { useWorkspaceCredentials } from '@/hooks/queries/credentials' import { useOAuthReturnRouter } from '@/hooks/use-oauth-return' +import { usePermissionConfig } from '@/hooks/use-permission-config' /** Maximum number of overlapping icon tiles rendered per template row. */ const TEMPLATE_CLUSTER_MAX = 3 as const @@ -64,6 +68,9 @@ export function IntegrationBlockDetail({ integration, workspaceId }: Integration const matchingTemplates = getTemplatesForBlock(integration.type) const suggestedSkills = getSuggestedSkillsForBlock(integration.type) const oauthService = resolveOAuthServiceForIntegration(integration) + const { integrationAvailability, isLoading: permissionConfigLoading } = usePermissionConfig() + const availability = integrationAvailability.get(integration.type.toLowerCase()) + const oauthAvailable = Boolean(oauthService) && (availability?.oauthAvailable ?? true) const [oauthOpen, setOAuthOpen] = useState(false) const { data: credentials = [], isPending: credentialsLoading } = useWorkspaceCredentials({ @@ -93,40 +100,61 @@ export function IntegrationBlockDetail({ integration, workspaceId }: Integration serviceName: oauthService?.serviceName, serviceIcon: oauthService?.serviceIcon, }) - const hasServiceAccount = Boolean(serviceAccountTarget) && !serviceAccountTarget?.hidden + const serviceAccountDeploymentAvailable = + availability?.state === 'ready' || availability?.state === 'limited' + const hasServiceAccount = + serviceAccountDeploymentAvailable && + Boolean(serviceAccountTarget) && + !serviceAccountTarget?.hidden const serviceAccountConnectLabel = serviceAccountTarget?.label ?? 'Add service account' const hasHandledConnectQueryRef = useRef(false) useEffect(() => { - if (hasHandledConnectQueryRef.current) return - if (!connectMode) return + if (hasHandledConnectQueryRef.current || !connectMode || permissionConfigLoading) return + + const availableConnectMode = resolveAvailableConnectMode(connectMode, { + oauth: Boolean(oauthService) && oauthAvailable, + serviceAccount: hasServiceAccount, + }) + if (!availableConnectMode) return - let handled = false - if (connectMode === CONNECT_MODE.oauth && oauthService) { + if (availableConnectMode === CONNECT_MODE.oauth) { setOAuthOpen(true) - handled = true - } else if (connectMode === CONNECT_MODE.serviceAccount && hasServiceAccount) { + } else { setServiceAccountOpen(true) - handled = true } - if (!handled) return hasHandledConnectQueryRef.current = true void setConnectMode(null, { history: 'replace', scroll: false }) - }, [connectMode, oauthService, hasServiceAccount, setConnectMode]) + }, [ + connectMode, + oauthService, + oauthAvailable, + hasServiceAccount, + permissionConfigLoading, + setConnectMode, + ]) const connectOptions = oauthService ? [ - { - value: CONNECT_MODE.oauth, - label: 'Connect with OAuth', - icon: oauthService.serviceIcon, - }, - { - value: CONNECT_MODE.serviceAccount, - label: serviceAccountConnectLabel, - icon: serviceAccountTarget?.serviceIcon ?? oauthService.serviceIcon, - }, + ...(oauthAvailable + ? [ + { + value: CONNECT_MODE.oauth, + label: 'Connect with OAuth', + icon: oauthService.serviceIcon, + }, + ] + : []), + ...(hasServiceAccount + ? [ + { + value: CONNECT_MODE.serviceAccount, + label: serviceAccountConnectLabel, + icon: serviceAccountTarget?.serviceIcon ?? oauthService.serviceIcon, + }, + ] + : []), ] : [] @@ -148,7 +176,7 @@ export function IntegrationBlockDetail({ integration, workspaceId }: Integration
{oauthService ? ( - hasServiceAccount ? ( + connectOptions.length > 1 ? ( - ) : ( + ) : oauthAvailable ? ( setOAuthOpen(true)}> Add to Sim + ) : hasServiceAccount ? ( + setServiceAccountOpen(true)}> + {serviceAccountConnectLabel} + + ) : ( + Unavailable ) ) : isChatEnabled ? ( @@ -170,7 +204,7 @@ export function IntegrationBlockDetail({ integration, workspaceId }: Integration ) : null}
- {oauthService && ( + {oauthService && oauthAvailable && ( { + it('keeps a service-account deep link pending until its modal is available', () => { + expect( + resolveAvailableConnectMode(CONNECT_MODE.serviceAccount, { + oauth: false, + serviceAccount: false, + }) + ).toBeNull() + + expect( + resolveAvailableConnectMode(CONNECT_MODE.serviceAccount, { + oauth: false, + serviceAccount: true, + }) + ).toBe(CONNECT_MODE.serviceAccount) + }) + + it('only resolves OAuth when OAuth is available', () => { + expect( + resolveAvailableConnectMode(CONNECT_MODE.oauth, { + oauth: false, + serviceAccount: true, + }) + ).toBeNull() + + expect( + resolveAvailableConnectMode(CONNECT_MODE.oauth, { + oauth: true, + serviceAccount: false, + }) + ).toBe(CONNECT_MODE.oauth) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/connect-route.ts b/apps/sim/app/workspace/[workspaceId]/integrations/connect-route.ts index c4bd0e08458..31ac7efd97e 100644 --- a/apps/sim/app/workspace/[workspaceId]/integrations/connect-route.ts +++ b/apps/sim/app/workspace/[workspaceId]/integrations/connect-route.ts @@ -10,3 +10,22 @@ export const CONNECT_MODE = { oauth: 'oauth', serviceAccount: 'service-account', } as const + +export type ConnectMode = (typeof CONNECT_MODE)[keyof typeof CONNECT_MODE] + +interface ConnectModeAvailability { + oauth: boolean + serviceAccount: boolean +} + +/** `null` lets callers preserve the deep-link while deployment and block visibility hydrate. */ +export function resolveAvailableConnectMode( + connectMode: ConnectMode, + availability: ConnectModeAvailability +): ConnectMode | null { + if (connectMode === CONNECT_MODE.oauth && availability.oauth) return connectMode + if (connectMode === CONNECT_MODE.serviceAccount && availability.serviceAccount) { + return connectMode + } + return null +} diff --git a/apps/sim/app/workspace/[workspaceId]/integrations/integrations.tsx b/apps/sim/app/workspace/[workspaceId]/integrations/integrations.tsx index df20fde691b..ed2f2ccd2ce 100644 --- a/apps/sim/app/workspace/[workspaceId]/integrations/integrations.tsx +++ b/apps/sim/app/workspace/[workspaceId]/integrations/integrations.tsx @@ -36,6 +36,7 @@ import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/compo import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' import { useWorkspaceCredentials, type WorkspaceCredential } from '@/hooks/queries/credentials' import { useDebouncedSearchSetter } from '@/hooks/use-debounced-search-setter' +import { usePermissionConfig } from '@/hooks/use-permission-config' /** Slugs surfaced in the pinned Featured section, in display order. */ const FEATURED_SLUGS = ['slack', 'gmail', 'jira', 'github', 'google-sheets', 'hubspot'] as const @@ -68,6 +69,7 @@ interface IntegrationItemProps { name: string description?: string | null icon: ComponentType<{ className?: string }> + unavailable?: boolean } function IntegrationItem({ @@ -77,16 +79,22 @@ function IntegrationItem({ name, description, icon: Icon, + unavailable = false, }: IntegrationItemProps) { return ( } title={name} - description={description || undefined} + description={ + unavailable + ? 'Unavailable in this deployment. Contact your administrator.' + : description || undefined + } href={`/workspace/${workspaceId}/integrations/${slug}`} clickLabel={`Open ${name}`} - navigable + navigable={!unavailable} + disabled={unavailable} /> ) } @@ -133,6 +141,7 @@ export function Integrations() { const scrollContainerRef = useRef(null) const params = useParams() const workspaceId = (params?.workspaceId as string) || '' + const { integrationAvailability } = usePermissionConfig() const [{ category: selectedCategory, search: urlSearchTerm }, setIntegrationFilters] = useQueryStates(integrationsParsers, integrationsUrlKeys) @@ -341,6 +350,9 @@ export function Integrations() { {section.integrations.map((integration) => { const Icon = blockTypeToIconMap[integration.type] if (!Icon) return null + const availability = integrationAvailability.get(integration.type.toLowerCase()) + const deploymentUnavailable = + availability?.state === 'unavailable' || availability?.state === 'misconfigured' return ( ) })} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/settings-resource-row/settings-resource-row.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/settings-resource-row/settings-resource-row.tsx index eb2c57ce186..f05e8b3c73e 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/settings-resource-row/settings-resource-row.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/settings-resource-row/settings-resource-row.tsx @@ -90,6 +90,8 @@ interface SettingsResourceRowProps { * the bleed would force a horizontal scrollbar. */ flush?: boolean + /** Renders the row as unavailable without an activation target. */ + disabled?: boolean } /** The one navigation chevron for every settings resource row. */ @@ -131,6 +133,7 @@ export function SettingsResourceRow({ clickLabel, navigable = false, flush = false, + disabled = false, }: SettingsResourceRowProps) { const describedById = useId() const isTile = iconVariant === 'tile' @@ -180,9 +183,13 @@ export function SettingsResourceRow({ // Row geometry is identical whether or not the row is activatable, so a list // mixing clickable and static rows keeps one height and one inset. - const rowClass = cn('flex items-center justify-between gap-2.5', !flush && '-mx-2 rounded-lg p-2') + const rowClass = cn( + 'flex items-center justify-between gap-2.5', + !flush && '-mx-2 rounded-lg p-2', + disabled && 'opacity-50' + ) - if (!onClick && !href) { + if (disabled || (!onClick && !href)) { return (
{cluster}
diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/integration-search-items.ts b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/integration-search-items.ts index 638eb4d2497..1d82af1f806 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/integration-search-items.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/components/search-modal/integration-search-items.ts @@ -22,6 +22,7 @@ const INTEGRATION_BASES: readonly { bgColor: string slug: string authType: string + blockType: string }[] = INTEGRATIONS.flatMap((integration) => { const icon = blockTypeToIconMap[integration.type] if (!icon) return [] @@ -33,6 +34,7 @@ const INTEGRATION_BASES: readonly { bgColor: integration.bgColor, slug: integration.slug, authType: integration.authType, + blockType: integration.type, }, ] }) @@ -43,10 +45,16 @@ const INTEGRATION_BASES: readonly { * the connect modal auto-opens (via the detail page's `useEffect` on * `CONNECT_QUERY_PARAM`). Non-OAuth integrations link to the plain detail page. */ -export function buildIntegrationSearchItems(workspaceId: string): IntegrationSearchItem[] { - return INTEGRATION_BASES.map((base) => { - const connectSuffix = - base.authType === 'oauth' ? `?${CONNECT_QUERY_PARAM}=${CONNECT_MODE.oauth}` : '' +export function buildIntegrationSearchItems( + workspaceId: string, + isBlockAllowed: (blockType: string) => boolean = () => true, + getConnectMode: ( + blockType: string + ) => (typeof CONNECT_MODE)[keyof typeof CONNECT_MODE] | null = () => CONNECT_MODE.oauth +): IntegrationSearchItem[] { + return INTEGRATION_BASES.filter((base) => isBlockAllowed(base.blockType)).map((base) => { + const connectMode = base.authType === 'oauth' ? getConnectMode(base.blockType) : null + const connectSuffix = connectMode ? `?${CONNECT_QUERY_PARAM}=${connectMode}` : '' return { id: base.id, name: base.name, diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx index e3f0741b269..ee944c23228 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx @@ -42,6 +42,7 @@ import { isChatEnabled } from '@/lib/core/config/env-flags' import { isMacPlatform } from '@/lib/core/utils/platform' import { buildFolderTree, getFolderPathNames } from '@/lib/folders/tree' import { captureEvent } from '@/lib/posthog/client' +import { CONNECT_MODE } from '@/app/workspace/[workspaceId]/integrations/connect-route' import { useRegisterGlobalCommands } from '@/app/workspace/[workspaceId]/providers/global-commands-provider' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import type { SettingsSection } from '@/app/workspace/[workspaceId]/settings/navigation' @@ -412,7 +413,12 @@ export const Sidebar = memo(function Sidebar({ const posthog = usePostHog() const { data: sessionData, isPending: sessionLoading } = useSession() const { canEdit, isLoading: permissionsLoading } = useUserPermissionsContext() - const { config: permissionConfig, filterBlocks } = usePermissionConfig() + const { + config: permissionConfig, + filterBlocks, + isBlockAllowed, + integrationAvailability, + } = usePermissionConfig() const { navigateToSettings } = useSettingsNavigation() const initializeSearchData = useSearchModalStore((state) => state.initializeData) const customBlockOverlayVersion = useCustomBlockOverlayVersion() @@ -1067,8 +1073,17 @@ export const Sidebar = memo(function Sidebar({ }) const searchModalIntegrations = useMemo( - () => (permissionConfig.hideIntegrationsTab ? [] : buildIntegrationSearchItems(workspaceId)), - [workspaceId, permissionConfig.hideIntegrationsTab] + () => + permissionConfig.hideIntegrationsTab + ? [] + : buildIntegrationSearchItems(workspaceId, isBlockAllowed, (blockType) => { + const availability = integrationAvailability.get(blockType.toLowerCase()) + if (!availability) return CONNECT_MODE.oauth + if (availability?.oauthAvailable) return CONNECT_MODE.oauth + if (availability?.state === 'limited') return CONNECT_MODE.serviceAccount + return null + }), + [workspaceId, permissionConfig.hideIntegrationsTab, isBlockAllowed, integrationAvailability] ) const searchModalConnectedAccounts = useMemo( diff --git a/apps/sim/executor/handlers/pi/babysit-backend.test.ts b/apps/sim/executor/handlers/pi/babysit-backend.test.ts index eba07fa6532..1c01d04d54d 100644 --- a/apps/sim/executor/handlers/pi/babysit-backend.test.ts +++ b/apps/sim/executor/handlers/pi/babysit-backend.test.ts @@ -1,7 +1,8 @@ /** * @vitest-environment node */ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { resetEnvMock, setEnv } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockWithPiSandbox, @@ -68,6 +69,8 @@ import { BABYSIT_ROUND_PATH } from '@/executor/handlers/pi/babysit-round' import type { PiBabysitContinuationParams } from '@/executor/handlers/pi/backend' import { DIFF_PATH } from '@/executor/handlers/pi/cloud-shared' +afterAll(resetEnvMock) + const OLD_SHA = 'a'.repeat(40) const NEW_SHA = 'c'.repeat(40) const SECOND_SHA = 'd'.repeat(40) @@ -235,6 +238,7 @@ function makeRunner(options: { describe('runBabysitPiWithOptions', () => { beforeEach(() => { vi.clearAllMocks() + setEnv({ SANDBOX_PROVIDER: 'e2b' }) mockWithPiSandbox.mockReset() mockFetchSnapshot.mockReset() mockFetchThreads.mockReset() diff --git a/apps/sim/hooks/use-permission-config.ts b/apps/sim/hooks/use-permission-config.ts index 3513c4c167d..a161bf9a924 100644 --- a/apps/sim/hooks/use-permission-config.ts +++ b/apps/sim/hooks/use-permission-config.ts @@ -3,15 +3,25 @@ import { useMemo } from 'react' import { useQuery } from '@tanstack/react-query' import { useParams } from 'next/navigation' -import { ApiClientError } from '@/lib/api/client/errors' import { requestJson } from '@/lib/api/client/request' -import { getAllowedIntegrationsContract } from '@/lib/api/contracts/common' +import { + type GetAllowedIntegrationsResponse, + getAllowedIntegrationsContract, + type IntegrationAvailabilityResponse, +} from '@/lib/api/contracts/common' import { getEnv, isTruthy } from '@/lib/core/config/env' +import { + isDeploymentGatedIntegrationType, + resolveIntegrationAvailabilityStateForVisibility, +} from '@/lib/integrations/availability' import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' +import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' import { DEFAULT_PERMISSION_GROUP_CONFIG, type PermissionGroupConfig, } from '@/lib/permission-groups/types' +import { useCustomBlockOverlayVersion } from '@/blocks/custom/client-overlay' +import { overlayVisibility } from '@/blocks/visibility/context' import { useUserPermissionConfig } from '@/ee/access-control/hooks/permission-groups' export interface PermissionConfigResult { @@ -26,10 +36,7 @@ export interface PermissionConfigResult { isToolAllowed: (toolId: string) => boolean isInvitationsDisabled: boolean isPublicApiDisabled: boolean -} - -interface AllowedIntegrationsResponse { - allowedIntegrations: string[] | null + integrationAvailability: ReadonlyMap } const allowedIntegrationsKeys = { @@ -38,37 +45,17 @@ const allowedIntegrationsKeys = { } function useAllowedIntegrationsFromEnv() { - return useQuery({ + return useQuery({ queryKey: allowedIntegrationsKeys.env(), - queryFn: async ({ signal }) => { - try { - return await requestJson(getAllowedIntegrationsContract, { signal }) - } catch (error) { - // Treat any auth/server failure as "no env allowlist configured" - // so the UI falls back to the permission-group-driven allowlist. - if (error instanceof ApiClientError) { - return { allowedIntegrations: null } - } - throw error - } - }, + queryFn: ({ signal }) => requestJson(getAllowedIntegrationsContract, { signal }), staleTime: 5 * 60 * 1000, }) } -/** - * Intersects two allowlists. If either is null (unrestricted), returns the other. - * If both are set, returns only items present in both. - */ -function intersectAllowlists(a: string[] | null, b: string[] | null): string[] | null { - if (a === null) return b - if (b === null) return a.map((i) => i.toLowerCase()) - return a.map((i) => i.toLowerCase()).filter((i) => b.includes(i)) -} - export function usePermissionConfig(): PermissionConfigResult { const params = useParams() const workspaceId = typeof params?.workspaceId === 'string' ? params.workspaceId : undefined + const blockOverlayVersion = useCustomBlockOverlayVersion() const { data: permissionData, isLoading: isPermissionLoading } = useUserPermissionConfig(workspaceId) @@ -88,16 +75,38 @@ export function usePermissionConfig(): PermissionConfigResult { const mergedAllowedIntegrations = useMemo(() => { const envAllowlist = envAllowlistData?.allowedIntegrations ?? null - return intersectAllowlists(config.allowedIntegrations, envAllowlist) + return intersectIntegrationAllowlists(config.allowedIntegrations, envAllowlist) }, [config.allowedIntegrations, envAllowlistData]) + const integrationAvailability = useMemo(() => { + const visibility = overlayVisibility() + return new Map( + (envAllowlistData?.integrationAvailability ?? []).map((availability) => [ + availability.type.toLowerCase(), + { + ...availability, + state: resolveIntegrationAvailabilityStateForVisibility(availability, visibility), + }, + ]) + ) + }, [envAllowlistData?.integrationAvailability, blockOverlayVersion]) + const isBlockAllowed = useMemo(() => { return (blockType: string) => { + const normalizedBlockType = blockType.toLowerCase() + const availability = integrationAvailability.get(normalizedBlockType) + if ( + isDeploymentGatedIntegrationType(normalizedBlockType) && + availability && + (availability.state === 'unavailable' || availability.state === 'misconfigured') + ) { + return false + } if (isBlockTypeAccessControlExempt(blockType)) return true if (mergedAllowedIntegrations === null) return true - return mergedAllowedIntegrations.includes(blockType.toLowerCase()) + return mergedAllowedIntegrations.includes(normalizedBlockType) } - }, [mergedAllowedIntegrations]) + }, [integrationAvailability, mergedAllowedIntegrations]) const isProviderAllowed = useMemo(() => { return (providerId: string) => { @@ -123,14 +132,9 @@ export function usePermissionConfig(): PermissionConfigResult { const filterBlocks = useMemo(() => { return (blocks: T[]): T[] => { - if (mergedAllowedIntegrations === null) return blocks - return blocks.filter( - (block) => - isBlockTypeAccessControlExempt(block.type) || - mergedAllowedIntegrations.includes(block.type.toLowerCase()) - ) + return blocks.filter((block) => isBlockAllowed(block.type)) } - }, [mergedAllowedIntegrations]) + }, [isBlockAllowed]) const filterProviders = useMemo(() => { return (providerIds: string[]): string[] => { @@ -167,6 +171,7 @@ export function usePermissionConfig(): PermissionConfigResult { isToolAllowed, isInvitationsDisabled, isPublicApiDisabled, + integrationAvailability, }), [ mergedConfig, @@ -180,6 +185,7 @@ export function usePermissionConfig(): PermissionConfigResult { isToolAllowed, isInvitationsDisabled, isPublicApiDisabled, + integrationAvailability, ] ) } diff --git a/apps/sim/lib/api/contracts/common.ts b/apps/sim/lib/api/contracts/common.ts index d79833499db..c7e18003136 100644 --- a/apps/sim/lib/api/contracts/common.ts +++ b/apps/sim/lib/api/contracts/common.ts @@ -57,6 +57,14 @@ export const getAllowedProvidersContract = defineRouteContract({ }, }) +export const integrationAvailabilitySchema = z.object({ + type: z.string().min(1), + state: z.enum(['ready', 'limited', 'unavailable', 'misconfigured']), + oauthAvailable: z.boolean(), +}) + +export type IntegrationAvailabilityResponse = z.output + export const getAllowedIntegrationsContract = defineRouteContract({ method: 'GET', path: '/api/settings/allowed-integrations', @@ -66,10 +74,15 @@ export const getAllowedIntegrationsContract = defineRouteContract({ // `null` means "no env-derived allowlist" (unrestricted); a non-null // array narrows the visible integrations. allowedIntegrations: z.array(z.string()).nullable(), + integrationAvailability: z.array(integrationAvailabilitySchema), }), }, }) +export type GetAllowedIntegrationsResponse = z.output< + typeof getAllowedIntegrationsContract.response.schema +> + export const getVoiceSettingsContract = defineRouteContract({ method: 'GET', path: '/api/settings/voice', diff --git a/apps/sim/lib/auth/connectors/providers.ts b/apps/sim/lib/auth/connectors/providers.ts index af3a1928354..03ef86cee61 100644 --- a/apps/sim/lib/auth/connectors/providers.ts +++ b/apps/sim/lib/auth/connectors/providers.ts @@ -6,6 +6,7 @@ import { generateId } from '@sim/utils/id' import type { GenericOAuthConfig } from 'better-auth/plugins' import { syntheticConnectorEmail } from '@/lib/auth/connector-email' import { env } from '@/lib/core/config/env' +import { inspectConfiguredOAuthClient } from '@/lib/core/config/env-capabilities.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -90,7 +91,7 @@ interface AttioWorkspaceMemberResponse { * `any`. */ export function buildConnectorProviders(): GenericOAuthConfig[] { - return [ + const providers: GenericOAuthConfig[] = [ { providerId: 'google-email', clientId: env.GOOGLE_CLIENT_ID as string, @@ -2407,4 +2408,8 @@ export function buildConnectorProviders(): GenericOAuthConfig[] { }, }, ] + + return providers.filter( + ({ providerId }) => inspectConfiguredOAuthClient(providerId).state === 'ready' + ) } diff --git a/apps/sim/lib/copilot/chat/payload.test.ts b/apps/sim/lib/copilot/chat/payload.test.ts index 0027554919e..f4c3f802ef5 100644 --- a/apps/sim/lib/copilot/chat/payload.test.ts +++ b/apps/sim/lib/copilot/chat/payload.test.ts @@ -1,15 +1,24 @@ /** * @vitest-environment node */ -import { workflowsUtilsMock } from '@sim/testing' +import { envFlagsMockFns, resetEnvFlagsMock, workflowsUtilsMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockCreateUserToolSchema, mockGetHighestPrioritySubscription, mockTrackChatUpload } = - vi.hoisted(() => ({ - mockCreateUserToolSchema: vi.fn(() => ({ type: 'object', properties: {} })), - mockGetHighestPrioritySubscription: vi.fn(), - mockTrackChatUpload: vi.fn(), - })) +const { + mockCreateUserToolSchema, + mockGetHighestPrioritySubscription, + mockGetUserPermissionConfig, + mockIsIntegrationDeploymentAvailable, + mockIsOAuthServiceDeploymentAvailable, + mockTrackChatUpload, +} = vi.hoisted(() => ({ + mockCreateUserToolSchema: vi.fn(() => ({ type: 'object', properties: {} })), + mockGetHighestPrioritySubscription: vi.fn(), + mockGetUserPermissionConfig: vi.fn(), + mockIsIntegrationDeploymentAvailable: vi.fn(() => true), + mockIsOAuthServiceDeploymentAvailable: vi.fn(() => true), + mockTrackChatUpload: vi.fn(), +})) vi.mock('@/lib/billing/core/subscription', () => ({ getHighestPrioritySubscription: mockGetHighestPrioritySubscription, @@ -65,7 +74,13 @@ vi.mock('@/lib/copilot/block-visibility', () => ({ })) vi.mock('@/lib/copilot/integration-tools', () => ({ - filterExposedIntegrationTools: vi.fn((tools: unknown[]) => tools), + filterExposedIntegrationTools: vi.fn( + ( + tools: Array<{ blockType: string; service: string }>, + _vis: unknown, + isOwnerAllowed: (owner: { blockType: string; service: string }) => boolean + ) => tools.filter((tool) => isOwnerAllowed(tool)) + ), getExposedIntegrationTools: vi.fn(() => [ { toolId: 'gmail_send', @@ -78,6 +93,7 @@ vi.mock('@/lib/copilot/integration-tools', () => ({ }, service: 'gmail', operation: 'send', + blockType: 'gmail', }, { toolId: 'brandfetch_search', @@ -88,6 +104,7 @@ vi.mock('@/lib/copilot/integration-tools', () => ({ }, service: 'brandfetch', operation: 'search', + blockType: 'brandfetch', }, { toolId: 'run_workflow', @@ -98,6 +115,7 @@ vi.mock('@/lib/copilot/integration-tools', () => ({ }, service: 'run', operation: 'workflow', + blockType: 'run', }, ]), })) @@ -110,6 +128,15 @@ vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ trackChatUpload: mockTrackChatUpload, })) +vi.mock('@/lib/integrations/availability.server', () => ({ + isIntegrationDeploymentAvailableForVisibility: mockIsIntegrationDeploymentAvailable, + isOAuthServiceDeploymentAvailable: mockIsOAuthServiceDeploymentAvailable, +})) + +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + getUserPermissionConfig: mockGetUserPermissionConfig, +})) + import { buildCopilotRequestPayload, buildIntegrationToolSchemas, @@ -119,8 +146,12 @@ import { describe('buildIntegrationToolSchemas', () => { beforeEach(() => { vi.clearAllMocks() + resetEnvFlagsMock() clearIntegrationToolSchemaCacheForTests() mockCreateUserToolSchema.mockReturnValue({ type: 'object', properties: {} }) + mockIsIntegrationDeploymentAvailable.mockReturnValue(true) + mockIsOAuthServiceDeploymentAvailable.mockReturnValue(true) + mockGetUserPermissionConfig.mockResolvedValue(null) }) it('appends the email footer prompt for free users', async () => { @@ -197,6 +228,63 @@ describe('buildIntegrationToolSchemas', () => { ) }) + it('removes tools whose canonical exposed block is unavailable', async () => { + mockGetHighestPrioritySubscription.mockResolvedValue({ plan: 'pro', status: 'active' }) + mockIsIntegrationDeploymentAvailable.mockImplementation((blockType: string) => { + return blockType !== 'gmail' + }) + + const toolSchemas = await buildIntegrationToolSchemas('user-deployment-filter') + + expect(toolSchemas.some((tool) => tool.name === 'gmail_send')).toBe(false) + expect(toolSchemas.some((tool) => tool.name === 'brandfetch_search')).toBe(true) + }) + + it('intersects workspace and deployment integration allowlists', async () => { + mockGetHighestPrioritySubscription.mockResolvedValue({ plan: 'pro', status: 'active' }) + mockGetUserPermissionConfig.mockResolvedValue({ + allowedIntegrations: ['gmail', 'brandfetch'], + }) + envFlagsMockFns.getAllowedIntegrationsFromEnv.mockReturnValue(['brandfetch']) + + const toolSchemas = await buildIntegrationToolSchemas( + 'user-intersection', + undefined, + { schemaSurface: 'copilot' }, + 'workspace-1' + ) + + expect(toolSchemas.some((tool) => tool.name === 'gmail_send')).toBe(false) + expect(toolSchemas.some((tool) => tool.name === 'brandfetch_search')).toBe(true) + }) + + it('keeps a limited integration callable without advertising OAuth', async () => { + mockGetHighestPrioritySubscription.mockResolvedValue({ plan: 'pro', status: 'active' }) + mockIsOAuthServiceDeploymentAvailable.mockImplementation( + (providerId: string) => providerId !== 'google-email' + ) + + const toolSchemas = await buildIntegrationToolSchemas('user-limited-integration') + const gmailTool = toolSchemas.find((tool) => tool.name === 'gmail_send') + + expect(gmailTool).toBeDefined() + expect(gmailTool).not.toHaveProperty('oauth') + }) + + it('fails closed when workspace integration permissions cannot be loaded', async () => { + mockGetUserPermissionConfig.mockRejectedValue(new Error('permission backend unavailable')) + + await expect( + buildIntegrationToolSchemas( + 'user-permission-error', + undefined, + { schemaSurface: 'copilot' }, + 'workspace-1' + ) + ).rejects.toThrow('permission backend unavailable') + expect(mockCreateUserToolSchema).not.toHaveBeenCalled() + }) + it('briefly reuses built schemas for the same user and surface', async () => { mockGetHighestPrioritySubscription.mockResolvedValue({ plan: 'pro', status: 'active' }) diff --git a/apps/sim/lib/copilot/chat/payload.ts b/apps/sim/lib/copilot/chat/payload.ts index 6d277dbd56d..3f8c138ab16 100644 --- a/apps/sim/lib/copilot/chat/payload.ts +++ b/apps/sim/lib/copilot/chat/payload.ts @@ -16,10 +16,19 @@ import { getToolEntry } from '@/lib/copilot/tool-executor/router' import { getCopilotToolDescription } from '@/lib/copilot/tools/descriptions' import { encodeVfsSegment } from '@/lib/copilot/vfs/path-utils' import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' -import { isDocSandboxEnabled, isHosted } from '@/lib/core/config/env-flags' +import { EnvCapabilityConfigurationError } from '@/lib/core/config/env-capabilities' +import { + getAllowedIntegrationsFromEnv, + isDocSandboxEnabled, + isHosted, +} from '@/lib/core/config/env-flags' +import { + isIntegrationDeploymentAvailableForVisibility, + isOAuthServiceDeploymentAvailable, +} from '@/lib/integrations/availability.server' +import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' import { trackChatUpload } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import { buildArchiveExtractGuidance, isArchiveFileName } from '@/lib/uploads/utils/file-utils' -import { stripVersionSuffix } from '@/tools/utils' const logger = createLogger('CopilotChatPayload') const INTEGRATION_TOOL_SCHEMA_CACHE_TTL_MS = 5_000 @@ -187,6 +196,19 @@ async function buildIntegrationToolSchemasUncached( ): Promise { const reqLogger = logger.withMetadata({ messageId }) const integrationTools: ToolSchema[] = [] + let allowedIntegrations = getAllowedIntegrationsFromEnv() + if (workspaceId) { + const { getUserPermissionConfig } = await import('@/ee/access-control/utils/permission-check') + const permissionConfig = await getUserPermissionConfig(userId, workspaceId) + allowedIntegrations = intersectIntegrationAllowlists( + permissionConfig?.allowedIntegrations ?? null, + allowedIntegrations + ) + } + const allowedIntegrationTypes = allowedIntegrations + ? new Set(allowedIntegrations.map((integration) => integration.toLowerCase())) + : null + try { const { createUserToolSchema } = await import('@/tools/params') let shouldAppendEmailTagline = false @@ -201,46 +223,16 @@ async function buildIntegrationToolSchemasUncached( }) } - let allowedIntegrations: Set | null = null - let toolIdToBlockType: Map | null = null - if (workspaceId) { - try { - const [{ getUserPermissionConfig }, { getAllBlocks }] = await Promise.all([ - import('@/ee/access-control/utils/permission-check'), - import('@/blocks/registry'), - ]) - const permissionConfig = await getUserPermissionConfig(userId, workspaceId) - if (permissionConfig?.allowedIntegrations) { - allowedIntegrations = new Set( - permissionConfig.allowedIntegrations.map((i) => i.toLowerCase()) - ) - toolIdToBlockType = new Map() - for (const blockConfig of getAllBlocks()) { - const access = blockConfig.tools?.access - if (!access) continue - for (const toolId of access) { - toolIdToBlockType.set(stripVersionSuffix(toolId), blockConfig.type.toLowerCase()) - } - } - } - } catch (error) { - reqLogger.warn('Failed to load permission config for tool schema filter', { - userId, - workspaceId, - error: toError(error).message, - }) - } - } - - const exposedTools = filterExposedIntegrationTools(getExposedIntegrationTools(), vis) + const exposedTools = filterExposedIntegrationTools( + getExposedIntegrationTools(), + vis, + (owner) => + isIntegrationDeploymentAvailableForVisibility(owner.blockType, vis) && + (allowedIntegrationTypes === null || + allowedIntegrationTypes.has(owner.blockType.toLowerCase())) + ) for (const { toolId, config: toolConfig, service, operation } of exposedTools) { try { - if (allowedIntegrations && toolIdToBlockType) { - const owningBlock = toolIdToBlockType.get(stripVersionSuffix(toolId)) - if (owningBlock && !allowedIntegrations.has(owningBlock)) { - continue - } - } const userSchema = createUserToolSchema(toolConfig, { surface: options.schemaSurface, // On hosted deployments the executor injects hosted keys server-side, @@ -272,14 +264,16 @@ async function buildIntegrationToolSchemasUncached( defer_loading: true, executeLocally: catalogEntry?.clientExecutable === true || catalogEntry?.route === 'client', - ...(toolConfig.oauth?.required && { - oauth: { - required: true, - provider: toolConfig.oauth.provider, - }, - }), + ...(toolConfig.oauth?.required && + isOAuthServiceDeploymentAvailable(toolConfig.oauth.provider) && { + oauth: { + required: true, + provider: toolConfig.oauth.provider, + }, + }), }) } catch (toolError) { + if (toolError instanceof EnvCapabilityConfigurationError) throw toolError logger.warn( messageId ? `Failed to build schema for tool, skipping [messageId:${messageId}]` @@ -292,6 +286,7 @@ async function buildIntegrationToolSchemasUncached( } } } catch (error) { + if (error instanceof EnvCapabilityConfigurationError) throw error logger.warn( messageId ? `Failed to build tool schemas [messageId:${messageId}]` diff --git a/apps/sim/lib/copilot/chat/process-contents.test.ts b/apps/sim/lib/copilot/chat/process-contents.test.ts index 588eca2c376..0e9b85a26f3 100644 --- a/apps/sim/lib/copilot/chat/process-contents.test.ts +++ b/apps/sim/lib/copilot/chat/process-contents.test.ts @@ -10,15 +10,36 @@ import { } from '@/lib/copilot/chat/selection-context' import type { ChatContext } from '@/stores/panel' -const { discoverServerTools, getSkillById, getWorkspaceFile, getTableById, getRowsByIds } = - vi.hoisted(() => ({ - discoverServerTools: vi.fn(), - getSkillById: vi.fn(), - getWorkspaceFile: vi.fn(), - getTableById: vi.fn(), - getRowsByIds: vi.fn(), - })) - +const { + discoverServerTools, + getBlock, + getBlockRegistry, + getSkillById, + getUserPermissionConfig, + getWorkspaceFile, + getTableById, + getRowsByIds, + getBlockVisibilityForCopilot, + isIntegrationDeploymentAvailable, +} = vi.hoisted(() => ({ + discoverServerTools: vi.fn(), + getBlock: vi.fn(), + getBlockRegistry: vi.fn(), + getSkillById: vi.fn(), + getUserPermissionConfig: vi.fn(), + getWorkspaceFile: vi.fn(), + getTableById: vi.fn(), + getRowsByIds: vi.fn(), + getBlockVisibilityForCopilot: vi.fn(async () => null), + isIntegrationDeploymentAvailable: vi.fn(() => true), +})) + +vi.mock('@/blocks/registry', () => ({ getBlock, getBlockRegistry })) +vi.mock('@/lib/copilot/block-visibility', () => ({ getBlockVisibilityForCopilot })) +vi.mock('@/ee/access-control/utils/permission-check', () => ({ getUserPermissionConfig })) +vi.mock('@/lib/integrations/availability.server', () => ({ + isIntegrationDeploymentAvailableForVisibility: isIntegrationDeploymentAvailable, +})) vi.mock('@/lib/workflows/skills/operations', () => ({ getSkillById })) vi.mock('@/lib/mcp/service', () => ({ mcpService: { discoverServerTools } })) vi.mock('@/lib/uploads/contexts/workspace/workspace-file-manager', () => ({ getWorkspaceFile })) @@ -32,6 +53,42 @@ vi.mock('@/lib/table/rows/service', () => ({ getRowsByIds })) import { processContextsServer } from './process-contents' +describe('processContextsServer - block contexts', () => { + beforeEach(() => { + vi.clearAllMocks() + const blocks = { + start_trigger: { type: 'start_trigger', hideFromToolbar: false }, + slack: { type: 'slack', hideFromToolbar: false }, + notion: { type: 'notion', hideFromToolbar: false }, + } + getBlockRegistry.mockReturnValue(blocks) + getBlock.mockImplementation((type: string) => blocks[type as keyof typeof blocks]) + getUserPermissionConfig.mockResolvedValue({ allowedIntegrations: ['slack'] }) + isIntegrationDeploymentAvailable.mockReturnValue(true) + }) + + it('keeps access-control-exempt blocks while filtering non-exempt integrations', async () => { + const result = await processContextsServer( + [ + { kind: 'blocks', blockIds: ['start_trigger'], label: 'Start' } as ChatContext, + { kind: 'blocks', blockIds: ['notion'], label: 'Notion' } as ChatContext, + ], + 'user-1', + 'hello', + 'workspace-1' + ) + + expect(result).toEqual([ + { + type: 'blocks', + tag: '@Start', + content: '', + path: 'components/blocks/start_trigger.json', + }, + ]) + }) +}) + describe('processContextsServer - skill contexts', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/apps/sim/lib/copilot/chat/process-contents.ts b/apps/sim/lib/copilot/chat/process-contents.ts index b80c7060705..28f6543841d 100644 --- a/apps/sim/lib/copilot/chat/process-contents.ts +++ b/apps/sim/lib/copilot/chat/process-contents.ts @@ -6,6 +6,7 @@ import { getActiveWorkflowRecord, } from '@sim/platform-authz/workflow' import { and, eq, isNull } from 'drizzle-orm' +import { getBlockVisibilityForCopilot } from '@/lib/copilot/block-visibility' import { MAX_TABLE_SELECTION_CONTENT_LENGTH, safeBrowserSelectionUrl, @@ -22,11 +23,15 @@ import { encodeVfsPathSegments, encodeVfsSegment, } from '@/lib/copilot/vfs/path-utils' +import { EnvCapabilityConfigurationError } from '@/lib/core/config/env-capabilities' import { getAllowedIntegrationsFromEnv } from '@/lib/core/config/env-flags' +import { isIntegrationDeploymentAvailableForVisibility } from '@/lib/integrations/availability.server' import { toOverview } from '@/lib/logs/log-views' import type { TraceSpan } from '@/lib/logs/types' import { mcpService } from '@/lib/mcp/service' import { createMcpToolId } from '@/lib/mcp/utils' +import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' +import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' import { getColumnId } from '@/lib/table/column-keys' import { getRowsByIds } from '@/lib/table/rows/service' import { getTableById } from '@/lib/table/service' @@ -598,11 +603,23 @@ async function processBlockMetadata( workspaceId?: string ): Promise { try { - const permissionConfig = - userId && workspaceId ? await getUserPermissionConfig(userId, workspaceId) : null - const allowedIntegrations = - permissionConfig?.allowedIntegrations ?? getAllowedIntegrationsFromEnv() - if (allowedIntegrations != null && !allowedIntegrations.includes(blockId.toLowerCase())) { + const [permissionConfig, visibility] = await Promise.all([ + userId && workspaceId ? getUserPermissionConfig(userId, workspaceId) : null, + userId ? getBlockVisibilityForCopilot(userId, workspaceId) : null, + ]) + const allowedIntegrations = intersectIntegrationAllowlists( + permissionConfig?.allowedIntegrations ?? null, + getAllowedIntegrationsFromEnv() + ) + if (!isIntegrationDeploymentAvailableForVisibility(blockId, visibility)) { + logger.debug('Block unavailable for this deployment', { blockId }) + return null + } + if ( + allowedIntegrations != null && + !isBlockTypeAccessControlExempt(blockId) && + !allowedIntegrations.includes(blockId.toLowerCase()) + ) { logger.debug('Block not allowed by integration allowlist', { blockId, userId }) return null } @@ -615,6 +632,7 @@ async function processBlockMetadata( return { type: 'blocks', tag, content: '', path: canonicalBlockVfsPath(blockId) } } catch (error) { + if (error instanceof EnvCapabilityConfigurationError) throw error logger.error('Error processing block metadata', { blockId, error }) return null } diff --git a/apps/sim/lib/copilot/integration-tools.test.ts b/apps/sim/lib/copilot/integration-tools.test.ts index d55e6f15f0a..bbfb423aecc 100644 --- a/apps/sim/lib/copilot/integration-tools.test.ts +++ b/apps/sim/lib/copilot/integration-tools.test.ts @@ -10,7 +10,7 @@ vi.mock('@/blocks/registry-maps', () => ({ tools: { access: ['svc_send_v2'] }, }, // Preview successor sharing the released block's tools (the slack/slack_v2 - // paradigm) — must not become the owner of the shared tools. + // paradigm) — both owners must remain available for projection. svc_v2: { type: 'svc_v2', preview: true, @@ -49,6 +49,7 @@ describe('getExposedIntegrationTools', () => { expect(send).toBeDefined() expect(send?.blockType).toBe('svc') expect(send?.preview).toBeFalsy() + expect(send?.owners.map((owner) => owner.blockType)).toEqual(['svc', 'svc_v2']) }) it('exposes shared tools to viewers without the preview reveal, but not preview-only tools', () => { @@ -57,6 +58,23 @@ describe('getExposedIntegrationTools', () => { expect(visible.some((t) => t.toolId === 'newsvc_do_v1')).toBe(false) }) + it('uses a revealed preview owner when the released owner is unavailable', () => { + const vis = { + revealed: new Set(['svc_v2']), + disabled: new Set(), + previewTagged: new Set(['svc_v2']), + } + const visible = filterExposedIntegrationTools( + getExposedIntegrationTools(), + vis, + (owner) => owner.blockType !== 'svc' + ) + const send = visible.find((tool) => tool.toolId === 'svc_send_v2') + + expect(send?.blockType).toBe('svc_v2') + expect(send?.preview).toBe(true) + }) + it('exposes only the latest version of each tool', () => { const exposed = getExposedIntegrationTools() expect(exposed.some((t) => t.toolId === 'svc_send_v1')).toBe(false) diff --git a/apps/sim/lib/copilot/integration-tools.ts b/apps/sim/lib/copilot/integration-tools.ts index 79b92e53bb5..aafadc452b7 100644 --- a/apps/sim/lib/copilot/integration-tools.ts +++ b/apps/sim/lib/copilot/integration-tools.ts @@ -5,7 +5,13 @@ import { tools as toolRegistry } from '@/tools/registry' import type { ToolConfig } from '@/tools/types' import { getLatestVersionTools, stripVersionSuffix } from '@/tools/utils' -export interface ExposedIntegrationTool { +export interface ExposedIntegrationToolOwner { + service: string + blockType: string + preview?: boolean +} + +export interface ExposedIntegrationTool extends ExposedIntegrationToolOwner { /** * Full registry tool id — also the agent-callable id and the schema `id` * field (e.g. gmail_read_v2). No stripping: discovery, the schema id, and the @@ -21,6 +27,8 @@ export interface ExposedIntegrationTool { blockType: string /** Owning block's static `preview` marker, for the per-viewer filter. */ preview?: boolean + /** Every visible block that declares this tool, including preview successors. */ + owners: readonly ExposedIntegrationToolOwner[] } let cached: ExposedIntegrationTool[] | null = null @@ -46,7 +54,7 @@ export function getExposedIntegrationTools(): ExposedIntegrationTool[] { // Map the tool ids each visible block exposes (both the raw id and its // version-stripped base name) to that block's service directory + type. - const toolToBlock = new Map() + const toolToBlocks = new Map() for (const block of Object.values(BLOCK_REGISTRY)) { if (block.hideFromToolbar) continue if (!block.tools?.access) continue @@ -54,13 +62,14 @@ export function getExposedIntegrationTools(): ExposedIntegrationTool[] { const owner = { service, blockType: block.type, preview: block.preview } for (const toolId of block.tools.access) { for (const key of [toolId, stripVersionSuffix(toolId)]) { - // A preview block must not steal ownership of tools it shares with a - // released block (e.g. slack_v2 spreads slack's tools.access), or the - // per-viewer filter would hide those tools from everyone without the - // preview reveal. - const existing = toolToBlock.get(key) - if (existing && !existing.preview && owner.preview) continue - toolToBlock.set(key, owner) + const owners = toolToBlocks.get(key) + if (owners) { + if (!owners.some((existing) => existing.blockType === owner.blockType)) { + owners.push(owner) + } + } else { + toolToBlocks.set(key, [owner]) + } } } } @@ -69,8 +78,9 @@ export function getExposedIntegrationTools(): ExposedIntegrationTool[] { const seen = new Set() for (const [toolId, config] of Object.entries(getLatestVersionTools(toolRegistry))) { const baseName = stripVersionSuffix(toolId) - const owner = toolToBlock.get(toolId) ?? toolToBlock.get(baseName) - if (!owner) continue + const owners = toolToBlocks.get(toolId) ?? toolToBlocks.get(baseName) + if (!owners || owners.length === 0) continue + const owner = owners.find((candidate) => !candidate.preview) ?? owners[0] if (seen.has(baseName)) continue seen.add(baseName) const prefix = `${owner.service}_` @@ -82,6 +92,7 @@ export function getExposedIntegrationTools(): ExposedIntegrationTool[] { operation, blockType: owner.blockType, preview: owner.preview, + owners, }) } @@ -97,11 +108,17 @@ export function getExposedIntegrationTools(): ExposedIntegrationTool[] { */ export function filterExposedIntegrationTools( tools: ExposedIntegrationTool[], - vis: BlockVisibilityState | null + vis: BlockVisibilityState | null, + isOwnerAllowed: (owner: ExposedIntegrationToolOwner) => boolean = () => true ): ExposedIntegrationTool[] { - return tools.filter( - (tool) => !isHiddenUnder(vis, { type: tool.blockType, preview: tool.preview }) - ) + return tools.flatMap((tool) => { + const owner = tool.owners.find( + (candidate) => + !isHiddenUnder(vis, { type: candidate.blockType, preview: candidate.preview }) && + isOwnerAllowed(candidate) + ) + return owner ? [{ ...tool, ...owner }] : [] + }) } /** Test-only: clears the memoized set so registry changes are picked up. */ diff --git a/apps/sim/lib/copilot/tools/handlers/integration-tools.ts b/apps/sim/lib/copilot/tools/handlers/integration-tools.ts index 0a4763bb39a..f05edb4dcf1 100644 --- a/apps/sim/lib/copilot/tools/handlers/integration-tools.ts +++ b/apps/sim/lib/copilot/tools/handlers/integration-tools.ts @@ -4,6 +4,10 @@ import { getExposedIntegrationTools, } from '@/lib/copilot/integration-tools' import type { ExecutionContext, ToolCallResult } from '@/lib/copilot/request/types' +import { getAllowedIntegrationsFromEnv } from '@/lib/core/config/env-flags' +import { isIntegrationDeploymentAvailableForVisibility } from '@/lib/integrations/availability.server' +import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' +import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check' import { stripVersionSuffix } from '@/tools/utils' export async function executeListIntegrationTools( @@ -18,7 +22,20 @@ export async function executeListIntegrationTools( // The exposed set is the ungated universe — project it for this viewer so // gated (preview / kill-switched) integrations stay undiscoverable. const vis = await getBlockVisibilityForCopilot(context.userId, context.workspaceId) - const all = filterExposedIntegrationTools(getExposedIntegrationTools(), vis) + const permissionConfig = context.workspaceId + ? await getUserPermissionConfig(context.userId, context.workspaceId) + : null + const allowedIntegrations = intersectIntegrationAllowlists( + permissionConfig?.allowedIntegrations ?? null, + getAllowedIntegrationsFromEnv() + ) + const all = filterExposedIntegrationTools( + getExposedIntegrationTools(), + vis, + (owner) => + isIntegrationDeploymentAvailableForVisibility(owner.blockType, vis) && + (allowedIntegrations === null || allowedIntegrations.includes(owner.blockType.toLowerCase())) + ) const service = stripVersionSuffix(raw.toLowerCase()) const matches = all.filter((tool) => tool.service === service) diff --git a/apps/sim/lib/copilot/tools/handlers/oauth.test.ts b/apps/sim/lib/copilot/tools/handlers/oauth.test.ts index 1168528e56a..77ff5a9922d 100644 --- a/apps/sim/lib/copilot/tools/handlers/oauth.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/oauth.test.ts @@ -3,9 +3,16 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockEnsureWorkspaceAccess, mockGetCredentialActorContext } = vi.hoisted(() => ({ +const { + mockEnsureWorkspaceAccess, + mockGetCredentialActorContext, + mockIsOAuthServiceDeploymentAvailable, + mockGetUserPermissionConfig, +} = vi.hoisted(() => ({ mockEnsureWorkspaceAccess: vi.fn(), mockGetCredentialActorContext: vi.fn(), + mockIsOAuthServiceDeploymentAvailable: vi.fn(() => true), + mockGetUserPermissionConfig: vi.fn(), })) vi.mock('@/lib/copilot/tools/handlers/access', () => ({ @@ -16,12 +23,30 @@ vi.mock('@/lib/credentials/access', () => ({ getCredentialActorContext: mockGetCredentialActorContext, })) +vi.mock('@/lib/integrations/availability.server', () => ({ + isOAuthServiceDeploymentAvailable: mockIsOAuthServiceDeploymentAvailable, +})) + +vi.mock('@/lib/core/config/env-flags', () => ({ + getAllowedIntegrationsFromEnv: vi.fn(() => null), +})) + +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + getUserPermissionConfig: mockGetUserPermissionConfig, +})) + vi.mock('@/lib/oauth/utils', () => ({ getAllOAuthServices: vi.fn(() => [ - { providerId: 'google-email', name: 'Gmail' }, - { providerId: 'slack', name: 'Slack' }, - { providerId: 'trello', name: 'Trello' }, - { providerId: 'shopify', name: 'Shopify' }, + { serviceId: 'gmail', providerId: 'google-email', name: 'Gmail', authType: 'oauth' }, + { serviceId: 'slack', providerId: 'slack', name: 'Slack', authType: 'oauth' }, + { serviceId: 'trello', providerId: 'trello', name: 'Trello', authType: 'oauth' }, + { serviceId: 'shopify', providerId: 'shopify', name: 'Shopify', authType: 'oauth' }, + { + serviceId: 'claude-platform', + providerId: 'claude-platform', + name: 'Claude Platform', + authType: 'service_account', + }, ]), })) @@ -69,6 +94,8 @@ describe('executeOAuthGetAuthLink', () => { vi.clearAllMocks() process.env.NEXT_PUBLIC_APP_URL = BASE_URL mockEnsureWorkspaceAccess.mockResolvedValue(WORKSPACE_ACCESS) + mockIsOAuthServiceDeploymentAvailable.mockReturnValue(true) + mockGetUserPermissionConfig.mockResolvedValue(null) }) describe('connect (no credentialId)', () => { @@ -82,6 +109,31 @@ describe('executeOAuthGetAuthLink', () => { expect(url.searchParams.get('credentialId')).toBeNull() expect(mockGetCredentialActorContext).not.toHaveBeenCalled() }) + + it('rejects a provider whose OAuth client is not configured', async () => { + mockIsOAuthServiceDeploymentAvailable.mockReturnValue(false) + + const result = await executeOAuthGetAuthLink({ providerName: 'google-email' }, context) + + expect(result.success).toBe(false) + expect(result.error).toContain('not configured for this deployment') + }) + + it('rejects a provider disallowed for the workspace member', async () => { + mockGetUserPermissionConfig.mockResolvedValue({ allowedIntegrations: ['slack'] }) + + const result = await executeOAuthGetAuthLink({ providerName: 'google-email' }, context) + + expect(result.success).toBe(false) + expect(result.error).toContain('not allowed for this workspace member') + }) + + it('does not treat service-account-only metadata as OAuth', async () => { + const result = await executeOAuthGetAuthLink({ providerName: 'Claude Platform' }, context) + + expect(result.success).toBe(false) + expect(result.error).toContain('not found') + }) }) describe('reconnect (credentialId passed)', () => { @@ -213,6 +265,8 @@ describe('executeOAuthGetAuthLink service account rejection', () => { vi.clearAllMocks() process.env.NEXT_PUBLIC_APP_URL = BASE_URL mockEnsureWorkspaceAccess.mockResolvedValue(WORKSPACE_ACCESS) + mockIsOAuthServiceDeploymentAvailable.mockReturnValue(true) + mockGetUserPermissionConfig.mockResolvedValue(null) }) /** diff --git a/apps/sim/lib/copilot/tools/handlers/oauth.ts b/apps/sim/lib/copilot/tools/handlers/oauth.ts index e392833e6e8..1c7936d8980 100644 --- a/apps/sim/lib/copilot/tools/handlers/oauth.ts +++ b/apps/sim/lib/copilot/tools/handlers/oauth.ts @@ -1,11 +1,16 @@ import { toError } from '@sim/utils/errors' import type { ExecutionContext, ToolCallResult } from '@/lib/copilot/request/types' import { ensureWorkspaceAccess } from '@/lib/copilot/tools/handlers/access' +import { getAllowedIntegrationsFromEnv } from '@/lib/core/config/env-flags' import { getBaseUrl } from '@/lib/core/utils/urls' import { getCredentialActorContext } from '@/lib/credentials/access' import { isServiceAccountProviderId } from '@/lib/credentials/service-account-provider-ids' +import { isOAuthServiceAllowedByIntegrationTypes } from '@/lib/integrations/availability' +import { isOAuthServiceDeploymentAvailable } from '@/lib/integrations/availability.server' import { getAllOAuthServices } from '@/lib/oauth/utils' +import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' import type { WorkspaceAccess } from '@/lib/workspaces/permissions/utils' +import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check' export async function executeOAuthGetAuthLink( rawParams: Record, @@ -43,12 +48,21 @@ export async function executeOAuthGetAuthLink( context.userId, 'write' ) + const permissionConfig = await getUserPermissionConfig(context.userId, context.workspaceId) + const configuredAllowedIntegrations = intersectIntegrationAllowlists( + permissionConfig?.allowedIntegrations ?? null, + getAllowedIntegrationsFromEnv() + ) + const allowedIntegrationTypes = configuredAllowedIntegrations + ? new Set(configuredAllowedIntegrations.map((type) => type.toLowerCase())) + : null const result = await generateOAuthLink( context.workspaceId, context.workflowId, context.chatId, providerName, baseUrl, + allowedIntegrationTypes, credentialId ? { credentialId, userId: context.userId, workspaceAccess } : undefined ) const action = credentialId ? 'reconnect' : 'connect' @@ -117,13 +131,14 @@ async function generateOAuthLink( chatId: string | undefined, providerName: string, baseUrl: string, + allowedIntegrationTypes: ReadonlySet | null, reconnect?: { credentialId: string; userId: string; workspaceAccess: WorkspaceAccess } ): Promise<{ url: string; providerId: string; serviceName: string }> { if (!workspaceId) { throw new Error('workspaceId is required to generate an OAuth link') } - const allServices = getAllOAuthServices() + const allServices = getAllOAuthServices().filter((service) => service.authType === 'oauth') const normalizedInput = providerName.toLowerCase().trim() const matched = @@ -144,6 +159,12 @@ async function generateOAuthLink( } const { providerId, name: serviceName } = matched + if (!isOAuthServiceAllowedByIntegrationTypes(matched.serviceId, allowedIntegrationTypes)) { + throw new Error(`${serviceName} is not allowed for this workspace member`) + } + if (!isOAuthServiceDeploymentAvailable(providerId)) { + throw new Error(`${serviceName} OAuth is not configured for this deployment`) + } if (reconnect) { if (providerId === 'trello' || providerId === 'shopify') { diff --git a/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-tool.test.ts b/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-tool.test.ts index f8d84b3fe93..f286dc7a0c5 100644 --- a/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-tool.test.ts +++ b/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-tool.test.ts @@ -2,15 +2,50 @@ * @vitest-environment node */ -import { describe, expect, it } from 'vitest' -import { computeBlockLevelInputs } from '@/lib/copilot/tools/server/blocks/get-blocks-metadata-tool' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetUserPermissionConfig, mockIsIntegrationDeploymentAvailable } = vi.hoisted(() => ({ + mockGetUserPermissionConfig: vi.fn(), + mockIsIntegrationDeploymentAvailable: vi.fn(() => true), +})) + +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + getUserPermissionConfig: mockGetUserPermissionConfig, +})) + +vi.mock('@/lib/integrations/availability.server', () => ({ + isIntegrationDeploymentAvailableForVisibility: mockIsIntegrationDeploymentAvailable, +})) + +import { + computeBlockLevelInputs, + getBlocksMetadataServerTool, +} from '@/lib/copilot/tools/server/blocks/get-blocks-metadata-tool' import { MothershipBlock } from '@/blocks/blocks/mothership' describe('get blocks metadata', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetUserPermissionConfig.mockResolvedValue({ allowedIntegrations: ['slack'] }) + mockIsIntegrationDeploymentAvailable.mockReturnValue(true) + }) + it('omits server-only Mothership policy inputs from block metadata definitions', () => { const definitions = computeBlockLevelInputs(MothershipBlock) expect(definitions).not.toHaveProperty('secretScope') expect(definitions).not.toHaveProperty('mountedSecrets') }) + + it('keeps access-control-exempt and special blocks under a restrictive allowlist', async () => { + const result = await getBlocksMetadataServerTool.execute( + { blockIds: ['start_trigger', 'loop', 'slack', 'notion'] }, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.metadata).toHaveProperty('start_trigger') + expect(result.metadata).toHaveProperty('loop') + expect(result.metadata).toHaveProperty('slack') + expect(result.metadata).not.toHaveProperty('notion') + }) }) diff --git a/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-tool.ts b/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-tool.ts index 5a7c1f2d3a9..9c46dc976cc 100644 --- a/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-tool.ts +++ b/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-tool.ts @@ -6,7 +6,10 @@ import { z } from 'zod' import { getCopilotToolDescription } from '@/lib/copilot/tools/descriptions' import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool' import { getAllowedIntegrationsFromEnv, isHosted } from '@/lib/core/config/env-flags' +import { isIntegrationDeploymentAvailableForVisibility } from '@/lib/integrations/availability.server' import { getServiceAccountProviderForProviderId } from '@/lib/oauth/utils' +import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' +import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' import { isCustomBlockType } from '@/blocks/custom/build-config' import { getBlock } from '@/blocks/registry' import { AuthMode, type BlockConfig, isHiddenFromDisplay } from '@/blocks/types' @@ -124,20 +127,32 @@ export const getBlocksMetadataServerTool: BaseServerTool< context?.userId && context?.workspaceId ? await getUserPermissionConfig(context.userId, context.workspaceId) : null - const allowedIntegrations = - permissionConfig?.allowedIntegrations ?? getAllowedIntegrationsFromEnv() + const allowedIntegrations = intersectIntegrationAllowlists( + permissionConfig?.allowedIntegrations ?? null, + getAllowedIntegrationsFromEnv() + ) + const visibility = overlayVisibility() const result: Record = {} for (const blockId of blockIds || []) { - if (allowedIntegrations != null && !allowedIntegrations.includes(blockId.toLowerCase())) { + const specialBlock = SPECIAL_BLOCKS_METADATA[blockId] + if (!isIntegrationDeploymentAvailableForVisibility(blockId, visibility)) { + logger.debug('Block unavailable for this deployment', { blockId }) + continue + } + if ( + allowedIntegrations != null && + !specialBlock && + !isBlockTypeAccessControlExempt(blockId) && + !allowedIntegrations.includes(blockId.toLowerCase()) + ) { logger.debug('Block not allowed by permission group', { blockId }) continue } let metadata: any - if (SPECIAL_BLOCKS_METADATA[blockId]) { - const specialBlock = SPECIAL_BLOCKS_METADATA[blockId] + if (specialBlock) { const { commonParameters, operationParameters } = splitParametersByOperation( specialBlock.subBlocks || [], specialBlock.inputs || {} @@ -170,7 +185,7 @@ export const getBlocksMetadataServerTool: BaseServerTool< // explicitly: unrevealed preview blocks and kill-switched types stay // out of the agent's metadata (the router wraps this tool in // withBlockVisibility). - if (isHiddenUnder(overlayVisibility(), blockConfig)) { + if (isHiddenUnder(visibility, blockConfig)) { logger.debug('Skipping block gated by visibility', { blockId }) continue } diff --git a/apps/sim/lib/copilot/tools/server/blocks/get-trigger-blocks.test.ts b/apps/sim/lib/copilot/tools/server/blocks/get-trigger-blocks.test.ts new file mode 100644 index 00000000000..8f699d66183 --- /dev/null +++ b/apps/sim/lib/copilot/tools/server/blocks/get-trigger-blocks.test.ts @@ -0,0 +1,56 @@ +/** + * @vitest-environment node + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockGetAllBlocks, + mockGetBlock, + mockGetUserPermissionConfig, + mockIsIntegrationDeploymentAvailable, +} = vi.hoisted(() => ({ + mockGetAllBlocks: vi.fn(), + mockGetBlock: vi.fn(), + mockGetUserPermissionConfig: vi.fn(), + mockIsIntegrationDeploymentAvailable: vi.fn(() => true), +})) + +vi.mock('@/blocks/registry', () => ({ + getAllBlocks: mockGetAllBlocks, + getBlock: mockGetBlock, +})) + +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + getUserPermissionConfig: mockGetUserPermissionConfig, +})) + +vi.mock('@/lib/integrations/availability.server', () => ({ + isIntegrationDeploymentAvailableForVisibility: mockIsIntegrationDeploymentAvailable, +})) + +import { getTriggerBlocksServerTool } from '@/lib/copilot/tools/server/blocks/get-trigger-blocks' + +describe('get trigger blocks', () => { + beforeEach(() => { + vi.clearAllMocks() + const blocks = [ + { type: 'start_trigger', category: 'triggers', subBlocks: [] }, + { type: 'slack', category: 'tools', triggerAllowed: true, subBlocks: [] }, + { type: 'notion', category: 'tools', triggerAllowed: true, subBlocks: [] }, + ] + mockGetAllBlocks.mockReturnValue(blocks) + mockGetBlock.mockImplementation((type: string) => blocks.find((block) => block.type === type)) + mockGetUserPermissionConfig.mockResolvedValue({ allowedIntegrations: ['slack'] }) + mockIsIntegrationDeploymentAvailable.mockReturnValue(true) + }) + + it('keeps the start trigger while filtering non-exempt integrations', async () => { + const result = await getTriggerBlocksServerTool.execute( + {}, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.triggerBlockIds).toEqual(['slack', 'start_trigger']) + }) +}) diff --git a/apps/sim/lib/copilot/tools/server/blocks/get-trigger-blocks.ts b/apps/sim/lib/copilot/tools/server/blocks/get-trigger-blocks.ts index 6a12632149b..d27bd205015 100644 --- a/apps/sim/lib/copilot/tools/server/blocks/get-trigger-blocks.ts +++ b/apps/sim/lib/copilot/tools/server/blocks/get-trigger-blocks.ts @@ -2,7 +2,11 @@ import { createLogger } from '@sim/logger' import { z } from 'zod' import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool' import { getAllowedIntegrationsFromEnv } from '@/lib/core/config/env-flags' +import { isIntegrationDeploymentAvailableForVisibility } from '@/lib/integrations/availability.server' +import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' +import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' import { getAllBlocks } from '@/blocks/registry' +import { overlayVisibility } from '@/blocks/visibility/context' import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check' export const GetTriggerBlocksInput = z.object({}) @@ -25,15 +29,23 @@ export const getTriggerBlocksServerTool: BaseServerTool< context?.userId && context?.workspaceId ? await getUserPermissionConfig(context.userId, context.workspaceId) : null - const allowedIntegrations = - permissionConfig?.allowedIntegrations ?? getAllowedIntegrationsFromEnv() + const allowedIntegrations = intersectIntegrationAllowlists( + permissionConfig?.allowedIntegrations ?? null, + getAllowedIntegrationsFromEnv() + ) + const visibility = overlayVisibility() const triggerBlockIds: string[] = [] for (const blockConfig of getAllBlocks()) { const blockType = blockConfig.type if (blockConfig.hideFromToolbar) continue - if (allowedIntegrations != null && !allowedIntegrations.includes(blockType.toLowerCase())) + if (!isIntegrationDeploymentAvailableForVisibility(blockType, visibility)) continue + if ( + allowedIntegrations != null && + !isBlockTypeAccessControlExempt(blockType) && + !allowedIntegrations.includes(blockType.toLowerCase()) + ) continue if (blockConfig.category === 'triggers') { diff --git a/apps/sim/lib/copilot/tools/server/router.ts b/apps/sim/lib/copilot/tools/server/router.ts index e1067314263..323738741b6 100644 --- a/apps/sim/lib/copilot/tools/server/router.ts +++ b/apps/sim/lib/copilot/tools/server/router.ts @@ -73,14 +73,16 @@ const logger = createLogger('ServerToolRouter') const CUSTOM_BLOCK_OVERLAY_TOOLS = new Set(['edit_workflow', 'get_blocks_metadata']) /** - * DISCOVERY tools that must run inside the viewer's block-visibility context so - * gated (preview / kill-switched) blocks disappear from what the agent can - * list. Deliberately a DIFFERENT set from {@link CUSTOM_BLOCK_OVERLAY_TOOLS}: - * `edit_workflow` is excluded because its registry use is functional - * (find-by-type over clones, never a discovery listing) and gating it would - * only risk leaking display projections into persisted state. + * Discovery tools that consume the viewer's block-visibility context to hide + * gated blocks and credentials. `edit_workflow` establishes a narrower scope + * around operation validation after it resolves the workflow's actual + * workspace. */ -const VISIBILITY_GATED_TOOLS = new Set(['get_blocks_metadata', 'get_trigger_blocks']) +const VISIBILITY_GATED_TOOLS = new Set([ + 'get_blocks_metadata', + 'get_credentials', + 'get_trigger_blocks', +]) const WRITE_ACTIONS: Record = { [KnowledgeBase.id]: [ diff --git a/apps/sim/lib/copilot/tools/server/user/get-credentials.test.ts b/apps/sim/lib/copilot/tools/server/user/get-credentials.test.ts index a57b7f4699c..133dd93704b 100644 --- a/apps/sim/lib/copilot/tools/server/user/get-credentials.test.ts +++ b/apps/sim/lib/copilot/tools/server/user/get-credentials.test.ts @@ -17,9 +17,22 @@ import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const SECRET_ACCESS_TOKEN = 'ya29.a0SECRET_GOOGLE_BEARER_TOKEN_DO_NOT_LEAK' -const { getAllOAuthServicesMock, decodeJwtMock } = vi.hoisted(() => ({ +const { + getAllOAuthServicesMock, + decodeJwtMock, + isOAuthServiceDeploymentAvailableMock, + createIntegrationCredentialVisibilityMock, + getUserPermissionConfigMock, + getAccessibleOAuthCredentialsMock, + checkWorkspaceAccessMock, +} = vi.hoisted(() => ({ getAllOAuthServicesMock: vi.fn(), decodeJwtMock: vi.fn(), + isOAuthServiceDeploymentAvailableMock: vi.fn(() => true), + createIntegrationCredentialVisibilityMock: vi.fn(), + getUserPermissionConfigMock: vi.fn(), + getAccessibleOAuthCredentialsMock: vi.fn(), + checkWorkspaceAccessMock: vi.fn(), })) const getPersonalAndWorkspaceEnvMock = environmentUtilsMockFns.mockGetPersonalAndWorkspaceEnv @@ -30,6 +43,30 @@ vi.mock('@/lib/oauth', () => ({ getAllOAuthServices: getAllOAuthServicesMock, })) +vi.mock('@/lib/integrations/availability.server', () => ({ + isOAuthServiceDeploymentAvailable: isOAuthServiceDeploymentAvailableMock, +})) + +vi.mock('@/lib/integrations/credential-visibility.server', () => ({ + createIntegrationCredentialVisibility: createIntegrationCredentialVisibilityMock, +})) + +vi.mock('@/lib/core/config/env-flags', () => ({ + getAllowedIntegrationsFromEnv: vi.fn(() => null), +})) + +vi.mock('@/ee/access-control/utils/permission-check', () => ({ + getUserPermissionConfig: getUserPermissionConfigMock, +})) + +vi.mock('@/lib/credentials/environment', () => ({ + getAccessibleOAuthCredentials: getAccessibleOAuthCredentialsMock, +})) + +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + checkWorkspaceAccess: checkWorkspaceAccessMock, +})) + vi.mock('jose', () => ({ decodeJwt: decodeJwtMock, })) @@ -72,16 +109,38 @@ describe('getCredentialsServerTool', () => { getAllOAuthServicesMock.mockReturnValue([ { + serviceId: 'gmail', providerId: 'google-default', name: 'Google', description: 'Google account', baseProvider: 'google', + authType: 'oauth', }, { + serviceId: 'slack', providerId: 'slack', + serviceAccountProviderId: 'slack-custom-bot', name: 'Slack', description: 'Slack workspace', baseProvider: 'slack', + authType: 'oauth', + }, + { + serviceId: 'notion', + providerId: 'notion', + serviceAccountProviderId: 'notion-service-account', + name: 'Notion', + description: 'Notion workspace', + baseProvider: 'notion', + authType: 'oauth', + }, + { + serviceId: 'claude-platform', + providerId: 'claude-platform', + name: 'Claude Platform', + description: 'Claude managed agents', + baseProvider: 'claude-platform', + authType: 'service_account', }, ]) @@ -92,6 +151,30 @@ describe('getCredentialsServerTool', () => { }) decodeJwtMock.mockReturnValue({ email: 'brent@cellular.so' }) + isOAuthServiceDeploymentAvailableMock.mockReturnValue(true) + getUserPermissionConfigMock.mockResolvedValue(null) + getAccessibleOAuthCredentialsMock.mockResolvedValue([]) + checkWorkspaceAccessMock.mockResolvedValue({ canAdmin: false }) + createIntegrationCredentialVisibilityMock.mockImplementation( + ({ allowedIntegrationTypes, oauthServices }) => { + const isAllowed = (service: { serviceId: string }) => + allowedIntegrationTypes === null || allowedIntegrationTypes.has(service.serviceId) + const isOAuthServiceVisible = (service: { serviceId: string; providerId: string }) => + isAllowed(service) && isOAuthServiceDeploymentAvailableMock(service.providerId) + return { + isOAuthServiceVisible, + isCredentialVisible: ({ providerId, type }: { providerId: string; type?: string }) => { + const service = oauthServices.find( + (candidate: { providerId: string; serviceAccountProviderId?: string }) => + candidate.providerId === providerId || + candidate.serviceAccountProviderId === providerId + ) + if (!service || !isAllowed(service)) return !service + return type === 'service_account' || isOAuthServiceVisible(service) + }, + } + } + ) }) it('never returns access tokens for connected OAuth credentials', async () => { @@ -127,6 +210,67 @@ describe('getCredentialsServerTool', () => { expect(JSON.stringify(result)).not.toContain('refresh-secret') }) + it('does not advertise OAuth providers unavailable in this deployment', async () => { + isOAuthServiceDeploymentAvailableMock.mockImplementation( + (providerId: string) => providerId !== 'slack' + ) + + const result = await getCredentialsServerTool.execute({}, { userId: 'user-1' }) + + expect( + result.oauth.notConnected.services.map( + (service: { providerId: string }) => service.providerId + ) + ).not.toContain('slack') + }) + + it('uses context.workspaceId and hides integrations disallowed for the viewer', async () => { + getUserPermissionConfigMock.mockResolvedValue({ allowedIntegrations: ['slack'] }) + + const result = await getCredentialsServerTool.execute( + {}, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(getUserPermissionConfigMock).toHaveBeenCalledWith('user-1', 'workspace-1') + expect(result.oauth.connected.credentials).toEqual([]) + expect( + result.oauth.notConnected.services.map( + (service: { providerId: string }) => service.providerId + ) + ).toEqual(['slack']) + }) + + it('does not advertise service-account-only entries as OAuth connections', async () => { + const result = await getCredentialsServerTool.execute({}, { userId: 'user-1' }) + + expect( + result.oauth.notConnected.services.map( + (service: { providerId: string }) => service.providerId + ) + ).not.toContain('claude-platform') + }) + + it('hides shared service-account credentials disallowed for the viewer', async () => { + getUserPermissionConfigMock.mockResolvedValue({ allowedIntegrations: ['slack'] }) + getAccessibleOAuthCredentialsMock.mockResolvedValue([ + { + id: 'notion-service-account-1', + providerId: 'notion-service-account', + type: 'service_account', + displayName: 'Notion token', + updatedAt: new Date('2026-04-17T02:26:05.546Z'), + }, + ]) + + const result = await getCredentialsServerTool.execute( + {}, + { userId: 'user-1', workspaceId: 'workspace-1' } + ) + + expect(result.oauth.connected.credentials).toEqual([]) + }) + it('rejects unauthenticated callers without touching the database', async () => { await expect(getCredentialsServerTool.execute({}, undefined)).rejects.toThrow( 'Authentication required' diff --git a/apps/sim/lib/copilot/tools/server/user/get-credentials.ts b/apps/sim/lib/copilot/tools/server/user/get-credentials.ts index b0d9221c9f2..9e6e1076e79 100644 --- a/apps/sim/lib/copilot/tools/server/user/get-credentials.ts +++ b/apps/sim/lib/copilot/tools/server/user/get-credentials.ts @@ -6,10 +6,15 @@ import { eq } from 'drizzle-orm' import { decodeJwt } from 'jose' import { createPermissionError, verifyWorkflowAccess } from '@/lib/copilot/auth/permissions' import type { BaseServerTool } from '@/lib/copilot/tools/server/base-tool' +import { getAllowedIntegrationsFromEnv } from '@/lib/core/config/env-flags' import { getAccessibleOAuthCredentials } from '@/lib/credentials/environment' import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' +import { createIntegrationCredentialVisibility } from '@/lib/integrations/credential-visibility.server' import { getAllOAuthServices } from '@/lib/oauth' +import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' import { checkWorkspaceAccess, type WorkspaceAccess } from '@/lib/workspaces/permissions/utils' +import { overlayVisibility } from '@/blocks/visibility/context' +import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check' interface GetCredentialsParams { workflowId?: string @@ -17,7 +22,7 @@ interface GetCredentialsParams { export const getCredentialsServerTool: BaseServerTool = { name: 'get_credentials', - async execute(params: GetCredentialsParams, context?: { userId: string }): Promise { + async execute(params, context): Promise { const logger = createLogger('GetCredentialsServerTool') if (!context?.userId) { @@ -27,7 +32,7 @@ export const getCredentialsServerTool: BaseServerTool const authenticatedUserId = context.userId - let workspaceId: string | undefined + let workspaceId = context.workspaceId if (params?.workflowId) { const { hasAccess, workspaceId: wId } = await verifyWorkflowAccess( @@ -69,8 +74,23 @@ export const getCredentialsServerTool: BaseServerTool .limit(1) const userEmail = userRecord.length > 0 ? userRecord[0]?.email : null - // Get all available OAuth services - const allOAuthServices = getAllOAuthServices() + const permissionConfig = workspaceId ? await getUserPermissionConfig(userId, workspaceId) : null + const configuredAllowedIntegrations = intersectIntegrationAllowlists( + permissionConfig?.allowedIntegrations ?? null, + getAllowedIntegrationsFromEnv() + ) + const allowedIntegrationTypes = configuredAllowedIntegrations + ? new Set(configuredAllowedIntegrations.map((type) => type.toLowerCase())) + : null + + const serviceMetadata = getAllOAuthServices() + const credentialVisibility = createIntegrationCredentialVisibility({ + allowedIntegrationTypes, + blockVisibility: overlayVisibility(), + oauthServices: serviceMetadata, + }) + const allOAuthServices = serviceMetadata.filter((service) => service.authType === 'oauth') + const visibleOAuthServices = allOAuthServices.filter(credentialVisibility.isOAuthServiceVisible) // Track connected provider IDs const connectedProviderIds = new Set() @@ -86,6 +106,8 @@ export const getCredentialsServerTool: BaseServerTool for (const acc of accounts) { const providerId = acc.providerId + const service = allOAuthServices.find((candidate) => candidate.providerId === providerId) + if (!credentialVisibility.isCredentialVisible({ providerId, type: 'oauth' })) continue connectedProviderIds.add(providerId) const [baseProvider, featureType = 'default'] = providerId.split('-') @@ -105,7 +127,6 @@ export const getCredentialsServerTool: BaseServerTool if (!displayName) displayName = `${acc.accountId} (${baseProvider})` // Find the service name for this provider ID - const service = allOAuthServices.find((s) => s.providerId === providerId) const serviceName = service?.name ?? providerId connectedCredentials.push({ @@ -129,14 +150,26 @@ export const getCredentialsServerTool: BaseServerTool const seenCredentialIds = new Set(connectedCredentials.map((c) => c.id)) for (const cred of sharedCredentials) { if (seenCredentialIds.has(cred.id)) continue + if ( + !credentialVisibility.isCredentialVisible({ + providerId: cred.providerId, + type: cred.type, + }) + ) { + continue + } + const service = allOAuthServices.find( + (candidate) => + candidate.providerId === cred.providerId || + candidate.serviceAccountProviderId === cred.providerId + ) connectedProviderIds.add(cred.providerId) const [, featureType = 'default'] = cred.providerId.split('-') connectedCredentials.push({ id: cred.id, name: cred.displayName, provider: cred.providerId, - serviceName: - allOAuthServices.find((s) => s.providerId === cred.providerId)?.name ?? cred.providerId, + serviceName: service?.name ?? cred.providerId, lastUsed: cred.updatedAt.toISOString(), isDefault: featureType === 'default', }) @@ -144,7 +177,7 @@ export const getCredentialsServerTool: BaseServerTool } // Build list of not connected services - const notConnectedServices = allOAuthServices + const notConnectedServices = visibleOAuthServices .filter((service) => !connectedProviderIds.has(service.providerId)) .map((service) => ({ providerId: service.providerId, diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/builders.test.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/builders.test.ts index e49d0279423..baf77b90ea3 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/builders.test.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/builders.test.ts @@ -5,9 +5,18 @@ import { describe, expect, it, vi } from 'vitest' import { applyTriggerConfigToBlockSubblocks, createBlockFromParams, + filterDisallowedTools, normalizeSubblockValue, } from '@/lib/copilot/tools/server/workflow/edit-workflow/builders' +const { mockIsIntegrationDeploymentAvailable } = vi.hoisted(() => ({ + mockIsIntegrationDeploymentAvailable: vi.fn(() => true), +})) + +vi.mock('@/lib/integrations/availability.server', () => ({ + isIntegrationDeploymentAvailableForVisibility: mockIsIntegrationDeploymentAvailable, +})) + const agentBlockConfig = { type: 'agent', name: 'Agent', @@ -133,6 +142,23 @@ describe('createBlockFromParams', () => { }) }) +describe('filterDisallowedTools', () => { + it('removes unavailable integration tools even without a permission group', () => { + mockIsIntegrationDeploymentAvailable.mockImplementation((type: string) => type !== 'slack') + const skippedItems: Parameters[3] = [] + + const tools = filterDisallowedTools( + [{ type: 'slack' }, { type: 'custom-tool', customToolId: 'custom-1' }], + null, + 'agent-1', + skippedItems + ) + + expect(tools).toEqual([{ type: 'custom-tool', customToolId: 'custom-1' }]) + expect(skippedItems[0]?.reason).toContain('unavailable in this deployment') + }) +}) + describe('normalizeSubblockValue', () => { it.each(['tagFilters', 'documentTags', 'conditions', 'routes'])( 'serializes %s to a JSON string the subblock component can parse', diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/builders.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/builders.ts index f669a7719d5..524567f7f96 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/builders.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/builders.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { generateId, isValidUuid } from '@sim/utils/id' import { sortObjectKeysDeep } from '@sim/utils/object' +import { isIntegrationDeploymentAvailableForVisibility } from '@/lib/integrations/availability.server' import type { PermissionGroupConfig } from '@/lib/permission-groups/types' import { getEffectiveBlockOutputs } from '@/lib/workflows/blocks/block-outputs' import { @@ -9,8 +10,9 @@ import { isCanonicalPair, } from '@/lib/workflows/subblocks/visibility' import { hasTriggerCapability } from '@/lib/workflows/triggers/trigger-utils' -import { getAllBlocks, getBlock } from '@/blocks/registry' +import { getBlock } from '@/blocks/registry' import type { BlockConfig } from '@/blocks/types' +import { overlayVisibility } from '@/blocks/visibility/context' import { TRIGGER_RUNTIME_SUBBLOCK_IDS } from '@/triggers/constants' import type { EditWorkflowOperation, SkippedItem, ValidationError } from './types' import { logSkippedItem } from './types' @@ -31,7 +33,7 @@ export function createBlockFromParams( permissionConfig?: PermissionGroupConfig | null, skippedItems?: SkippedItem[] ): any { - const blockConfig = getAllBlocks().find((b) => b.type === params.type) + const blockConfig = getBlock(params.type) // Validate inputs against block configuration let validatedInputs: Record | undefined @@ -648,13 +650,30 @@ export function filterDisallowedTools( blockId: string, skippedItems: SkippedItem[] ): any[] { - if (!permissionConfig) { - return tools + const deploymentAvailableTools: any[] = [] + + for (const tool of tools) { + if ( + typeof tool?.type === 'string' && + getBlock(tool.type) && + !isIntegrationDeploymentAvailableForVisibility(tool.type, overlayVisibility()) + ) { + logSkippedItem(skippedItems, { + type: 'tool_not_allowed', + operationType: 'add', + blockId, + reason: `Tool block type "${tool.type}" is unavailable in this deployment - tool not added`, + details: { toolType: tool.type }, + }) + continue + } + deploymentAvailableTools.push(tool) } - const allowedTools: any[] = [] + if (!permissionConfig) return deploymentAvailableTools - for (const tool of tools) { + const allowedTools: any[] = [] + for (const tool of deploymentAvailableTools) { if (tool.type === 'custom-tool' && permissionConfig.disableCustomTools) { logSkippedItem(skippedItems, { type: 'tool_not_allowed', diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/index.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/index.ts index 23aa7e4fe41..a869462e928 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/index.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/index.ts @@ -7,6 +7,7 @@ import { } from '@sim/platform-authz/workflow' import { toError } from '@sim/utils/errors' import { eq } from 'drizzle-orm' +import { getBlockVisibilityForCopilot } from '@/lib/copilot/block-visibility' import { EditWorkflow } from '@/lib/copilot/generated/tool-catalog-v1' import { assertServerToolNotAborted, @@ -30,6 +31,7 @@ import { saveWorkflowToNormalizedTables, } from '@/lib/workflows/persistence/utils' import { validateWorkflowState } from '@/lib/workflows/sanitization/validation' +import { withBlockVisibility } from '@/blocks/visibility/server-context' import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check' import { generateLoopBlocks, generateParallelBlocks } from '@/stores/workflows/workflow/utils' import { normalizeWorkflowState } from '@/stores/workflows/workflow/validation' @@ -137,10 +139,10 @@ export const editWorkflowServerTool: BaseServerTool workflowState = fromDb.workflowState } - const permissionConfig = - context?.userId && workspaceId - ? await getUserPermissionConfig(context.userId, workspaceId) - : null + const [permissionConfig, blockVisibility] = await Promise.all([ + workspaceId ? getUserPermissionConfig(context.userId, workspaceId) : null, + getBlockVisibilityForCopilot(context.userId, workspaceId), + ]) // Pre-validate credential and apiKey inputs before applying operations // This filters out invalid credentials and apiKeys for hosted models @@ -161,7 +163,9 @@ export const editWorkflowServerTool: BaseServerTool state: modifiedWorkflowState, validationErrors, skippedItems, - } = applyOperationsToWorkflowState(workflowState, operationsToApply, permissionConfig) + } = await withBlockVisibility(blockVisibility, async () => + applyOperationsToWorkflowState(workflowState, operationsToApply, permissionConfig) + ) // Add credential validation errors validationErrors.push(...credentialErrors) diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/operations.test.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/operations.test.ts index 7914f4eaf59..666d5c4e973 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/operations.test.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/operations.test.ts @@ -56,6 +56,10 @@ vi.mock('@/blocks/registry', () => ({ }, })) +vi.mock('@/lib/integrations/availability.server', () => ({ + isIntegrationDeploymentAvailableForVisibility: () => true, +})) + function makeLoopWorkflow() { return { blocks: { diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/operations.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/operations.ts index 015791daad1..eda45aa2e6c 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/operations.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/operations.ts @@ -566,7 +566,7 @@ export function handleEditOperation(op: EditWorkflowOperation, ctx: OperationCon type: 'block_not_allowed', operationType: 'edit', blockId: block_id, - reason: `Block type "${params.type}" is not allowed by permission group - type change skipped`, + reason: `Block type "${params.type}" is unavailable in this deployment or blocked by access control - type change skipped`, details: { requestedType: params.type }, }) } else { @@ -744,7 +744,7 @@ export function handleAddOperation(op: EditWorkflowOperation, ctx: OperationCont type: 'block_not_allowed', operationType: 'add', blockId: block_id, - reason: `Block type "${params.type}" is not allowed by permission group - block not added`, + reason: `Block type "${params.type}" is unavailable in this deployment or blocked by access control - block not added`, details: { requestedType: params.type }, }) return @@ -970,7 +970,7 @@ export function handleInsertIntoSubflowOperation( type: 'block_not_allowed', operationType: 'insert_into_subflow', blockId: block_id, - reason: `Block type "${params.type}" is not allowed by permission group - block not inserted`, + reason: `Block type "${params.type}" is unavailable in this deployment or blocked by access control - block not inserted`, details: { requestedType: params.type, subflowId }, }) return diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts index 651c816f811..e523a0ffbc1 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts @@ -12,6 +12,7 @@ const { mockGetCustomToolById, mockGetSkillById, mockGetHostedModels, + mockIsIntegrationDeploymentAvailable, } = vi.hoisted(() => ({ mockValidateSelectorIds: vi.fn(), mockGetModelOptions: vi.fn(() => []), @@ -19,6 +20,7 @@ const { mockGetCustomToolById: vi.fn(), mockGetSkillById: vi.fn(), mockGetHostedModels: vi.fn(() => [] as string[]), + mockIsIntegrationDeploymentAvailable: vi.fn(() => true), })) const conditionBlockConfig = { @@ -251,6 +253,10 @@ vi.mock('@/providers/utils', () => ({ getHostedModels: mockGetHostedModels, })) +vi.mock('@/lib/integrations/availability.server', () => ({ + isIntegrationDeploymentAvailableForVisibility: mockIsIntegrationDeploymentAvailable, +})) + import { collectUnresolvedAgentToolReferences, collectUnresolvedReferences, @@ -263,6 +269,10 @@ const CTX = { userId: 'user-1', workspaceId: 'workspace-1' } afterAll(resetEnvFlagsMock) +beforeEach(() => { + mockIsIntegrationDeploymentAvailable.mockReturnValue(true) +}) + describe('validateInputsForBlock', () => { beforeEach(() => { vi.clearAllMocks() @@ -1229,6 +1239,19 @@ describe('validateInputsForBlock - agent tools (tool-input)', () => { expect(result.validInputs.tools).toBeDefined() }) + it('rejects an integration tool unavailable in this deployment', () => { + mockIsIntegrationDeploymentAvailable.mockReturnValue(false) + + const result = validateInputsForBlock( + 'agent', + { tools: [{ type: 'slack', operation: 'send', usageControl: 'auto' }] }, + 'agent-1' + ) + + expect(result.validInputs.tools).toBeUndefined() + expect(result.errors[0]?.error).toContain('unavailable in this deployment') + }) + it('rejects an unrecognized tool type', () => { const result = validateInputsForBlock( 'agent', diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts index 48d44f21dbc..bbada28edbb 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts @@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors' import { omit } from '@sim/utils/object' import { validateSelectorIds } from '@/lib/copilot/validation/selector-validator' import { isHosted as isHostedDeployment } from '@/lib/core/config/env-flags' +import { isIntegrationDeploymentAvailableForVisibility } from '@/lib/integrations/availability.server' import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' import type { PermissionGroupConfig } from '@/lib/permission-groups/types' import { getCustomToolById } from '@/lib/workflows/custom-tools/operations' @@ -16,6 +17,7 @@ import { import { getBlock } from '@/blocks/registry' import type { SubBlockConfig } from '@/blocks/types' import { getModelOptions } from '@/blocks/utils' +import { overlayVisibility } from '@/blocks/visibility/context' import { BlockType, EDGE, normalizeName } from '@/executor/constants' import { isAutoModel, isKnownModelId, suggestModelIdsForUnknownModel } from '@/providers/models' import { isPiByokOnlyMode } from '@/providers/pi-providers' @@ -243,6 +245,9 @@ function validateAgentToolEntry(item: any, index: number): string | null { if (!Array.isArray(block.tools?.access) || block.tools.access.length === 0) { return `${where} block type "${type}" cannot be attached as an agent tool (it exposes no callable tools)` } + if (!isIntegrationDeploymentAvailableForVisibility(type, overlayVisibility())) { + return `${where} block type "${type}" is unavailable in this deployment` + } } return null @@ -924,6 +929,7 @@ export function isBlockTypeAllowed( blockType: string, permissionConfig: PermissionGroupConfig | null ): boolean { + if (!isIntegrationDeploymentAvailableForVisibility(blockType, overlayVisibility())) return false if (isBlockTypeAccessControlExempt(blockType)) { return true } diff --git a/apps/sim/lib/copilot/vfs/serializers.test.ts b/apps/sim/lib/copilot/vfs/serializers.test.ts index 4394b746e0b..4cb647b5c3f 100644 --- a/apps/sim/lib/copilot/vfs/serializers.test.ts +++ b/apps/sim/lib/copilot/vfs/serializers.test.ts @@ -354,6 +354,17 @@ describe('serializeIntegrationSchema — service-account auth', () => { expect(schema.auth.serviceAccount).toBeUndefined() }) + it('keeps service-account auth while suppressing an unavailable OAuth connection', () => { + const schema = JSON.parse( + serializeIntegrationSchema(oauthTool('notion_read', 'notion'), { + oauthAvailable: false, + }) + ) + + expect(schema.auth.serviceAccount).toEqual({ connectNoun: 'integration secret' }) + expect(schema.oauth).toBeUndefined() + }) + // The preview-gate behavior (slack custom bot ↔ slack_v2) is covered in // service-account-gate.test.ts, which mocks getBlock — the block registry is // globally stubbed here, so slack_v2's real `preview: true` isn't observable diff --git a/apps/sim/lib/copilot/vfs/serializers.ts b/apps/sim/lib/copilot/vfs/serializers.ts index 53109f4062b..abbbded736c 100644 --- a/apps/sim/lib/copilot/vfs/serializers.ts +++ b/apps/sim/lib/copilot/vfs/serializers.ts @@ -53,11 +53,13 @@ export type VfsToolAuth = * per-tool `auth.serviceAccount` field and the `oauth-integrations.json` * roll-up, so the two never disagree. Returns `undefined` when the service has * no service-account flow, or its flow is gated by a preview block (a custom - * Slack bot needs slack_v2) — GA-only discovery, so the agent never proactively - * offers a preview flow, matching the per-viewer gate the renderer applies. + * Slack bot needs slack_v2) that is not the visible owner being serialized. + * This keeps the default projection GA-only while allowing a revealed preview + * block's own schema to describe the credential flow it enables. */ export function describeServiceAccountForOAuthProvider( - oauthProvider: string + oauthProvider: string, + ownerBlockType?: string ): VfsServiceAccountAuth | undefined { const serviceAccountProviderId = getServiceAccountProviderForProviderId(oauthProvider) if (!serviceAccountProviderId) return undefined @@ -69,7 +71,9 @@ export function describeServiceAccountForOAuthProvider( // static preview check — so once the block GAs and drops `preview`, it is // no longer hidden and discovery includes it again, matching the renderer. // Hand-rolling `?.preview ?? true` would keep it omitted forever after GA. - if (!gatingBlock || isHiddenUnder(null, gatingBlock)) return undefined + if (!gatingBlock || (ownerBlockType !== gatingBlockType && isHiddenUnder(null, gatingBlock))) { + return undefined + } } return { connectNoun: getServiceAccountConnectNoun(serviceAccountProviderId) } } @@ -77,15 +81,23 @@ export function describeServiceAccountForOAuthProvider( export interface ComponentSerializationOptions { hosted?: boolean toolConfigs?: ReadonlyMap + ownerBlockType?: string } /** * Project runtime tool authentication into a stable, machine-readable VFS contract. * ToolConfig.hosting remains the source of truth for every hosted-key integration. */ -export function serializeToolAuth(tool: ToolConfig, hosted = isHosted): VfsToolAuth | undefined { +export function serializeToolAuth( + tool: ToolConfig, + hosted = isHosted, + ownerBlockType?: string +): VfsToolAuth | undefined { if (tool.oauth) { - const serviceAccount = describeServiceAccountForOAuthProvider(tool.oauth.provider) + const serviceAccount = describeServiceAccountForOAuthProvider( + tool.oauth.provider, + ownerBlockType + ) return { type: 'oauth', required: tool.oauth.required, @@ -600,7 +612,7 @@ export function serializeBlockSchema( for (const toolId of block.tools.access) { const tool = options?.toolConfigs?.get(toolId) if (!tool) continue - const auth = serializeToolAuth(tool, hosted) + const auth = serializeToolAuth(tool, hosted, block.type) if (auth) toolAuth[toolId] = auth } @@ -711,7 +723,6 @@ interface ApiKeyIntegrationTool { config: ToolConfig service: string operation: string - preview?: boolean } /** @@ -719,7 +730,7 @@ interface ApiKeyIntegrationTool { * ToolConfig.hosting is the only provider registry used to build this index. */ export function serializeApiKeyIntegrations( - tools: ApiKeyIntegrationTool[], + tools: readonly ApiKeyIntegrationTool[], hosted = isHosted ): string { const services = new Map< @@ -732,8 +743,8 @@ export function serializeApiKeyIntegrations( } >() - for (const { config: tool, service, operation, preview } of tools) { - if (preview || !tool.hosting?.apiKeyParam) continue + for (const { config: tool, service, operation } of tools) { + if (!tool.hosting?.apiKeyParam) continue const metadata = services.get(service) ?? { params: [], @@ -959,10 +970,12 @@ export function serializeSkill(s: { */ export function serializeIntegrationSchema( tool: ToolConfig, - options?: Pick + options?: Pick & { + oauthAvailable?: boolean + } ): string { const hosted = options?.hosted ?? isHosted - const auth = serializeToolAuth(tool, hosted) + const auth = serializeToolAuth(tool, hosted, options?.ownerBlockType) const hostedApiKeyParam = auth?.type === 'api_key' && auth.mode === 'hosted_or_byok' ? auth.param : null @@ -976,9 +989,10 @@ export function serializeIntegrationSchema( description: getCopilotToolDescription(tool, { isHosted: hosted }), version: tool.version, auth, - oauth: tool.oauth - ? { required: tool.oauth.required, provider: tool.oauth.provider } - : undefined, + oauth: + tool.oauth && options?.oauthAvailable !== false + ? { required: tool.oauth.required, provider: tool.oauth.provider } + : undefined, params: tool.params ? { ...Object.fromEntries( diff --git a/apps/sim/lib/copilot/vfs/service-account-gate.test.ts b/apps/sim/lib/copilot/vfs/service-account-gate.test.ts index e525dae124b..958ba995d77 100644 --- a/apps/sim/lib/copilot/vfs/service-account-gate.test.ts +++ b/apps/sim/lib/copilot/vfs/service-account-gate.test.ts @@ -27,6 +27,14 @@ describe('describeServiceAccountForOAuthProvider — preview gate', () => { expect(describeServiceAccountForOAuthProvider('slack')).toEqual({ connectNoun: 'custom bot' }) }) + it('includes it for the revealed preview block that owns the serialized tool', () => { + mockGetBlock.mockReturnValue({ type: 'slack_v2', preview: true }) + + expect(describeServiceAccountForOAuthProvider('slack', 'slack_v2')).toEqual({ + connectNoun: 'custom bot', + }) + }) + it('fail-closes (omits) when the gating block is missing entirely', () => { mockGetBlock.mockReturnValue(undefined) expect(describeServiceAccountForOAuthProvider('slack')).toBeUndefined() diff --git a/apps/sim/lib/copilot/vfs/workspace-vfs.ts b/apps/sim/lib/copilot/vfs/workspace-vfs.ts index 8e112a553ca..bdc264e4eaa 100644 --- a/apps/sim/lib/copilot/vfs/workspace-vfs.ts +++ b/apps/sim/lib/copilot/vfs/workspace-vfs.ts @@ -26,7 +26,11 @@ import { } from '@/lib/copilot/chat/workspace-context' import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1' import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1' -import { getExposedIntegrationTools } from '@/lib/copilot/integration-tools' +import { + type ExposedIntegrationTool, + filterExposedIntegrationTools, + getExposedIntegrationTools, +} from '@/lib/copilot/integration-tools' import { recordVfsMaterialize } from '@/lib/copilot/request/metrics' import { markSpanForError } from '@/lib/copilot/request/otel' import { compileDoc, getE2BDocFormat } from '@/lib/copilot/tools/server/files/doc-compile' @@ -83,7 +87,11 @@ import { serializeWorkflowMeta, } from '@/lib/copilot/vfs/serializers' import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' -import { isDocSandboxEnabled, isHosted } from '@/lib/core/config/env-flags' +import { + getAllowedIntegrationsFromEnv, + isDocSandboxEnabled, + isHosted, +} from '@/lib/core/config/env-flags' import { getAccessibleEnvCredentials, getAccessibleOAuthCredentials, @@ -91,8 +99,15 @@ import { import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' import { BINARY_DOC_TASKS, MAX_DOCUMENT_PREVIEW_CODE_BYTES } from '@/lib/execution/constants' import { runSandboxTask, SandboxUserCodeError } from '@/lib/execution/sandbox/run-task' +import { + isIntegrationDeploymentAvailableForVisibility, + isOAuthServiceDeploymentAvailable, +} from '@/lib/integrations/availability.server' +import { createIntegrationCredentialVisibility } from '@/lib/integrations/credential-visibility.server' import { getKnowledgeBases } from '@/lib/knowledge/service' import { validateMermaidSource } from '@/lib/mermaid/validate' +import { isBlockTypeAccessControlExempt } from '@/lib/permission-groups/block-access' +import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' import { getWorkspaceShares } from '@/lib/public-shares/share-manager' import { listTables } from '@/lib/table/service' import { listWorkspaceFileFolders } from '@/lib/uploads/contexts/workspace/workspace-file-folder-manager' @@ -123,6 +138,7 @@ import { BLOCK_REGISTRY } from '@/blocks/registry-maps' import type { BlockConfig, BlockIcon } from '@/blocks/types' import { isHiddenUnder, overlayVisibility } from '@/blocks/visibility/context' import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' +import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check' import type { WorkflowState } from '@/stores/workflows/workflow/types' import type { ToolConfig } from '@/tools/types' import { TRIGGER_REGISTRY } from '@/triggers/registry' @@ -149,7 +165,7 @@ let staticComponentFiles: Map | null = null * basename, but integration paths use the version-stripped service name — so * their owners need this lookup for the stamp-time visibility filter. */ -const integrationPathOwners = new Map>() +const integrationPathOwners = new Map>>() /** * Owning block(s) for each `components/triggers/{provider}/{id}.json` file, @@ -168,18 +184,126 @@ const triggerPathOwners = new Map, + vis: BlockVisibilityState | null, + allowedIntegrationTypes: ReadonlySet | null +): boolean { + const config = BLOCK_REGISTRY[owner.type] + if (config?.hideFromToolbar) return true + if (!isIntegrationDeploymentAvailableForVisibility(owner.type, vis)) return true + if ( + allowedIntegrationTypes !== null && + !isBlockTypeAccessControlExempt(owner.type) && + !allowedIntegrationTypes.has(owner.type.toLowerCase()) + ) { + return true + } + return isHiddenUnder(vis, owner) +} + +function isStaticFileHidden( + path: string, + vis: BlockVisibilityState | null, + allowedIntegrationTypes: ReadonlySet | null = null +): boolean { const blockMatch = path.match(/^components\/(?:blocks|triggers\/sim)\/([^/]+)\.json$/) if (blockMatch) { const config = BLOCK_REGISTRY[blockMatch[1]!] - return config ? isHiddenUnder(vis, config) : false + return config ? isBlockOwnerHidden(config, vis, allowedIntegrationTypes) : false } const triggerOwners = triggerPathOwners.get(path) if (triggerOwners) { - return triggerOwners.length > 0 && triggerOwners.every((owner) => isHiddenUnder(vis, owner)) + return ( + triggerOwners.length > 0 && + triggerOwners.every((owner) => isBlockOwnerHidden(owner, vis, allowedIntegrationTypes)) + ) + } + const owners = integrationPathOwners.get(path) + return owners + ? owners.length > 0 && + owners.every((owner) => isBlockOwnerHidden(owner, vis, allowedIntegrationTypes)) + : false +} + +function buildIntegrationAggregateFiles( + exposedTools: readonly ExposedIntegrationTool[] +): Map { + const oauthServices = new Map< + string, + { + provider: string + operations: string[] + oauthAvailable: boolean + serviceAccount?: VfsServiceAccountAuth + } + >() + for (const { config: tool, service, operation, blockType } of exposedTools) { + if (!tool.oauth?.required) continue + const oauthAvailable = isOAuthServiceDeploymentAvailable(tool.oauth.provider) + const serviceAccount = describeServiceAccountForOAuthProvider(tool.oauth.provider, blockType) + if (!oauthAvailable && !serviceAccount) continue + const existing = oauthServices.get(service) + if (existing) { + existing.operations.push(operation) + existing.oauthAvailable ||= oauthAvailable + existing.serviceAccount ??= serviceAccount + } else { + oauthServices.set(service, { + provider: tool.oauth.provider, + operations: [operation], + oauthAvailable, + serviceAccount, + }) + } } - const owner = integrationPathOwners.get(path) - return owner ? isHiddenUnder(vis, owner) : false + + return new Map([ + [ + 'environment/oauth-integrations.json', + JSON.stringify(Object.fromEntries(oauthServices), null, 2), + ], + ['environment/api-key-integrations.json', serializeApiKeyIntegrations(exposedTools, isHosted)], + ]) +} + +function buildTriggerOverview( + vis: BlockVisibilityState | null, + allowedIntegrationTypes: ReadonlySet | null +): string { + const builtinTriggers = Object.values(BLOCK_REGISTRY) + .filter( + (block) => + block.category === 'triggers' && + !block.preview && + !isStaticFileHidden( + `components/triggers/sim/${block.type}.json`, + vis, + allowedIntegrationTypes + ) + ) + .map((block) => ({ + id: block.type, + name: block.name, + provider: 'sim', + description: block.description, + })) + const externalTriggers = Object.entries(TRIGGER_REGISTRY) + .filter( + ([id, trigger]) => + !isStaticFileHidden( + `components/triggers/${trigger.provider}/${id}.json`, + vis, + allowedIntegrationTypes + ) + ) + .map(([id, trigger]) => ({ + id, + name: trigger.name, + provider: trigger.provider, + description: trigger.description, + })) + return serializeTriggerOverview(builtinTriggers, externalTriggers) } // On-the-fly doc reads (render/extract) download the binary into the Sim process @@ -214,11 +338,10 @@ function getStaticComponentFiles(): Map { // Raw registry, never the visibility-projected getAllBlocks: this map is a // process-global shared cache, so it must hold the deterministic ungated - // universe. Preview blocks get schema files here (path-filterable at stamp - // time for revealed viewers) but are EXCLUDED from the shared aggregate - // files (overviews, oauth/api-key summaries) that all viewers receive. + // universe. Preview blocks get schema files here and are filtered per viewer + // at stamp time. Viewer-specific aggregate files are built during materialization. const allBlocks = Object.values(BLOCK_REGISTRY) - const visibleBlocks = allBlocks.filter((b) => !b.hideFromToolbar) + const visibleBlocks = allBlocks.filter((block) => !block.hideFromToolbar) const exposedTools = getExposedIntegrationTools() const toolConfigs = new Map() for (const { toolId, config } of exposedTools) { @@ -235,53 +358,28 @@ function getStaticComponentFiles(): Map { let integrationCount = 0 - // `serviceAccount` marks services that also accept a shared service-account - // credential (connect AS AN APPLICATION, not as the user) — the same - // `auth.serviceAccount` shape the per-operation schemas carry, so the agent - // discovers all three auth modes (oauth / api_key / service account) from one - // uniform field instead of a separate file. - const oauthServices = new Map< - string, - { provider: string; operations: string[]; serviceAccount?: VfsServiceAccountAuth } - >() - // Integration tools come from the shared exposed-tool set (latest version of // each operation owned by a visible block), the same set used to build the // deferred callable tools — so discovery and execution can never drift. for (const exposedTool of exposedTools) { - const { config: tool, service, operation, blockType, preview } = exposedTool + const { config: tool, service, operation } = exposedTool const path = `components/integrations/${service}/${operation}.json` - files.set(path, serializeIntegrationSchema(tool)) - integrationPathOwners.set(path, { type: blockType, preview }) - integrationCount++ - - // Preview-owned tools stay out of the shared oauth/api-key aggregates — - // those files are identical for every viewer. - if (preview) continue - - if (tool.oauth?.required) { - const existing = oauthServices.get(service) - if (existing) { - existing.operations.push(operation) - } else { - oauthServices.set(service, { - provider: tool.oauth.provider, - operations: [operation], - serviceAccount: describeServiceAccountForOAuthProvider(tool.oauth.provider), - }) + files.set( + path, + serializeIntegrationSchema(tool, { + oauthAvailable: !tool.oauth || isOAuthServiceDeploymentAvailable(tool.oauth.provider), + }) + ) + const owners = integrationPathOwners.get(path) ?? [] + for (const owner of exposedTool.owners) { + if (!owners.some((existing) => existing.type === owner.blockType)) { + owners.push({ type: owner.blockType, preview: owner.preview }) } } + integrationPathOwners.set(path, owners) + integrationCount++ } - files.set( - 'environment/oauth-integrations.json', - JSON.stringify(Object.fromEntries(oauthServices), null, 2) - ) - files.set( - 'environment/api-key-integrations.json', - serializeApiKeyIntegrations(exposedTools, isHosted) - ) - files.set( 'components/blocks/loop.json', JSON.stringify( @@ -389,33 +487,7 @@ function getStaticComponentFiles(): Map { externalTriggerCount++ } - files.set( - 'components/triggers/triggers.md', - serializeTriggerOverview( - // The overview is a shared file — preview trigger blocks stay out of it - // (their per-type schema file remains discoverable for revealed viewers). - builtinTriggerBlocks - .filter((b) => !b.preview) - .map((b) => ({ - id: b.type, - name: b.name, - provider: 'sim', - description: b.description, - })), - // Same for external triggers: a trigger owned solely by preview blocks is - // hidden under the null (no-viewer) state this shared file is built with. - Object.entries(TRIGGER_REGISTRY) - .filter( - ([id, t]) => !isStaticFileHidden(`components/triggers/${t.provider}/${id}.json`, null) - ) - .map(([id, t]) => ({ - id, - name: t.name, - provider: t.provider, - description: t.description, - })) - ) - ) + files.set('components/triggers/triggers.md', buildTriggerOverview(null, null)) logger.info('Static component files built', { blocks: visibleBlocks.length, @@ -659,7 +731,6 @@ export class WorkspaceVFS { phaseMs[phase] = Date.now() - t0 }) } - await trace .getTracer('sim-copilot-vfs', '1.0.0') .startActiveSpan( @@ -667,6 +738,11 @@ export class WorkspaceVFS { { attributes: { [TraceAttr.WorkspaceId]: workspaceId } }, async (span) => { try { + const blockVisibility = overlayVisibility() + const permissionConfigPromise = timed( + 'permissions', + getUserPermissionConfig(userId, workspaceId) + ) const [ wfSummary, kbSummary, @@ -679,18 +755,28 @@ export class WorkspaceVFS { skillsSummary, wsRow, members, + permissionConfig, ] = await Promise.all([ timed('workflows', this.materializeWorkflows(workspaceId)), timed('knowledge_bases', this.materializeKnowledgeBases(workspaceId, userId)), timed('tables', this.materializeTables(workspaceId)), timed('files', this.materializeFiles(workspaceId)), - timed('environment', this.materializeEnvironment(workspaceId, userId)), + timed( + 'environment', + this.materializeEnvironment( + workspaceId, + userId, + permissionConfigPromise, + blockVisibility + ) + ), timed('custom_tools', this.materializeCustomTools(workspaceId, userId)), timed('custom_blocks', this.materializeCustomBlocks(workspaceId)), timed('mcp_servers', this.materializeMcpServers(workspaceId)), timed('skills', this.materializeSkills(workspaceId)), timed('workspace_row', getWorkspaceWithOwner(workspaceId)), timed('members', getUsersWithPermissions(workspaceId)), + permissionConfigPromise, // Writes tasks/ files only — WORKSPACE.md has no Tasks section // (recent chats reorder every turn and would bust the cached // prompt prefix), so nothing is destructured from this one. @@ -719,11 +805,43 @@ export class WorkspaceVFS { // Per-viewer gating happens HERE, not in the shared builder: files // owned by blocks hidden for this viewer are skipped at stamp time. - const blockVisibility = overlayVisibility() + const configuredAllowedIntegrations = intersectIntegrationAllowlists( + permissionConfig?.allowedIntegrations ?? null, + getAllowedIntegrationsFromEnv() + ) + const allowedIntegrationTypes = configuredAllowedIntegrations + ? new Set(configuredAllowedIntegrations.map((type) => type.toLowerCase())) + : null for (const [path, content] of getStaticComponentFiles()) { - if (isStaticFileHidden(path, blockVisibility)) continue + if (isStaticFileHidden(path, blockVisibility, allowedIntegrationTypes)) continue + this.files.set(path, content) + } + const viewerIntegrationTools = filterExposedIntegrationTools( + getExposedIntegrationTools(), + blockVisibility, + (owner) => + isIntegrationDeploymentAvailableForVisibility(owner.blockType, blockVisibility) && + (allowedIntegrationTypes === null || + allowedIntegrationTypes.has(owner.blockType.toLowerCase())) + ) + for (const exposedTool of viewerIntegrationTools) { + const { config: tool, service, operation, blockType } = exposedTool + this.files.set( + `components/integrations/${service}/${operation}.json`, + serializeIntegrationSchema(tool, { + oauthAvailable: + !tool.oauth || isOAuthServiceDeploymentAvailable(tool.oauth.provider), + ownerBlockType: blockType, + }) + ) + } + for (const [path, content] of buildIntegrationAggregateFiles(viewerIntegrationTools)) { this.files.set(path, content) } + this.files.set( + 'components/triggers/triggers.md', + buildTriggerOverview(blockVisibility, allowedIntegrationTypes) + ) span.setAttributes({ [TraceAttr.CopilotVfsMaterializeFileCount]: this.files.size, @@ -2229,19 +2347,39 @@ export class WorkspaceVFS { */ private async materializeEnvironment( workspaceId: string, - userId: string + userId: string, + permissionConfigPromise: ReturnType, + blockVisibility: BlockVisibilityState | null ): Promise<{ oauthIntegrations: WorkspaceMdData['oauthIntegrations'] envVariables: WorkspaceMdData['envVariables'] }> { try { const isWorkspaceAdmin = await hasWorkspaceAdminAccess(userId, workspaceId) - const [envCredentials, oauthCredentials, apiKeyRows, envData] = await Promise.all([ - getAccessibleEnvCredentials(workspaceId, userId, { isWorkspaceAdmin }), - getAccessibleOAuthCredentials(workspaceId, userId, { isWorkspaceAdmin }), - listApiKeys(workspaceId), - getPersonalAndWorkspaceEnv(userId, workspaceId), - ]) + const [envCredentials, oauthCredentials, apiKeyRows, envData, permissionConfig] = + await Promise.all([ + getAccessibleEnvCredentials(workspaceId, userId, { isWorkspaceAdmin }), + getAccessibleOAuthCredentials(workspaceId, userId, { isWorkspaceAdmin }), + listApiKeys(workspaceId), + getPersonalAndWorkspaceEnv(userId, workspaceId), + permissionConfigPromise, + ]) + const configuredAllowedIntegrations = intersectIntegrationAllowlists( + permissionConfig?.allowedIntegrations ?? null, + getAllowedIntegrationsFromEnv() + ) + const credentialVisibility = createIntegrationCredentialVisibility({ + allowedIntegrationTypes: configuredAllowedIntegrations + ? new Set(configuredAllowedIntegrations.map((type) => type.toLowerCase())) + : null, + blockVisibility, + }) + const visibleOAuthCredentials = oauthCredentials.filter((credential) => + credentialVisibility.isCredentialVisible({ + providerId: credential.providerId, + type: credential.type, + }) + ) this.files.set( 'environment/credentials.json', @@ -2251,7 +2389,7 @@ export class WorkspaceVFS { scope: c.type === 'env_workspace' ? 'workspace' : 'personal', createdAt: c.updatedAt, })), - ...oauthCredentials.map((c) => ({ + ...visibleOAuthCredentials.map((c) => ({ id: c.id, providerId: c.providerId, displayName: c.displayName, @@ -2274,7 +2412,7 @@ export class WorkspaceVFS { const envKeys = [...new Set(envCredentials.map((c) => c.envKey))] return { - oauthIntegrations: oauthCredentials.map((c) => ({ + oauthIntegrations: visibleOAuthCredentials.map((c) => ({ id: c.id, providerId: c.providerId, displayName: c.displayName, diff --git a/apps/sim/lib/core/async-jobs/config.ts b/apps/sim/lib/core/async-jobs/config.ts index 7f1e3797b34..ef9df1ae3d0 100644 --- a/apps/sim/lib/core/async-jobs/config.ts +++ b/apps/sim/lib/core/async-jobs/config.ts @@ -1,7 +1,7 @@ import { createLogger } from '@sim/logger' import { taskContext } from '@trigger.dev/core/v3' import type { AsyncBackendType, JobQueueBackend } from '@/lib/core/async-jobs/types' -import { isTriggerDevEnabled } from '@/lib/core/config/env-flags' +import { getConfiguredAsyncJobsProvider } from '@/lib/core/config/env-capabilities.server' const logger = createLogger('AsyncJobsConfig') @@ -19,11 +19,10 @@ let cachedInlineBackend: JobQueueBackend | null = null * the database backend that nothing's draining. */ export function getAsyncBackendType(): AsyncBackendType { - if (isTriggerDevEnabled || taskContext.isInsideTask) { + if (taskContext.isInsideTask) { return 'trigger-dev' } - - return 'database' + return getConfiguredAsyncJobsProvider() } /** diff --git a/apps/sim/lib/core/config/api-keys.ts b/apps/sim/lib/core/config/api-keys.ts index 839df609a1a..3135374ae0a 100644 --- a/apps/sim/lib/core/config/api-keys.ts +++ b/apps/sim/lib/core/config/api-keys.ts @@ -1,4 +1,5 @@ import { env } from '@/lib/core/config/env' +import { LLM_KEY_POOLS } from '@/lib/core/config/env-capabilities' /** * Rotates through available API keys for a provider @@ -7,56 +8,15 @@ import { env } from '@/lib/core/config/env' * @throws Error if no API keys are configured for rotation */ export function getRotatingApiKey(provider: string): string { - if ( - provider !== 'openai' && - provider !== 'anthropic' && - provider !== 'gemini' && - provider !== 'cohere' && - provider !== 'zai' && - provider !== 'xai' && - provider !== 'kimi' && - provider !== 'fireworks' - ) { + if (!(provider in LLM_KEY_POOLS)) { throw new Error(`No rotation implemented for provider: ${provider}`) } - const keys = [] - - if (provider === 'openai') { - if (env.OPENAI_API_KEY_1) keys.push(env.OPENAI_API_KEY_1) - if (env.OPENAI_API_KEY_2) keys.push(env.OPENAI_API_KEY_2) - if (env.OPENAI_API_KEY_3) keys.push(env.OPENAI_API_KEY_3) - } else if (provider === 'anthropic') { - if (env.ANTHROPIC_API_KEY_1) keys.push(env.ANTHROPIC_API_KEY_1) - if (env.ANTHROPIC_API_KEY_2) keys.push(env.ANTHROPIC_API_KEY_2) - if (env.ANTHROPIC_API_KEY_3) keys.push(env.ANTHROPIC_API_KEY_3) - } else if (provider === 'gemini') { - if (env.GEMINI_API_KEY_1) keys.push(env.GEMINI_API_KEY_1) - if (env.GEMINI_API_KEY_2) keys.push(env.GEMINI_API_KEY_2) - if (env.GEMINI_API_KEY_3) keys.push(env.GEMINI_API_KEY_3) - } else if (provider === 'cohere') { - if (env.COHERE_API_KEY_1) keys.push(env.COHERE_API_KEY_1) - if (env.COHERE_API_KEY_2) keys.push(env.COHERE_API_KEY_2) - if (env.COHERE_API_KEY_3) keys.push(env.COHERE_API_KEY_3) - } else if (provider === 'zai') { - if (env.ZAI_API_KEY_1) keys.push(env.ZAI_API_KEY_1) - if (env.ZAI_API_KEY_2) keys.push(env.ZAI_API_KEY_2) - if (env.ZAI_API_KEY_3) keys.push(env.ZAI_API_KEY_3) - } else if (provider === 'xai') { - if (env.XAI_API_KEY_1) keys.push(env.XAI_API_KEY_1) - if (env.XAI_API_KEY_2) keys.push(env.XAI_API_KEY_2) - if (env.XAI_API_KEY_3) keys.push(env.XAI_API_KEY_3) - } else if (provider === 'kimi') { - if (env.KIMI_API_KEY_1) keys.push(env.KIMI_API_KEY_1) - if (env.KIMI_API_KEY_2) keys.push(env.KIMI_API_KEY_2) - if (env.KIMI_API_KEY_3) keys.push(env.KIMI_API_KEY_3) - } else if (provider === 'fireworks') { - if (env.FIREWORKS_API_KEY_1) keys.push(env.FIREWORKS_API_KEY_1) - if (env.FIREWORKS_API_KEY_2) keys.push(env.FIREWORKS_API_KEY_2) - if (env.FIREWORKS_API_KEY_3) keys.push(env.FIREWORKS_API_KEY_3) - // The platform Fireworks key predates the rotation slots and ships as a - // single secret; it stands in as a one-key pool until slots are populated. - if (keys.length === 0 && env.FIREWORKS_API_KEY) keys.push(env.FIREWORKS_API_KEY) + const definition = LLM_KEY_POOLS[provider as keyof typeof LLM_KEY_POOLS] + const keys = definition.keys.map((key) => env[key]).filter((key): key is string => Boolean(key)) + if (keys.length === 0 && 'fallbackKey' in definition) { + const fallback = env[definition.fallbackKey] + if (fallback) keys.push(fallback) } if (keys.length === 0) { diff --git a/apps/sim/lib/core/config/env-capabilities.server.test.ts b/apps/sim/lib/core/config/env-capabilities.server.test.ts new file mode 100644 index 00000000000..aa2156b15f7 --- /dev/null +++ b/apps/sim/lib/core/config/env-capabilities.server.test.ts @@ -0,0 +1,75 @@ +/** + * @vitest-environment node + */ +import { resetEnvMock, setEnv } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, expectTypeOf, it } from 'vitest' +import { + inspectConfiguredOAuthClient, + requireConfiguredOAuthClient, +} from '@/lib/core/config/env-capabilities.server' + +describe('server environment capabilities', () => { + beforeEach(() => { + setEnv({ + SHOPIFY_CLIENT_ID: undefined, + SHOPIFY_CLIENT_SECRET: undefined, + SLACK_CLIENT_ID: undefined, + SLACK_CLIENT_SECRET: undefined, + }) + }) + + afterAll(resetEnvMock) + + it('inspects partial OAuth configuration without throwing', () => { + setEnv({ SLACK_CLIENT_ID: 'slack-client' }) + + expect(inspectConfiguredOAuthClient('slack')).toEqual({ + state: 'partial', + missingFields: ['SLACK_CLIENT_SECRET'], + setupCommand: 'bun run setup integration slack', + }) + }) + + it('fails fast when an OAuth client is absent', () => { + expect(() => requireConfiguredOAuthClient('shopify')).toThrow( + 'OAuth client shopify is not configured. Run bun run setup integration shopify.' + ) + }) + + it('fails fast when an OAuth client is partially configured', () => { + setEnv({ SLACK_CLIENT_ID: 'slack-client' }) + + expect(() => requireConfiguredOAuthClient('slack')).toThrow( + 'OAuth client slack is partially configured — missing SLACK_CLIENT_SECRET. Run bun run setup integration slack.' + ) + }) + + it('does not expose non-string OAuth values as configured credentials', () => { + setEnv({ + SHOPIFY_CLIENT_ID: true, + SHOPIFY_CLIENT_SECRET: 'shopify-secret', + }) + + expect(inspectConfiguredOAuthClient('shopify')).toMatchObject({ + state: 'partial', + missingFields: ['SHOPIFY_CLIENT_ID'], + }) + expect(() => requireConfiguredOAuthClient('shopify')).toThrow(/SHOPIFY_CLIENT_ID/) + }) + + it('returns the validated values with capability-specific field types', () => { + setEnv({ + SHOPIFY_CLIENT_ID: 'shopify-client', + SHOPIFY_CLIENT_SECRET: 'shopify-secret', + }) + + const configured = requireConfiguredOAuthClient('shopify') + + expect(configured.values).toEqual({ + SHOPIFY_CLIENT_ID: 'shopify-client', + SHOPIFY_CLIENT_SECRET: 'shopify-secret', + }) + expectTypeOf(configured.values.SHOPIFY_CLIENT_ID).toEqualTypeOf() + expectTypeOf(configured.values.SHOPIFY_CLIENT_SECRET).toEqualTypeOf() + }) +}) diff --git a/apps/sim/lib/core/config/env-capabilities.server.ts b/apps/sim/lib/core/config/env-capabilities.server.ts new file mode 100644 index 00000000000..e412ba26ab4 --- /dev/null +++ b/apps/sim/lib/core/config/env-capabilities.server.ts @@ -0,0 +1,56 @@ +/** + * Binds the pure capability definitions to the application's validated server environment. + * + * @packageDocumentation + */ +import { env } from '@/lib/core/config/env' +import { + ASYNC_JOBS_CAPABILITY, + CACHE_CAPABILITY, + type ConfiguredOAuthClient, + type FallbackCapabilityDefinition, + inspectOAuthClientCapability, + type OAuthClientCapabilityField, + type OAuthClientCapabilityId, + requireCapability, + requireOAuthClientCapability, + SANDBOX_CAPABILITY, + STORAGE_CAPABILITY, + type WireFallbackOptions, + wireFallback, +} from '@/lib/core/config/env-capabilities' + +export function getConfiguredStorageProviderId() { + return requireCapability(STORAGE_CAPABILITY, env).providerId +} + +export function getConfiguredSandboxProviderId() { + return requireCapability(SANDBOX_CAPABILITY, env).providerId +} + +export function getConfiguredAsyncJobsProvider() { + return requireCapability(ASYNC_JOBS_CAPABILITY, env).providerId +} + +export function getConfiguredCacheProvider() { + return requireCapability(CACHE_CAPABILITY, env).providerId +} + +export function inspectConfiguredOAuthClient(serviceId: string) { + return inspectOAuthClientCapability(serviceId, env) +} + +export function requireConfiguredOAuthClient( + serviceId: TCapabilityId +): ConfiguredOAuthClient> +export function requireConfiguredOAuthClient(serviceId: string): ConfiguredOAuthClient +export function requireConfiguredOAuthClient(serviceId: string): ConfiguredOAuthClient { + return requireOAuthClientCapability(serviceId, env) +} + +export function wireServerFallback< + const TDefinition extends FallbackCapabilityDefinition, + TProvider, +>(options: Omit, 'values'>) { + return wireFallback({ ...options, values: env }) +} diff --git a/apps/sim/lib/core/config/env-capabilities.test.ts b/apps/sim/lib/core/config/env-capabilities.test.ts new file mode 100644 index 00000000000..c6d03d5b773 --- /dev/null +++ b/apps/sim/lib/core/config/env-capabilities.test.ts @@ -0,0 +1,702 @@ +import { describe, expect, it, vi } from 'vitest' +import { + ASYNC_JOBS_CAPABILITY, + CACHE_CAPABILITY, + DEPLOYMENT_CONFIGURATION_KEYS, + defineCapability, + EMAIL_CAPABILITY, + EnvCapabilityConfigurationError, + envField, + inspectCapability, + inspectOAuthClientCapability, + LLM_KEY_POOLS, + OCR_CAPABILITY, + requireCapability, + requireOAuthClientCapability, + resolveOAuthClientCapabilityId, + SANDBOX_CAPABILITY, + STORAGE_CAPABILITY, + validateCapabilityFieldInput, + wireFallback, +} from '@/lib/core/config/env-capabilities' +import integrationsJson from '@/lib/integrations/integrations.json' +import type { Integration } from '@/lib/integrations/types' +import { getServiceConfigByServiceId } from '@/lib/oauth/utils' + +const READY_STORAGE_VALUES = { + azure: { + AZURE_CONNECTION_STRING: 'UseDevelopmentStorage=true', + AZURE_STORAGE_CONTAINER_NAME: 'azure-files', + }, + s3: { + AWS_REGION: 'us-east-1', + S3_BUCKET_NAME: 's3-files', + }, + gcs: { + GCS_BUCKET_NAME: 'gcs-files', + }, +} as const + +const STORAGE_COMBINATIONS = [ + { azure: false, s3: false, gcs: false, expected: 'local' }, + { azure: false, s3: false, gcs: true, expected: 'gcs' }, + { azure: false, s3: true, gcs: false, expected: 's3' }, + { azure: false, s3: true, gcs: true, expected: 's3' }, + { azure: true, s3: false, gcs: false, expected: 'azure' }, + { azure: true, s3: false, gcs: true, expected: 'azure' }, + { azure: true, s3: true, gcs: false, expected: 'azure' }, + { azure: true, s3: true, gcs: true, expected: 'azure' }, +] as const + +const READY_EMAIL_VALUES = { + resend: { RESEND_API_KEY: 're_test' }, + ses: { AWS_SES_REGION: 'us-east-1' }, + smtp: { SMTP_HOST: 'localhost', SMTP_PORT: '1025' }, + azure: { AZURE_ACS_CONNECTION_STRING: 'endpoint=https://email.example.com' }, + gmail: { + GMAIL_CREDENTIALS_JSON: JSON.stringify({ + client_email: 'mailer@example.com', + private_key: 'private-key', + }), + GMAIL_SENDER: 'mailer@example.com', + }, +} as const + +const EMAIL_PROVIDER_ORDER = ['resend', 'ses', 'smtp', 'azure', 'gmail'] as const + +function storageValues({ + azure, + s3, + gcs, +}: Pick<(typeof STORAGE_COMBINATIONS)[number], 'azure' | 's3' | 'gcs'>): Record { + return Object.assign( + {}, + azure ? READY_STORAGE_VALUES.azure : {}, + s3 ? READY_STORAGE_VALUES.s3 : {}, + gcs ? READY_STORAGE_VALUES.gcs : {} + ) +} + +describe('env capabilities', () => { + it('fails fast on invalid runtime capability definitions', () => { + const provider = { + id: 'remote', + label: 'Remote', + activation: { mode: 'any-present', keys: ['REMOTE_KEY'] } as const, + requires: envField('REMOTE_KEY'), + } + const base = { + strategy: 'selected', + id: 'sample', + label: 'Sample', + whenUnset: 'default', + } as const + + expect(() => + defineCapability({ + ...base, + defaultProvider: { id: 'remote', kind: 'provider' }, + providers: [provider, provider], + }) + ).toThrow(/duplicate provider ids/) + expect(() => + defineCapability({ + ...base, + defaultProvider: { id: 'missing', kind: 'provider' }, + providers: [provider], + }) + ).toThrow(/default provider missing is not declared/) + }) + + describe('fallback capabilities', () => { + it('resolves every ready email provider subset in declaration order', () => { + for (let mask = 0; mask < 1 << EMAIL_PROVIDER_ORDER.length; mask += 1) { + const expected = EMAIL_PROVIDER_ORDER.filter((_, index) => (mask & (1 << index)) !== 0) + const values = Object.assign( + {}, + ...expected.map((providerId) => READY_EMAIL_VALUES[providerId]) + ) + + expect( + inspectCapability(EMAIL_CAPABILITY, values).providerIds, + `provider mask ${mask}` + ).toEqual(expected) + } + }) + + it('reports a broken email provider only when no ready alternative exists', () => { + const partialOnly = inspectCapability(EMAIL_CAPABILITY, { SMTP_HOST: 'localhost' }) + expect(partialOnly).toMatchObject({ + configured: false, + providerIds: [], + error: expect.any(EnvCapabilityConfigurationError), + }) + expect(() => requireCapability(EMAIL_CAPABILITY, { SMTP_HOST: 'localhost' })).toThrow( + /SMTP_PORT/ + ) + + const withAlternative = inspectCapability(EMAIL_CAPABILITY, { + RESEND_API_KEY: 're_test', + SMTP_HOST: 'localhost', + }) + expect(withAlternative).toMatchObject({ + configured: true, + providerIds: ['resend'], + error: null, + }) + expect(withAlternative.providers.find((provider) => provider.id === 'smtp')).toMatchObject({ + state: 'partial', + missingFields: ['SMTP_PORT'], + }) + + const withMalformedAlternative = inspectCapability(EMAIL_CAPABILITY, { + RESEND_API_KEY: 're_test', + GMAIL_CREDENTIALS_JSON: '{}', + GMAIL_SENDER: 'mailer@example.com', + }) + expect(withMalformedAlternative).toMatchObject({ + configured: true, + providerIds: ['resend'], + error: null, + }) + expect( + withMalformedAlternative.providers.find((provider) => provider.id === 'gmail') + ).toMatchObject({ state: 'invalid', invalidFields: ['GMAIL_CREDENTIALS_JSON'] }) + expect(() => + requireCapability(EMAIL_CAPABILITY, { + GMAIL_CREDENTIALS_JSON: '{}', + GMAIL_SENDER: 'mailer@example.com', + }) + ).toThrow(/GMAIL_CREDENTIALS_JSON/) + }) + + it('preserves anonymous SMTP when only one optional auth field is set', () => { + expect( + requireCapability(EMAIL_CAPABILITY, { + SMTP_HOST: 'localhost', + SMTP_PORT: '1025', + SMTP_USER: 'unused-for-anonymous-relay', + }).providerIds + ).toEqual(['smtp']) + }) + + it('validates setup input with the canonical field rules', () => { + expect(validateCapabilityFieldInput(EMAIL_CAPABILITY, 'SMTP_PORT', '')).toBe('required') + expect(validateCapabilityFieldInput(EMAIL_CAPABILITY, 'SMTP_PORT', '1025')).toBeUndefined() + expect(validateCapabilityFieldInput(EMAIL_CAPABILITY, 'SMTP_PORT', '99999')).toMatch( + /valid port/i + ) + expect( + validateCapabilityFieldInput(EMAIL_CAPABILITY, 'GMAIL_CREDENTIALS_JSON', '{}') + ).toMatch(/service account/i) + expect(() => + validateCapabilityFieldInput(EMAIL_CAPABILITY, 'UNKNOWN_EMAIL_FIELD', 'value') + ).toThrow(/no validation definition/i) + }) + + it('executes email providers in order and stops after the first success', async () => { + const resend = { send: vi.fn().mockRejectedValue(new Error('resend down')) } + const ses = { send: vi.fn().mockResolvedValue('sent') } + const smtp = { send: vi.fn().mockResolvedValue('should not run') } + const onFailure = vi.fn() + const fallback = wireFallback({ + definition: EMAIL_CAPABILITY, + values: { + RESEND_API_KEY: 're_test', + AWS_SES_REGION: 'us-east-1', + SMTP_HOST: 'localhost', + SMTP_PORT: '1025', + }, + factories: { + resend: () => resend, + ses: () => ses, + smtp: () => smtp, + azure: () => null, + gmail: () => null, + }, + onFailure, + }) + + await expect(fallback.execute((provider) => provider.send())).resolves.toBe('sent') + expect(resend.send).toHaveBeenCalledOnce() + expect(ses.send).toHaveBeenCalledOnce() + expect(smtp.send).not.toHaveBeenCalled() + expect(onFailure).toHaveBeenCalledWith('resend', expect.any(Error)) + }) + + it('aggregates failures after every ready email provider fails', async () => { + const resendError = new Error('resend down') + const sesError = new Error('ses down') + const resend = { send: vi.fn().mockRejectedValue(resendError) } + const ses = { send: vi.fn().mockRejectedValue(sesError) } + const onFailure = vi.fn() + const fallback = wireFallback({ + definition: EMAIL_CAPABILITY, + values: { RESEND_API_KEY: 're_test', AWS_SES_REGION: 'us-east-1' }, + factories: { + resend: () => resend, + ses: () => ses, + smtp: () => null, + azure: () => null, + gmail: () => null, + }, + onFailure, + }) + + const rejection = fallback.execute((provider) => provider.send()) + await expect(rejection).rejects.toThrow(/All Email providers failed: resend, ses/) + await expect(rejection).rejects.toMatchObject({ errors: [resendError, sesError] }) + expect(onFailure.mock.calls.map(([providerId]) => providerId)).toEqual(['resend', 'ses']) + }) + + it('fails immediately when a ready provider has no runtime implementation', () => { + expect(() => + wireFallback({ + definition: EMAIL_CAPABILITY, + values: { RESEND_API_KEY: 're_test' }, + factories: { + resend: () => null, + ses: () => null, + smtp: () => null, + azure: () => null, + gmail: () => null, + }, + }) + ).toThrow(/factory returned null/) + }) + }) + + describe('storage selection', () => { + it('preserves legacy Azure, S3, GCS, local precedence for unset and blank selectors', () => { + for (const selector of [undefined, '', ' '] as const) { + for (const combination of STORAGE_COMBINATIONS) { + const values = { + ...storageValues(combination), + ...(selector === undefined ? {} : { STORAGE_PROVIDER: selector }), + } + expect( + inspectCapability(STORAGE_CAPABILITY, values).providerId, + `selector=${JSON.stringify(selector)} combination=${JSON.stringify(combination)}` + ).toBe(combination.expected) + } + } + }) + + it.each([ + { + name: 'partial Azure before ready S3', + values: { + AZURE_STORAGE_CONTAINER_NAME: 'azure-files', + ...READY_STORAGE_VALUES.s3, + }, + expected: 's3', + }, + { + name: 'partial Azure before ready GCS', + values: { + AZURE_STORAGE_CONTAINER_NAME: 'azure-files', + ...READY_STORAGE_VALUES.gcs, + }, + expected: 'gcs', + }, + { + name: 'partial S3 before ready GCS', + values: { + S3_BUCKET_NAME: 's3-files', + ...READY_STORAGE_VALUES.gcs, + }, + expected: 'gcs', + }, + { + name: 'partial Azure and S3 before ready GCS', + values: { + AZURE_STORAGE_CONTAINER_NAME: 'azure-files', + S3_BUCKET_NAME: 's3-files', + ...READY_STORAGE_VALUES.gcs, + }, + expected: 'gcs', + }, + ])('skips $name', ({ values, expected }) => { + expect(requireCapability(STORAGE_CAPABILITY, values).providerId).toBe(expected) + }) + + it('accepts both legacy Azure credential forms and GCS application-default credentials', () => { + expect( + requireCapability(STORAGE_CAPABILITY, { + AZURE_ACCOUNT_NAME: 'storage-account', + AZURE_ACCOUNT_KEY: 'storage-key', + AZURE_STORAGE_CONTAINER_NAME: 'azure-files', + }).providerId + ).toBe('azure') + expect( + requireCapability(STORAGE_CAPABILITY, { GCS_BUCKET_NAME: 'gcs-files' }).providerId + ).toBe('gcs') + }) + + it('does not activate S3 from general AWS credentials alone', () => { + expect( + requireCapability(STORAGE_CAPABILITY, { + AWS_REGION: 'us-east-1', + AWS_ACCESS_KEY_ID: 'access', + AWS_SECRET_ACCESS_KEY: 'secret', + }).providerId + ).toBe('local') + }) + + it('fails fast when legacy storage configuration is partial and no provider is ready', () => { + expect(() => + requireCapability(STORAGE_CAPABILITY, { + AZURE_STORAGE_CONTAINER_NAME: 'azure-files', + }) + ).toThrow(/AZURE_CONNECTION_STRING/) + expect(() => + requireCapability(STORAGE_CAPABILITY, { + S3_BUCKET_NAME: 's3-files', + }) + ).toThrow(/AWS_REGION/) + }) + + it('reports one sufficient Azure credential repair path', () => { + const inspection = inspectCapability(STORAGE_CAPABILITY, { + AZURE_ACCOUNT_NAME: 'storage-account', + AZURE_STORAGE_CONTAINER_NAME: 'azure-files', + }) + const azure = inspection.providers.find((provider) => provider.id === 'azure') + expect(azure?.missingFields).toHaveLength(1) + expect(azure?.missingFields[0]).toMatch(/AZURE_(CONNECTION_STRING|ACCOUNT_KEY)/) + }) + + it('requires paired S3 credentials when either static credential is present', () => { + expect( + requireCapability(STORAGE_CAPABILITY, { + ...READY_STORAGE_VALUES.s3, + AWS_ACCESS_KEY_ID: 'access', + AWS_SECRET_ACCESS_KEY: 'secret', + }).providerId + ).toBe('s3') + expect(() => + requireCapability(STORAGE_CAPABILITY, { + STORAGE_PROVIDER: 's3', + ...READY_STORAGE_VALUES.s3, + AWS_ACCESS_KEY_ID: 'access', + }) + ).toThrow(/AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set together/) + }) + + it('validates only the explicitly selected storage provider', () => { + expect( + requireCapability(STORAGE_CAPABILITY, { + STORAGE_PROVIDER: 'gcs', + GCS_BUCKET_NAME: 'gcs-files', + S3_ENDPOINT: 'ftp://storage.example.com', + }).providerId + ).toBe('gcs') + expect( + requireCapability(STORAGE_CAPABILITY, { + STORAGE_PROVIDER: 'local', + AZURE_STORAGE_CONTAINER_NAME: 'partial-azure', + }).providerId + ).toBe('local') + }) + + it('reports missing or invalid fields for an explicitly selected storage provider', () => { + const missing = inspectCapability(STORAGE_CAPABILITY, { + STORAGE_PROVIDER: 's3', + S3_BUCKET_NAME: 's3-files', + }) + expect(missing).toMatchObject({ + providerId: 's3', + error: expect.any(EnvCapabilityConfigurationError), + }) + expect(() => + requireCapability(STORAGE_CAPABILITY, { + STORAGE_PROVIDER: 's3', + S3_BUCKET_NAME: 's3-files', + }) + ).toThrow(/AWS_REGION/) + expect(() => + requireCapability(STORAGE_CAPABILITY, { + STORAGE_PROVIDER: 's3', + ...READY_STORAGE_VALUES.s3, + S3_ENDPOINT: 'ftp://storage.example.com', + }) + ).toThrow(/S3_ENDPOINT/) + expect(inspectCapability(STORAGE_CAPABILITY, { STORAGE_PROVIDER: 'unknown' })).toMatchObject({ + providerId: null, + error: expect.any(EnvCapabilityConfigurationError), + }) + }) + + it('fails fast on a complete invalid higher-priority legacy provider', () => { + expect(() => + requireCapability(STORAGE_CAPABILITY, { + ...READY_STORAGE_VALUES.s3, + S3_ENDPOINT: 'ftp://storage.example.com', + ...READY_STORAGE_VALUES.gcs, + }) + ).toThrow(/S3_ENDPOINT/) + }) + }) + + describe('OCR selection', () => { + it('preserves Azure, Mistral, local precedence for every unset-provider combination', () => { + const azureFields = [ + ['OCR_AZURE_API_KEY', 'azure-key'], + ['OCR_AZURE_ENDPOINT', 'https://ocr.example.com'], + ['OCR_AZURE_MODEL_NAME', 'mistral-ocr'], + ] as const + + for (const selector of [undefined, '', ' '] as const) { + for (let mask = 0; mask < 1 << (azureFields.length + 1); mask += 1) { + const values: Record = {} + azureFields.forEach(([key, value], index) => { + if ((mask & (1 << index)) !== 0) values[key] = value + }) + const hasMistral = (mask & (1 << azureFields.length)) !== 0 + if (hasMistral) values.MISTRAL_API_KEY = 'mistral-key' + if (selector !== undefined) values.OCR_PROVIDER = selector + + const azureComplete = (mask & 0b111) === 0b111 + const azurePartial = (mask & 0b111) !== 0 && !azureComplete + const inspection = inspectCapability(OCR_CAPABILITY, values) + + if (azureComplete) { + expect(inspection.providerId, `selector=${JSON.stringify(selector)} mask=${mask}`).toBe( + 'azure-mistral' + ) + expect(inspection.error).toBeNull() + } else if (hasMistral) { + expect(inspection.providerId, `selector=${JSON.stringify(selector)} mask=${mask}`).toBe( + 'mistral' + ) + expect(inspection.error).toBeNull() + } else if (azurePartial) { + expect(inspection.error).toBeInstanceOf(EnvCapabilityConfigurationError) + expect(() => requireCapability(OCR_CAPABILITY, values)).toThrow() + } else { + expect(inspection).toMatchObject({ providerId: 'local', error: null }) + } + } + } + }) + + it('validates only the explicitly selected OCR provider', () => { + expect( + requireCapability(OCR_CAPABILITY, { + OCR_PROVIDER: 'local', + MISTRAL_API_KEY: 'mistral-key', + }).providerId + ).toBe('local') + expect(() => requireCapability(OCR_CAPABILITY, { OCR_PROVIDER: 'mistral' })).toThrow( + /MISTRAL_API_KEY/ + ) + expect(() => + requireCapability(OCR_CAPABILITY, { + OCR_PROVIDER: 'azure-mistral', + OCR_AZURE_API_KEY: 'azure-key', + OCR_AZURE_ENDPOINT: 'ftp://ocr.example.com', + OCR_AZURE_MODEL_NAME: 'mistral-ocr', + }) + ).toThrow(/OCR_AZURE_ENDPOINT/) + expect(inspectCapability(OCR_CAPABILITY, { OCR_PROVIDER: 'unknown' })).toMatchObject({ + providerId: null, + error: expect.any(EnvCapabilityConfigurationError), + }) + }) + + it('preserves legacy Azure validation precedence over Mistral', () => { + expect(() => + requireCapability(OCR_CAPABILITY, { + OCR_AZURE_API_KEY: 'azure-key', + OCR_AZURE_ENDPOINT: 'ftp://ocr.example.com', + OCR_AZURE_MODEL_NAME: 'mistral-ocr', + MISTRAL_API_KEY: 'mistral-key', + }) + ).toThrow(/OCR_AZURE_ENDPOINT/) + expect( + requireCapability(OCR_CAPABILITY, { + OCR_AZURE_ENDPOINT: 'ftp://ocr.example.com', + MISTRAL_API_KEY: 'mistral-key', + }).providerId + ).toBe('mistral') + }) + }) + + describe('sandbox selection', () => { + it('inspects the legacy E2B default without throwing but requires runtime configuration', () => { + const inspection = inspectCapability(SANDBOX_CAPABILITY, {}) + expect(inspection).toMatchObject({ + providerId: 'e2b', + error: null, + }) + expect(() => requireCapability(SANDBOX_CAPABILITY, {})).toThrow(/E2B_API_KEY/) + }) + + it('requires E2B credentials when E2B is selected', () => { + expect( + requireCapability(SANDBOX_CAPABILITY, { + E2B_ENABLED: 'true', + E2B_API_KEY: 'e2b-key', + }).providerId + ).toBe('e2b') + expect(() => + requireCapability(SANDBOX_CAPABILITY, { + SANDBOX_PROVIDER: 'e2b', + E2B_ENABLED: 'false', + }) + ).toThrow(/E2B_API_KEY.*E2B_ENABLED/) + expect(() => + requireCapability(SANDBOX_CAPABILITY, { + SANDBOX_PROVIDER: 'e2b', + E2B_ENABLED: 'false', + E2B_API_KEY: 'e2b-key', + }) + ).toThrow(/E2B_ENABLED must be enabled/) + const disabled = inspectCapability(SANDBOX_CAPABILITY, { + SANDBOX_PROVIDER: 'e2b', + E2B_ENABLED: 'false', + }) + expect(disabled).toMatchObject({ providerId: 'e2b', error: null }) + expect(disabled.providers.find((provider) => provider.id === 'e2b')).toMatchObject({ + active: false, + state: 'absent', + }) + }) + + it('requires Daytona credentials and a pinned shell snapshot', () => { + expect( + requireCapability(SANDBOX_CAPABILITY, { + SANDBOX_PROVIDER: 'daytona', + DAYTONA_API_KEY: 'daytona-key', + DAYTONA_SHELL_SNAPSHOT_ID: 'mothership-shell:v1', + }).providerId + ).toBe('daytona') + expect(() => + requireCapability(SANDBOX_CAPABILITY, { + SANDBOX_PROVIDER: 'daytona', + DAYTONA_API_KEY: 'daytona-key', + }) + ).toThrow(/DAYTONA_SHELL_SNAPSHOT_ID/) + for (const snapshot of ['mothership-shell', 'mothership-shell:latest']) { + expect(() => + requireCapability(SANDBOX_CAPABILITY, { + SANDBOX_PROVIDER: 'daytona', + DAYTONA_API_KEY: 'daytona-key', + DAYTONA_SHELL_SNAPSHOT_ID: snapshot, + }) + ).toThrow(/explicit, non-floating name:tag/) + } + }) + + it('reports an unknown sandbox selector without throwing during inspection', () => { + expect(inspectCapability(SANDBOX_CAPABILITY, { SANDBOX_PROVIDER: 'unknown' })).toMatchObject({ + providerId: null, + error: expect.any(EnvCapabilityConfigurationError), + }) + expect(() => requireCapability(SANDBOX_CAPABILITY, { SANDBOX_PROVIDER: 'unknown' })).toThrow( + /Unknown SANDBOX_PROVIDER/ + ) + }) + }) + + describe('jobs and cache selection', () => { + it('uses database jobs unless Trigger.dev is enabled and configured', () => { + expect(requireCapability(ASYNC_JOBS_CAPABILITY, {}).providerId).toBe('database') + expect( + requireCapability(ASYNC_JOBS_CAPABILITY, { TRIGGER_DEV_ENABLED: 'false' }).providerId + ).toBe('database') + expect( + requireCapability(ASYNC_JOBS_CAPABILITY, { + TRIGGER_DEV_ENABLED: 'true', + TRIGGER_PROJECT_ID: 'project-id', + TRIGGER_SECRET_KEY: 'secret-key', + }).providerId + ).toBe('trigger-dev') + expect(() => + requireCapability(ASYNC_JOBS_CAPABILITY, { + TRIGGER_DEV_ENABLED: 'true', + TRIGGER_PROJECT_ID: 'project-id', + }) + ).toThrow(/TRIGGER_SECRET_KEY/) + }) + + it('uses Redis only when REDIS_URL is present and valid', () => { + expect(requireCapability(CACHE_CAPABILITY, {}).providerId).toBe('database') + expect( + requireCapability(CACHE_CAPABILITY, { REDIS_URL: 'redis://cache.example.com:6379' }) + .providerId + ).toBe('redis') + expect(() => + requireCapability(CACHE_CAPABILITY, { REDIS_URL: 'https://cache.example.com' }) + ).toThrow(/redis:\/\/ or rediss:\/\//) + }) + + it('requires a TLS server name for rediss IP addresses', () => { + expect(() => + requireCapability(CACHE_CAPABILITY, { REDIS_URL: 'rediss://10.0.0.1:6379' }) + ).toThrow(/REDIS_TLS_SERVERNAME/) + expect( + requireCapability(CACHE_CAPABILITY, { + REDIS_URL: 'rediss://10.0.0.1:6379', + REDIS_TLS_SERVERNAME: 'cache.internal', + }).providerId + ).toBe('redis') + expect( + requireCapability(CACHE_CAPABILITY, { + REDIS_URL: 'rediss://cache.example.com:6379', + }).providerId + ).toBe('redis') + }) + }) + + describe('OAuth and deployment metadata', () => { + it('uses exact OAuth environment names and reports partial pairs', () => { + expect(inspectOAuthClientCapability('zoho-desk', { ZOHO_CLIENT_ID: 'client' })).toMatchObject( + { + state: 'partial', + missingFields: ['ZOHO_CLIENT_SECRET'], + } + ) + }) + + it('fails fast when an OAuth client is partially configured', () => { + expect(() => requireOAuthClientCapability('slack', { SLACK_CLIENT_ID: 'client' })).toThrow( + /SLACK_CLIENT_SECRET/ + ) + }) + + it('covers every OAuth integration', () => { + const integrations = integrationsJson.integrations as readonly Integration[] + const uncovered = integrations.flatMap((integration) => { + if (integration.authType !== 'oauth' || !integration.oauthServiceId) return [] + if (resolveOAuthClientCapabilityId(integration.oauthServiceId)) return [] + return [integration.slug] + }) + + expect(uncovered).toEqual([]) + expect(getServiceConfigByServiceId('trello')?.serviceAccountProviderId).toBe( + 'trello-service-account' + ) + }) + + it('tracks setup-owned options as deployment configuration', () => { + expect(DEPLOYMENT_CONFIGURATION_KEYS).toEqual( + expect.arrayContaining([ + 'DAYTONA_SHELL_SNAPSHOT_ID', + 'S3_FORCE_PATH_STYLE', + 'STORAGE_PROVIDER', + 'OCR_PROVIDER', + ]) + ) + }) + + it('tracks singular runtime LLM keys as pool fallbacks and deployment configuration', () => { + expect(LLM_KEY_POOLS.openai.fallbackKey).toBe('OPENAI_API_KEY') + expect(LLM_KEY_POOLS.gemini.fallbackKey).toBe('GEMINI_API_KEY') + expect(LLM_KEY_POOLS.cohere.fallbackKey).toBe('COHERE_API_KEY') + expect(DEPLOYMENT_CONFIGURATION_KEYS).toEqual( + expect.arrayContaining(['OPENAI_API_KEY', 'GEMINI_API_KEY', 'COHERE_API_KEY']) + ) + }) + }) +}) diff --git a/apps/sim/lib/core/config/env-capabilities.ts b/apps/sim/lib/core/config/env-capabilities.ts new file mode 100644 index 00000000000..dd803bb6585 --- /dev/null +++ b/apps/sim/lib/core/config/env-capabilities.ts @@ -0,0 +1,1395 @@ +/** + * Canonical runtime deployment-capability definitions. Keep this module dependency-free so + * setup and diagnostics can consume the provider rules the application actually enforces. + * + * @packageDocumentation + */ +export type EnvCapabilityValue = string | number | boolean | null | undefined + +export const CORE_CONFIGURATION_KEYS = [ + 'DATABASE_URL', + 'BETTER_AUTH_SECRET', + 'BETTER_AUTH_URL', + 'NEXT_PUBLIC_APP_URL', + 'ENCRYPTION_KEY', + 'INTERNAL_API_SECRET', +] as const + +export type EnvCapabilityValues = + | ReadonlyMap + | Readonly> + +export type EnvValueValidation = + | { + kind: 'integer' + min?: number + max?: number + message: string + } + | { + kind: 'json-object' + requiredStringFields?: readonly string[] + message: string + } + | { + kind: 'pattern' + pattern: RegExp + message: string + } + | { + kind: 'url' + protocols?: readonly string[] + message: string + } + +export interface EnvFieldRequirement { + type: 'field' + key: string + validation?: EnvValueValidation +} + +export interface AllOfRequirement { + type: 'allOf' + requirements: readonly EnvRequirement[] +} + +export interface AnyOfRequirement { + type: 'anyOf' + requirements: readonly EnvRequirement[] +} + +export type EnvRequirement = EnvFieldRequirement | AllOfRequirement | AnyOfRequirement + +export type EnvProviderActivation = + | { mode: 'any-present'; keys: readonly string[] } + | { mode: 'enabled'; key: string } + +export interface EnvProviderValidationIssue { + kind: 'missing' | 'invalid' + fields: readonly string[] + message: string +} + +export interface EnvProviderDefinition { + id: TId + label: string + activation: EnvProviderActivation + requires: EnvRequirement + pairedFields?: readonly (readonly [string, string])[] + optionalFields?: readonly EnvFieldRequirement[] + validate?: (values: EnvCapabilityValues) => readonly EnvProviderValidationIssue[] +} + +export interface FallbackCapabilityDefinition< + TId extends string = string, + TProvider extends EnvProviderDefinition = EnvProviderDefinition, +> { + strategy: 'fallback' + id: TId + label: string + providers: readonly TProvider[] +} + +export type EnvDefaultProviderDefinition = + | { id: string; kind: 'built-in'; label: string } + | { id: string; kind: 'provider' } + +export interface SelectedCapabilityDefinition< + TId extends string = string, + TProvider extends EnvProviderDefinition = EnvProviderDefinition, +> { + strategy: 'selected' + id: TId + label: string + selectorKey?: string + whenUnset: 'default' | 'first-ready' + defaultProvider: EnvDefaultProviderDefinition + providers: readonly TProvider[] +} + +export type CapabilityDefinition = FallbackCapabilityDefinition | SelectedCapabilityDefinition + +export type DeclaredProviderId = + TDefinition['providers'][number]['id'] + +export type ProviderId = + TDefinition extends SelectedCapabilityDefinition + ? DeclaredProviderId | TDefinition['defaultProvider']['id'] + : DeclaredProviderId + +export type FallbackFactories = { + [TId in DeclaredProviderId]: () => TProvider | null +} + +export type ProviderConfigurationState = 'absent' | 'partial' | 'ready' | 'invalid' + +export interface ProviderInspection { + id: TId + label: string + active: boolean + state: ProviderConfigurationState + missingFields: readonly string[] + invalidFields: readonly string[] + invalidDetails: readonly string[] +} + +export interface FallbackCapabilityInspection { + strategy: 'fallback' + configured: boolean + providerIds: readonly TId[] + providers: readonly ProviderInspection[] + error: EnvCapabilityConfigurationError | null +} + +export interface SelectedCapabilityInspection< + TProviderId extends string = string, + TDeclaredProviderId extends string = TProviderId, +> { + strategy: 'selected' + providerId: TProviderId | null + providers: readonly ProviderInspection[] + error: EnvCapabilityConfigurationError | null +} + +export type CapabilityInspection = + TDefinition extends SelectedCapabilityDefinition + ? SelectedCapabilityInspection, DeclaredProviderId> + : FallbackCapabilityInspection> + +export class EnvCapabilityConfigurationError extends Error { + constructor( + readonly capabilityId: string, + message: string + ) { + super(message) + this.name = 'EnvCapabilityConfigurationError' + } +} + +function readValue(values: EnvCapabilityValues, key: string): EnvCapabilityValue { + if (values instanceof Map) return values.get(key) + return (values as Readonly>)[key] +} + +function hasValue(values: EnvCapabilityValues, key: string): boolean { + const value = readValue(values, key) + if (value === undefined || value === null || value === false) return false + if (typeof value !== 'string') return true + const normalized = value.trim().toLowerCase() + return normalized !== '' && normalized !== 'placeholder' +} + +function isTruthyValue(values: EnvCapabilityValues, key: string): boolean { + const value = readValue(values, key) + if (value === true || value === 1) return true + if (typeof value !== 'string') return false + const normalized = value.toLowerCase() + return normalized === 'true' || normalized === '1' +} + +/** Returns whether an environment field contains a usable configuration value. */ +export function hasEnvCapabilityValue(values: EnvCapabilityValues, key: string): boolean { + return hasValue(values, key) +} + +/** Resolves the boolean semantics shared by capability selectors and status reporting. */ +export function isTruthyEnvCapabilityValue(values: EnvCapabilityValues, key: string): boolean { + return isTruthyValue(values, key) +} + +function unique(values: readonly string[]): string[] { + return [...new Set(values)] +} + +export function envField( + key: string, + options: Pick = {} +): EnvFieldRequirement { + return { type: 'field', key, ...options } +} + +export function allOf(...requirements: readonly EnvRequirement[]): AllOfRequirement { + return { type: 'allOf', requirements } +} + +export function anyOf(...requirements: readonly EnvRequirement[]): AnyOfRequirement { + return { type: 'anyOf', requirements } +} + +function requirementKeys(requirement: EnvRequirement): string[] { + return requirement.type === 'field' + ? [requirement.key] + : requirement.requirements.flatMap(requirementKeys) +} + +function activationKeys(activation: EnvProviderActivation): readonly string[] { + return activation.mode === 'enabled' ? [activation.key] : activation.keys +} + +function providerKeys(provider: EnvProviderDefinition): string[] { + return [ + ...activationKeys(provider.activation), + ...requirementKeys(provider.requires), + ...(provider.pairedFields ?? []).flat(), + ...(provider.optionalFields ?? []).map((field) => field.key), + ] +} + +/** Returns every environment field that can affect one provider at runtime. */ +export function getProviderFields(provider: EnvProviderDefinition): readonly string[] { + return unique(providerKeys(provider)) +} + +function providerIsActive(provider: EnvProviderDefinition, values: EnvCapabilityValues): boolean { + return provider.activation.mode === 'enabled' + ? isTruthyValue(values, provider.activation.key) + : provider.activation.keys.some((key) => hasValue(values, key)) +} + +function capabilityKeys(definition: CapabilityDefinition): string[] { + return [ + ...(definition.strategy === 'selected' && definition.selectorKey + ? [definition.selectorKey] + : []), + ...definition.providers.flatMap(providerKeys), + ] +} + +/** Returns every environment field that can affect a capability at runtime. */ +export function getCapabilityFields(definition: CapabilityDefinition): readonly string[] { + return unique(capabilityKeys(definition)) +} + +function assertRequirementDefinition( + capabilityId: string, + providerId: string, + requirement: EnvRequirement +): void { + if (requirement.type === 'field') { + if (!requirement.key) { + throw new Error(`Capability ${capabilityId} provider ${providerId} has an empty field key`) + } + return + } + if (requirement.requirements.length === 0) { + throw new Error( + `Capability ${capabilityId} provider ${providerId} has an empty ${requirement.type}` + ) + } + for (const child of requirement.requirements) { + assertRequirementDefinition(capabilityId, providerId, child) + } +} + +function assertCapabilityDefinition(definition: CapabilityDefinition): void { + if (definition.providers.length === 0) { + throw new Error(`Capability ${definition.id} must declare at least one provider`) + } + + const providerIds = definition.providers.map((provider) => provider.id) + if (new Set(providerIds).size !== providerIds.length) { + throw new Error(`Capability ${definition.id} has duplicate provider ids`) + } + + if (definition.strategy === 'selected') { + const defaultIsDeclared = providerIds.includes(definition.defaultProvider.id) + if (definition.defaultProvider.kind === 'provider' && !defaultIsDeclared) { + throw new Error( + `Capability ${definition.id} default provider ${definition.defaultProvider.id} is not declared` + ) + } + if (definition.defaultProvider.kind === 'built-in' && defaultIsDeclared) { + throw new Error( + `Capability ${definition.id} built-in default ${definition.defaultProvider.id} also appears in providers` + ) + } + } + + for (const provider of definition.providers) { + if (provider.activation.mode === 'any-present' && provider.activation.keys.length === 0) { + throw new Error(`Capability ${definition.id} provider ${provider.id} has no activation keys`) + } + assertRequirementDefinition(definition.id, provider.id, provider.requires) + } +} + +export function defineCapability( + definition: TDefinition +): TDefinition { + assertCapabilityDefinition(definition) + return definition +} + +/** Returns the canonical command for configuring a runtime capability. */ +export function getCapabilitySetupCommand(definition: CapabilityDefinition): string { + return `bun run setup ${definition.id}` +} + +interface RequirementInspection { + ready: boolean + missingFields: readonly string[] + invalidFields: readonly string[] + invalidDetails: readonly string[] +} + +function isValidEnvCapabilityFieldValue( + validation: EnvValueValidation, + value: EnvCapabilityValue +): boolean { + const serialized = String(value) + if (validation.kind === 'integer') { + const number = Number(serialized) + return ( + Number.isInteger(number) && + (validation.min === undefined || number >= validation.min) && + (validation.max === undefined || number <= validation.max) + ) + } + if (validation.kind === 'json-object') { + try { + const parsed: unknown = JSON.parse(serialized) + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) return false + return (validation.requiredStringFields ?? []).every( + (field) => + field in parsed && + typeof (parsed as Record)[field] === 'string' && + ((parsed as Record)[field] as string).length > 0 + ) + } catch { + return false + } + } + if (validation.kind === 'pattern') { + validation.pattern.lastIndex = 0 + return validation.pattern.test(serialized) + } + try { + const parsed = new URL(serialized) + return !validation.protocols || validation.protocols.includes(parsed.protocol) + } catch { + return false + } +} + +function inspectField( + requirement: EnvFieldRequirement, + values: EnvCapabilityValues +): RequirementInspection { + if (!hasValue(values, requirement.key)) { + return { + ready: false, + missingFields: [requirement.key], + invalidFields: [], + invalidDetails: [], + } + } + + const value = readValue(values, requirement.key) + const valid = requirement.validation + ? isValidEnvCapabilityFieldValue(requirement.validation, value) + : true + + return valid + ? { ready: true, missingFields: [], invalidFields: [], invalidDetails: [] } + : { + ready: false, + missingFields: [], + invalidFields: [requirement.key], + invalidDetails: [`${requirement.key} ${requirement.validation?.message ?? 'is invalid'}`], + } +} + +function inspectRequirement( + requirement: EnvRequirement, + values: EnvCapabilityValues +): RequirementInspection { + if (requirement.type === 'field') return inspectField(requirement, values) + + const inspections = requirement.requirements.map((child) => inspectRequirement(child, values)) + if (requirement.type === 'anyOf') { + const ready = inspections.find((inspection) => inspection.ready) + if (ready) return ready + return inspections.reduce((best, candidate) => { + const bestIssueCount = best.missingFields.length + best.invalidFields.length + const candidateIssueCount = candidate.missingFields.length + candidate.invalidFields.length + return candidateIssueCount < bestIssueCount ? candidate : best + }) + } + + return { + ready: inspections.every((inspection) => inspection.ready), + missingFields: unique(inspections.flatMap((inspection) => inspection.missingFields)), + invalidFields: unique(inspections.flatMap((inspection) => inspection.invalidFields)), + invalidDetails: unique(inspections.flatMap((inspection) => inspection.invalidDetails)), + } +} + +function inspectProviderRequirements( + provider: TProvider, + values: EnvCapabilityValues, + active = true +): ProviderInspection { + const inspection = inspectRequirement(provider.requires, values) + const invalidOptionalFields = (provider.optionalFields ?? []).flatMap((field) => { + if (!hasValue(values, field.key)) return [] + return inspectField(field, values).invalidFields + }) + const invalidPairs = (provider.pairedFields ?? []).flatMap(([left, right]) => + hasValue(values, left) === hasValue(values, right) ? [] : [left, right] + ) + const customIssues = provider.validate?.(values) ?? [] + const missingFields = unique([ + ...inspection.missingFields, + ...customIssues + .filter((customIssue) => customIssue.kind === 'missing') + .flatMap((customIssue) => customIssue.fields), + ]) + const invalidFields = unique([ + ...inspection.invalidFields, + ...invalidOptionalFields, + ...invalidPairs, + ...customIssues + .filter((customIssue) => customIssue.kind === 'invalid') + .flatMap((customIssue) => customIssue.fields), + ]) + const invalidDetails = unique([ + ...inspection.invalidDetails, + ...(provider.optionalFields ?? []).flatMap((field) => { + if (!hasValue(values, field.key)) return [] + return inspectField(field, values).invalidDetails + }), + ...(provider.pairedFields ?? []).flatMap(([left, right]) => + hasValue(values, left) === hasValue(values, right) + ? [] + : [`${left} and ${right} must be set together`] + ), + ...customIssues.map((customIssue) => customIssue.message), + ]) + return { + id: provider.id, + label: provider.label, + active, + state: + invalidFields.length > 0 + ? 'invalid' + : missingFields.length > 0 || !inspection.ready + ? 'partial' + : 'ready', + missingFields, + invalidFields, + invalidDetails, + } +} + +function inspectRequiredProvider( + provider: TProvider, + values: EnvCapabilityValues +): ProviderInspection { + const active = providerIsActive(provider, values) + const inspection = inspectProviderRequirements(provider, values, active) + if (active || provider.activation.mode !== 'enabled') return inspection + + return { + ...inspection, + state: 'invalid', + invalidFields: unique([...inspection.invalidFields, provider.activation.key]), + invalidDetails: unique([ + ...inspection.invalidDetails, + `${provider.activation.key} must be enabled`, + ]), + } +} + +export function inspectProvider( + provider: TProvider, + values: EnvCapabilityValues +): ProviderInspection { + const active = providerIsActive(provider, values) + return active + ? inspectProviderRequirements(provider, values, true) + : { + id: provider.id, + label: provider.label, + active: false, + state: 'absent', + missingFields: [], + invalidFields: [], + invalidDetails: [], + } +} + +function providerProblems(inspection: ProviderInspection): string { + return [ + inspection.missingFields.length > 0 ? `missing ${inspection.missingFields.join(', ')}` : null, + inspection.invalidDetails.length > 0 + ? inspection.invalidDetails.join(', ') + : inspection.invalidFields.length > 0 + ? `invalid ${inspection.invalidFields.join(', ')}` + : null, + ] + .filter(Boolean) + .join('; ') +} + +export function getCapabilityConfigurationError( + definition: CapabilityDefinition, + inspections: readonly ProviderInspection[] +): EnvCapabilityConfigurationError | null { + const broken = inspections.filter( + (inspection) => inspection.state === 'partial' || inspection.state === 'invalid' + ) + if (broken.length === 0) return null + + const details = broken.map((inspection) => `${inspection.label}: ${providerProblems(inspection)}`) + + return new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} is partially or incorrectly configured (${details.join(' | ')}). Run ${getCapabilitySetupCommand(definition)}.` + ) +} + +function replaceProviderInspection( + providers: readonly ProviderInspection[], + replacement: ProviderInspection +): ProviderInspection[] { + return providers.map((provider) => (provider.id === replacement.id ? replacement : provider)) +} + +function inspectSelectedCapability( + definition: TDefinition, + values: EnvCapabilityValues +): SelectedCapabilityInspection, DeclaredProviderId> { + const rawSelector = definition.selectorKey ? readValue(values, definition.selectorKey) : undefined + const selector = + definition.selectorKey && hasValue(values, definition.selectorKey) + ? String(rawSelector).trim().toLowerCase() + : null + let providers = definition.providers.map((provider) => inspectProvider(provider, values)) + const known = new Set([ + definition.defaultProvider.id, + ...definition.providers.map((provider) => provider.id), + ]) + + if (selector && !known.has(selector)) { + const error = new EnvCapabilityConfigurationError( + definition.id, + `Unknown ${definition.selectorKey} "${rawSelector}". Expected one of: ${[...known].join(', ')}` + ) + return { strategy: 'selected', providerId: null, providers, error } + } + + if (selector) { + const selectedDefinition = definition.providers.find((provider) => provider.id === selector) + if (!selectedDefinition) { + return { + strategy: 'selected', + providerId: definition.defaultProvider.id as ProviderId, + providers, + error: null, + } + } + const active = providerIsActive(selectedDefinition, values) + const selected = + !active && selectedDefinition.activation.mode === 'enabled' + ? inspectProvider(selectedDefinition, values) + : inspectProviderRequirements(selectedDefinition, values, active) + providers = replaceProviderInspection(providers, selected) + const error = + selected.state === 'ready' || + (selected.state === 'absent' && selectedDefinition.activation.mode === 'enabled') + ? null + : new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} selects ${selector}, but that provider is not configured (${providerProblems(selected)}). Run ${getCapabilitySetupCommand(definition)}.` + ) + return { + strategy: 'selected', + providerId: selector as ProviderId, + providers, + error, + } + } + + if (definition.whenUnset === 'default') { + const defaultDefinition = definition.providers.find( + (provider) => provider.id === definition.defaultProvider.id + ) + if (!defaultDefinition) { + return { + strategy: 'selected', + providerId: definition.defaultProvider.id as ProviderId, + providers, + error: null, + } + } + const selected = providers.find((provider) => provider.id === definition.defaultProvider.id) + return { + strategy: 'selected', + providerId: definition.defaultProvider.id as ProviderId, + providers, + error: + !selected || selected.state === 'ready' || selected.state === 'absent' + ? null + : new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} selects ${definition.defaultProvider.id}, but that provider is not configured (${providerProblems(selected)}). Run ${getCapabilitySetupCommand(definition)}.` + ), + } + } + + const candidates = providers.filter((provider) => provider.id !== definition.defaultProvider.id) + for (const candidate of candidates) { + if (candidate.state === 'ready') { + return { + strategy: 'selected', + providerId: candidate.id as ProviderId, + providers, + error: null, + } + } + if ( + candidate.state === 'invalid' && + candidate.missingFields.length === 0 && + candidate.invalidFields.length > 0 + ) { + return { + strategy: 'selected', + providerId: candidate.id as ProviderId, + providers, + error: new EnvCapabilityConfigurationError( + definition.id, + `${candidate.label} is incorrectly configured (${providerProblems(candidate)}). Run ${getCapabilitySetupCommand(definition)}.` + ), + } + } + } + + const error = getCapabilityConfigurationError(definition, candidates) + const broken = candidates.find( + (provider) => provider.state === 'partial' || provider.state === 'invalid' + ) + + return { + strategy: 'selected', + providerId: (broken?.id ?? definition.defaultProvider.id) as ProviderId, + providers, + error, + } +} + +function inspectFallbackCapability( + definition: TDefinition, + values: EnvCapabilityValues +): FallbackCapabilityInspection> { + const providers = definition.providers.map((provider) => inspectProvider(provider, values)) + const providerIds = providers + .filter((provider) => provider.state === 'ready') + .map((provider) => provider.id) as DeclaredProviderId[] + const configurationError = getCapabilityConfigurationError(definition, providers) + + return { + strategy: 'fallback', + configured: providerIds.length > 0, + providerIds, + providers, + error: providerIds.length === 0 ? configurationError : null, + } +} + +export function inspectCapability( + definition: TDefinition, + values: EnvCapabilityValues +): SelectedCapabilityInspection, DeclaredProviderId> +export function inspectCapability( + definition: TDefinition, + values: EnvCapabilityValues +): FallbackCapabilityInspection> +export function inspectCapability( + definition: CapabilityDefinition, + values: EnvCapabilityValues +): SelectedCapabilityInspection | FallbackCapabilityInspection +export function inspectCapability( + definition: CapabilityDefinition, + values: EnvCapabilityValues +): SelectedCapabilityInspection | FallbackCapabilityInspection { + return definition.strategy === 'selected' + ? inspectSelectedCapability(definition, values) + : inspectFallbackCapability(definition, values) +} + +export function requireCapability( + definition: TDefinition, + values: EnvCapabilityValues +): Omit< + SelectedCapabilityInspection, DeclaredProviderId>, + 'error' | 'providerId' +> & { + providerId: ProviderId +} +export function requireCapability( + definition: TDefinition, + values: EnvCapabilityValues +): Omit>, 'error'> +export function requireCapability( + definition: CapabilityDefinition, + values: EnvCapabilityValues +): + | (Omit & { + providerId: string + }) + | Omit { + const inspection = + definition.strategy === 'selected' + ? inspectSelectedCapability(definition, values) + : inspectFallbackCapability(definition, values) + if (inspection.error) throw inspection.error + if (inspection.strategy === 'selected') { + const providerId = inspection.providerId + if (providerId === null) { + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} has no selected provider. Run ${getCapabilitySetupCommand(definition)}.` + ) + } + const selected = inspection.providers.find((provider) => provider.id === providerId) + if (selected && selected.state !== 'ready') { + const selectedDefinition = definition.providers.find((provider) => provider.id === providerId) + const strictInspection = + selected.state === 'absent' && selectedDefinition + ? inspectRequiredProvider(selectedDefinition, values) + : selected + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} selects ${providerId}, but that provider is not configured (${providerProblems(strictInspection)}). Run ${getCapabilitySetupCommand(definition)}.` + ) + } + return { + strategy: 'selected', + providerId, + providers: inspection.providers, + } + } + if (!inspection.configured) { + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} is not configured. Run ${getCapabilitySetupCommand(definition)}.` + ) + } + const { error: _, ...resolution } = inspection + return resolution +} + +function findFieldRequirement( + requirement: EnvRequirement, + key: string +): EnvFieldRequirement | null { + if (requirement.type === 'field') return requirement.key === key ? requirement : null + for (const child of requirement.requirements) { + const field = findFieldRequirement(child, key) + if (field) return field + } + return null +} + +/** Validates a candidate environment value with the runtime field rule. */ +export function validateCapabilityFieldInput( + definition: CapabilityDefinition, + key: string, + value: string +): string | undefined { + if (!value) return 'required' + for (const provider of definition.providers) { + const field = + findFieldRequirement(provider.requires, key) ?? + provider.optionalFields?.find((candidate) => candidate.key === key) + if (!field) continue + if (!field.validation || isValidEnvCapabilityFieldValue(field.validation, value)) { + return undefined + } + return field.validation.message + } + throw new Error(`${definition.label} has no validation definition for ${key}`) +} + +export interface WireFallbackOptions { + definition: TDefinition + values: EnvCapabilityValues + factories: FallbackFactories + onFailure?: (providerId: DeclaredProviderId, error: unknown) => void +} + +export function wireFallback({ + definition, + values, + factories, + onFailure, +}: WireFallbackOptions) { + const resolution = inspectCapability(definition, values) + if (resolution.error) throw resolution.error + const providers = resolution.providerIds.map((providerId) => { + const provider = factories[providerId]() + if (!provider) { + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} provider ${providerId} resolved as ready but its factory returned null` + ) + } + return { id: providerId, provider } + }) + + return { + configured: resolution.configured, + providerIds: resolution.providerIds, + providers: providers.map(({ provider }) => provider), + async execute( + operation: ( + provider: TProvider, + providerId: DeclaredProviderId + ) => Promise + ): Promise { + if (resolution.providerIds.length === 0) { + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} is not configured. Run ${getCapabilitySetupCommand(definition)}.` + ) + } + + const failures: unknown[] = [] + for (const { id: providerId, provider } of providers) { + try { + return await operation(provider, providerId) + } catch (error) { + failures.push(error) + onFailure?.(providerId, error) + } + } + + throw new AggregateError( + failures, + `All ${definition.label} providers failed: ${resolution.providerIds.join(', ')}` + ) + }, + } +} + +export const EMAIL_CAPABILITY = defineCapability({ + strategy: 'fallback', + id: 'email', + label: 'Email', + providers: [ + { + id: 'resend', + label: 'Resend', + activation: { mode: 'any-present', keys: ['RESEND_API_KEY'] }, + requires: envField('RESEND_API_KEY'), + }, + { + id: 'ses', + label: 'Amazon SES', + activation: { mode: 'any-present', keys: ['AWS_SES_REGION'] }, + requires: envField('AWS_SES_REGION'), + }, + { + id: 'smtp', + label: 'SMTP', + activation: { + mode: 'any-present', + keys: ['SMTP_HOST', 'SMTP_PORT', 'SMTP_USER', 'SMTP_PASS'], + }, + requires: allOf( + envField('SMTP_HOST'), + envField('SMTP_PORT', { + validation: { + kind: 'integer', + min: 1, + max: 65535, + message: 'must be a valid port between 1 and 65535', + }, + }) + ), + optionalFields: [envField('SMTP_USER'), envField('SMTP_PASS')], + }, + { + id: 'azure', + label: 'Azure Communication Services', + activation: { + mode: 'any-present', + keys: ['AZURE_ACS_CONNECTION_STRING'], + }, + requires: envField('AZURE_ACS_CONNECTION_STRING'), + }, + { + id: 'gmail', + label: 'Gmail', + activation: { + mode: 'any-present', + keys: ['GMAIL_CREDENTIALS_JSON', 'GMAIL_SENDER'], + }, + requires: allOf( + envField('GMAIL_CREDENTIALS_JSON', { + validation: { + kind: 'json-object', + requiredStringFields: ['client_email', 'private_key'], + message: 'must be service account JSON with client_email and private_key', + }, + }), + envField('GMAIL_SENDER') + ), + }, + ], +} as const) + +export const STORAGE_CAPABILITY = defineCapability({ + strategy: 'selected', + id: 'storage', + label: 'File storage', + selectorKey: 'STORAGE_PROVIDER', + whenUnset: 'first-ready', + defaultProvider: { id: 'local', kind: 'built-in', label: 'Local disk' }, + providers: [ + { + id: 'azure', + label: 'Azure Blob Storage', + activation: { + mode: 'any-present', + keys: [ + 'AZURE_CONNECTION_STRING', + 'AZURE_ACCOUNT_NAME', + 'AZURE_ACCOUNT_KEY', + 'AZURE_STORAGE_CONTAINER_NAME', + ], + }, + requires: allOf( + envField('AZURE_STORAGE_CONTAINER_NAME'), + anyOf( + envField('AZURE_CONNECTION_STRING'), + allOf(envField('AZURE_ACCOUNT_NAME'), envField('AZURE_ACCOUNT_KEY')) + ) + ), + }, + { + id: 's3', + label: 'S3', + activation: { + mode: 'any-present', + keys: [ + 'S3_BUCKET_NAME', + 'S3_KB_BUCKET_NAME', + 'S3_EXECUTION_FILES_BUCKET_NAME', + 'S3_CHAT_BUCKET_NAME', + 'S3_COPILOT_BUCKET_NAME', + 'S3_PROFILE_PICTURES_BUCKET_NAME', + 'S3_OG_IMAGES_BUCKET_NAME', + 'S3_WORKSPACE_LOGOS_BUCKET_NAME', + 'S3_ENDPOINT', + ], + }, + requires: allOf(envField('AWS_REGION'), envField('S3_BUCKET_NAME')), + pairedFields: [['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY']], + optionalFields: [ + envField('S3_ENDPOINT', { + validation: { + kind: 'url', + protocols: ['http:', 'https:'], + message: 'must be a valid http:// or https:// URL', + }, + }), + envField('S3_FORCE_PATH_STYLE'), + ], + }, + { + id: 'gcs', + label: 'Google Cloud Storage', + activation: { mode: 'any-present', keys: ['GCS_BUCKET_NAME'] }, + requires: envField('GCS_BUCKET_NAME'), + optionalFields: [ + envField('GCS_CREDENTIALS_JSON', { + validation: { + kind: 'json-object', + requiredStringFields: ['client_email', 'private_key'], + message: 'must be service account JSON with client_email and private_key', + }, + }), + envField('GCS_PROJECT_ID'), + ], + }, + ], +} as const) + +export const SANDBOX_CAPABILITY = defineCapability({ + strategy: 'selected', + id: 'sandbox', + label: 'Remote sandbox', + selectorKey: 'SANDBOX_PROVIDER', + whenUnset: 'default', + defaultProvider: { id: 'e2b', kind: 'provider' }, + providers: [ + { + id: 'e2b', + label: 'E2B', + activation: { mode: 'enabled', key: 'E2B_ENABLED' }, + requires: envField('E2B_API_KEY'), + optionalFields: [ + envField('NEXT_PUBLIC_E2B_ENABLED'), + envField('NEXT_PUBLIC_SANDBOX_ENABLED'), + ], + }, + { + id: 'daytona', + label: 'Daytona', + activation: { + mode: 'any-present', + keys: ['DAYTONA_API_KEY', 'DAYTONA_SHELL_SNAPSHOT_ID'], + }, + requires: allOf( + envField('DAYTONA_API_KEY'), + envField('DAYTONA_SHELL_SNAPSHOT_ID', { + validation: { + kind: 'pattern', + pattern: /^(?!.*:(?:latest|lts|stable)$)[^:\s]+:[^:\s]+$/i, + message: 'must use an explicit, non-floating name:tag', + }, + }) + ), + optionalFields: [ + envField('NEXT_PUBLIC_E2B_ENABLED'), + envField('NEXT_PUBLIC_SANDBOX_ENABLED'), + ], + }, + ], +} as const) + +export const ASYNC_JOBS_CAPABILITY = defineCapability({ + strategy: 'selected', + id: 'jobs', + label: 'Async jobs', + whenUnset: 'first-ready', + defaultProvider: { + id: 'database', + kind: 'built-in', + label: 'Database queue', + }, + providers: [ + { + id: 'trigger-dev', + label: 'Trigger.dev', + activation: { mode: 'enabled', key: 'TRIGGER_DEV_ENABLED' }, + requires: allOf(envField('TRIGGER_PROJECT_ID'), envField('TRIGGER_SECRET_KEY')), + }, + ], +} as const) + +/** Validates the one provider dependency that cannot be expressed as a field-shape rule. */ +function validateRedisProvider(values: EnvCapabilityValues): readonly EnvProviderValidationIssue[] { + if (!hasValue(values, 'REDIS_URL')) return [] + let redisUrl: URL + try { + redisUrl = new URL(String(readValue(values, 'REDIS_URL'))) + } catch { + return [] + } + if ( + redisUrl.protocol === 'rediss:' && + /^\d+\.\d+\.\d+\.\d+$/.test(redisUrl.hostname) && + !hasValue(values, 'REDIS_TLS_SERVERNAME') + ) { + return [ + { + kind: 'missing', + fields: ['REDIS_TLS_SERVERNAME'], + message: 'REDIS_TLS_SERVERNAME is required for rediss:// IP addresses', + }, + ] + } + return [] +} + +export const CACHE_CAPABILITY = defineCapability({ + strategy: 'selected', + id: 'cache', + label: 'Cache', + whenUnset: 'first-ready', + defaultProvider: { id: 'database', kind: 'built-in', label: 'Postgres' }, + providers: [ + { + id: 'redis', + label: 'Redis', + activation: { mode: 'any-present', keys: ['REDIS_URL'] }, + requires: envField('REDIS_URL', { + validation: { + kind: 'url', + protocols: ['redis:', 'rediss:'], + message: 'must be a valid redis:// or rediss:// URL', + }, + }), + optionalFields: [envField('REDIS_TLS_SERVERNAME')], + validate: validateRedisProvider, + }, + ], +} as const) + +export const OCR_CAPABILITY = defineCapability({ + strategy: 'selected', + id: 'knowledge', + label: 'PDF OCR', + selectorKey: 'OCR_PROVIDER', + whenUnset: 'first-ready', + defaultProvider: { id: 'local', kind: 'built-in', label: 'Local parser' }, + providers: [ + { + id: 'azure-mistral', + label: 'Azure Mistral OCR', + activation: { + mode: 'any-present', + keys: ['OCR_AZURE_API_KEY', 'OCR_AZURE_ENDPOINT', 'OCR_AZURE_MODEL_NAME'], + }, + requires: allOf( + envField('OCR_AZURE_API_KEY'), + envField('OCR_AZURE_ENDPOINT', { + validation: { + kind: 'url', + protocols: ['http:', 'https:'], + message: 'must be a valid HTTP(S) URL', + }, + }), + envField('OCR_AZURE_MODEL_NAME') + ), + }, + { + id: 'mistral', + label: 'Mistral OCR', + activation: { mode: 'any-present', keys: ['MISTRAL_API_KEY'] }, + requires: envField('MISTRAL_API_KEY'), + }, + ], +} as const) + +export const OAUTH_CLIENT_CAPABILITIES = { + google: ['GOOGLE_CLIENT_ID', 'GOOGLE_CLIENT_SECRET'], + x: ['X_CLIENT_ID', 'X_CLIENT_SECRET'], + tiktok: ['TIKTOK_CLIENT_ID', 'TIKTOK_CLIENT_SECRET'], + confluence: ['CONFLUENCE_CLIENT_ID', 'CONFLUENCE_CLIENT_SECRET'], + jira: ['JIRA_CLIENT_ID', 'JIRA_CLIENT_SECRET'], + calcom: ['CALCOM_CLIENT_ID'], + airtable: ['AIRTABLE_CLIENT_ID', 'AIRTABLE_CLIENT_SECRET'], + notion: ['NOTION_CLIENT_ID', 'NOTION_CLIENT_SECRET'], + microsoft: ['MICROSOFT_CLIENT_ID', 'MICROSOFT_CLIENT_SECRET'], + clickup: ['CLICKUP_CLIENT_ID', 'CLICKUP_CLIENT_SECRET'], + linear: ['LINEAR_CLIENT_ID', 'LINEAR_CLIENT_SECRET'], + attio: ['ATTIO_CLIENT_ID', 'ATTIO_CLIENT_SECRET'], + box: ['BOX_CLIENT_ID', 'BOX_CLIENT_SECRET'], + docusign: ['DOCUSIGN_CLIENT_ID', 'DOCUSIGN_CLIENT_SECRET'], + dropbox: ['DROPBOX_CLIENT_ID', 'DROPBOX_CLIENT_SECRET'], + slack: ['SLACK_CLIENT_ID', 'SLACK_CLIENT_SECRET'], + reddit: ['REDDIT_CLIENT_ID', 'REDDIT_CLIENT_SECRET'], + wealthbox: ['WEALTHBOX_CLIENT_ID', 'WEALTHBOX_CLIENT_SECRET'], + webflow: ['WEBFLOW_CLIENT_ID', 'WEBFLOW_CLIENT_SECRET'], + asana: ['ASANA_CLIENT_ID', 'ASANA_CLIENT_SECRET'], + pipedrive: ['PIPEDRIVE_CLIENT_ID', 'PIPEDRIVE_CLIENT_SECRET'], + hubspot: ['HUBSPOT_CLIENT_ID', 'HUBSPOT_CLIENT_SECRET'], + linkedin: ['LINKEDIN_CLIENT_ID', 'LINKEDIN_CLIENT_SECRET'], + instagram: ['INSTAGRAM_CLIENT_ID', 'INSTAGRAM_CLIENT_SECRET'], + salesforce: ['SALESFORCE_CLIENT_ID', 'SALESFORCE_CLIENT_SECRET'], + shopify: ['SHOPIFY_CLIENT_ID', 'SHOPIFY_CLIENT_SECRET'], + zoom: ['ZOOM_CLIENT_ID', 'ZOOM_CLIENT_SECRET'], + wordpress: ['WORDPRESS_CLIENT_ID', 'WORDPRESS_CLIENT_SECRET'], + spotify: ['SPOTIFY_CLIENT_ID', 'SPOTIFY_CLIENT_SECRET'], + monday: ['MONDAY_CLIENT_ID', 'MONDAY_CLIENT_SECRET'], + trello: ['TRELLO_API_KEY'], + 'zoho-desk': ['ZOHO_CLIENT_ID', 'ZOHO_CLIENT_SECRET'], +} as const + +/** Single registry consumed by runtime status and environment-source detection. */ +export const ENV_CAPABILITIES = [ + EMAIL_CAPABILITY, + STORAGE_CAPABILITY, + SANDBOX_CAPABILITY, + ASYNC_JOBS_CAPABILITY, + CACHE_CAPABILITY, + OCR_CAPABILITY, +] as const + +export const LLM_KEY_POOLS = { + openai: { + keys: ['OPENAI_API_KEY_1', 'OPENAI_API_KEY_2', 'OPENAI_API_KEY_3'], + fallbackKey: 'OPENAI_API_KEY', + }, + anthropic: { + keys: ['ANTHROPIC_API_KEY_1', 'ANTHROPIC_API_KEY_2', 'ANTHROPIC_API_KEY_3'], + }, + gemini: { + keys: ['GEMINI_API_KEY_1', 'GEMINI_API_KEY_2', 'GEMINI_API_KEY_3'], + fallbackKey: 'GEMINI_API_KEY', + }, + cohere: { + keys: ['COHERE_API_KEY_1', 'COHERE_API_KEY_2', 'COHERE_API_KEY_3'], + fallbackKey: 'COHERE_API_KEY', + }, + zai: { keys: ['ZAI_API_KEY_1', 'ZAI_API_KEY_2', 'ZAI_API_KEY_3'] }, + xai: { keys: ['XAI_API_KEY_1', 'XAI_API_KEY_2', 'XAI_API_KEY_3'] }, + kimi: { keys: ['KIMI_API_KEY_1', 'KIMI_API_KEY_2', 'KIMI_API_KEY_3'] }, + fireworks: { + keys: ['FIREWORKS_API_KEY_1', 'FIREWORKS_API_KEY_2', 'FIREWORKS_API_KEY_3'], + fallbackKey: 'FIREWORKS_API_KEY', + }, +} as const + +/** + * Environment keys whose process-level values can change setup status or make a + * setup write ineffective. The setup CLI uses this exact runtime-owned list to + * avoid claiming it manages a development configuration shadowed by the shell. + */ +export const DEPLOYMENT_CONFIGURATION_KEYS: readonly string[] = [ + ...new Set([ + ...CORE_CONFIGURATION_KEYS, + ...ENV_CAPABILITIES.flatMap(capabilityKeys), + 'EMAIL_VERIFICATION_ENABLED', + 'NEXT_PUBLIC_E2B_ENABLED', + 'NEXT_PUBLIC_SANDBOX_ENABLED', + ...Object.values(LLM_KEY_POOLS).flatMap((pool) => [ + ...pool.keys, + ...('fallbackKey' in pool ? [pool.fallbackKey] : []), + ]), + ...Object.values(OAUTH_CLIENT_CAPABILITIES).flat(), + ]), +] + +export type OAuthClientCapabilityId = keyof typeof OAUTH_CLIENT_CAPABILITIES +export type OAuthClientCapabilityField = + (typeof OAUTH_CLIENT_CAPABILITIES)[TCapabilityId][number] + +export interface ConfiguredOAuthClient { + state: 'ready' + missingFields: readonly [] + setupCommand: string + values: Readonly> +} + +const GOOGLE_OAUTH_SERVICES = new Set([ + 'gmail', + 'google-email', + 'google-drive', + 'google-docs', + 'google-sheets', + 'google-calendar', + 'google-contacts', + 'google-ads', + 'google-bigquery', + 'google-tasks', + 'google-vault', + 'google-forms', + 'google-groups', + 'google-meet', + 'vertex-ai', +]) + +const MICROSOFT_OAUTH_SERVICES = new Set([ + 'microsoft', + 'outlook', + 'onedrive', + 'sharepoint', + 'microsoft-ad', + 'microsoft-dataverse', + 'microsoft-excel', + 'microsoft-teams', + 'microsoft-planner', +]) + +export function resolveOAuthClientCapabilityId(serviceId: string): OAuthClientCapabilityId | null { + const normalized = serviceId.toLowerCase().replace(/_/g, '-') + if (GOOGLE_OAUTH_SERVICES.has(normalized)) return 'google' + if (MICROSOFT_OAUTH_SERVICES.has(normalized)) return 'microsoft' + if (normalized === 'zoho') return 'zoho-desk' + return normalized in OAUTH_CLIENT_CAPABILITIES ? (normalized as OAuthClientCapabilityId) : null +} + +export function getOAuthClientCapabilityFields(serviceId: string): readonly string[] | null { + const providerId = resolveOAuthClientCapabilityId(serviceId) + return providerId ? OAUTH_CLIENT_CAPABILITIES[providerId] : null +} + +export interface OAuthClientCapabilityInspection { + state: ProviderConfigurationState + missingFields: readonly string[] + setupCommand: string +} + +function readOAuthClientFieldValue(values: EnvCapabilityValues, key: string): string | null { + const value = readValue(values, key) + if (typeof value !== 'string' || !hasValue(values, key)) return null + return value +} + +export function inspectOAuthClientCapability( + providerId: string, + values: EnvCapabilityValues +): OAuthClientCapabilityInspection { + const capabilityId = resolveOAuthClientCapabilityId(providerId) + const fields = capabilityId ? OAUTH_CLIENT_CAPABILITIES[capabilityId] : null + if (!fields) { + return { + state: 'absent', + missingFields: [], + setupCommand: `bun run setup integration ${providerId}`, + } + } + + const present = fields.filter((key) => readOAuthClientFieldValue(values, key) !== null) + return { + state: present.length === 0 ? 'absent' : present.length === fields.length ? 'ready' : 'partial', + missingFields: fields.filter((key) => readOAuthClientFieldValue(values, key) === null), + setupCommand: `bun run setup integration ${providerId}`, + } +} + +export function requireOAuthClientCapability( + providerId: TCapabilityId, + values: EnvCapabilityValues +): ConfiguredOAuthClient> +export function requireOAuthClientCapability( + providerId: string, + values: EnvCapabilityValues +): ConfiguredOAuthClient +export function requireOAuthClientCapability( + providerId: string, + values: EnvCapabilityValues +): ConfiguredOAuthClient { + const inspection = inspectOAuthClientCapability(providerId, values) + if (inspection.state !== 'ready') { + const detail = + inspection.state === 'partial' || inspection.state === 'invalid' + ? ` is partially configured — missing ${inspection.missingFields.join(', ')}` + : ' is not configured' + throw new EnvCapabilityConfigurationError( + 'oauth', + `OAuth client ${providerId}${detail}. Run ${inspection.setupCommand}.` + ) + } + + const fields = getOAuthClientCapabilityFields(providerId) + if (!fields) { + throw new EnvCapabilityConfigurationError( + 'oauth', + `OAuth client ${providerId} has no capability definition. Run ${inspection.setupCommand}.` + ) + } + + const configuredValues: Record = {} + for (const field of fields) { + const value = readOAuthClientFieldValue(values, field) + if (value === null) { + throw new EnvCapabilityConfigurationError( + 'oauth', + `OAuth client ${providerId} has an invalid ${field}. Run ${inspection.setupCommand}.` + ) + } + configuredValues[field] = value + } + + return { + ...inspection, + state: 'ready', + missingFields: [], + values: configuredValues, + } +} diff --git a/apps/sim/lib/core/config/env-flags.ts b/apps/sim/lib/core/config/env-flags.ts index 93d12e032b7..cb83d3bea2c 100644 --- a/apps/sim/lib/core/config/env-flags.ts +++ b/apps/sim/lib/core/config/env-flags.ts @@ -1,12 +1,15 @@ /** - * Environment utility functions for consistent environment detection across the application + * Loaded by `next.config.ts` before the `@/` alias is available, so + * config-boundary dependencies in this module must use relative imports. */ + import { ENTERPRISE_FEATURE_LEGACY_DEFAULTS, type EnterpriseFeature, resolveEnterpriseEntitlement, } from './enterprise-entitlements' import { env, envBoolean, getEnv, isFalsy, isTruthy } from './env' +import { hasEnvCapabilityValue, inspectCapability, SANDBOX_CAPABILITY } from './env-capabilities' /** * Is the application running in production mode @@ -390,15 +393,14 @@ export const isForkingEnabled = enterpriseFeatureEnabled( * Availability below is derived from THIS provider's credentials, so a * Daytona-only deployment (E2B unset) still enables remote execution. */ -const sandboxProvider = (env.SANDBOX_PROVIDER || 'e2b').toLowerCase() +const sandboxProvider = inspectCapability(SANDBOX_CAPABILITY, env).providerId /** * Whether remote code/shell execution is available with the selected provider. * * E2B keeps its explicit `E2B_ENABLED` switch; Daytona is available once its API - * key is set (the shell snapshot is verified at create time, failing closed). - * Mirrors the E2B gate exactly when the provider is E2B, so existing behavior is - * unchanged. + * key is set. Strict credential and snapshot validation runs when the selected + * remote backend is used, so unrelated app paths preserve legacy enablement. * * The browser twin is `NEXT_PUBLIC_SANDBOX_ENABLED`, read by the Function * block's `showWhenEnvSet` gates. It exists because `NEXT_PUBLIC_E2B_ENABLED` @@ -408,7 +410,11 @@ const sandboxProvider = (env.SANDBOX_PROVIDER || 'e2b').toLowerCase() * `bun run setup --doctor` flags the mismatch. */ export const isRemoteSandboxEnabled = - sandboxProvider === 'daytona' ? Boolean(env.DAYTONA_API_KEY) : isTruthy(env.E2B_ENABLED) + sandboxProvider === 'daytona' + ? hasEnvCapabilityValue(env, 'DAYTONA_API_KEY') + : sandboxProvider === 'e2b' + ? isTruthy(env.E2B_ENABLED) + : false /** * Whether the document-generation sandbox is available with the selected @@ -424,10 +430,13 @@ export const isRemoteSandboxEnabled = */ export const isDocSandboxEnabled = sandboxProvider === 'daytona' - ? Boolean(env.DAYTONA_API_KEY) && Boolean(env.DAYTONA_DOC_SNAPSHOT_ID) - : isTruthy(env.E2B_ENABLED) && - Boolean(env.E2B_API_KEY) && - Boolean(env.MOTHERSHIP_E2B_DOC_TEMPLATE_ID) + ? hasEnvCapabilityValue(env, 'DAYTONA_API_KEY') && + hasEnvCapabilityValue(env, 'DAYTONA_DOC_SNAPSHOT_ID') + : sandboxProvider === 'e2b' + ? isTruthy(env.E2B_ENABLED) && + hasEnvCapabilityValue(env, 'E2B_API_KEY') && + hasEnvCapabilityValue(env, 'MOTHERSHIP_E2B_DOC_TEMPLATE_ID') + : false /** * Whether Ollama is configured (OLLAMA_URL is set). diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 290e97d6617..0b034d4cca3 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -70,6 +70,10 @@ export const env = createEnv({ // Database & Storage REDIS_URL: z.string().url().optional(), // Redis connection string for caching/sessions REDIS_TLS_SERVERNAME: z.string().min(1).optional(), // TLS SNI override; required when REDIS_URL targets an IP over rediss:// (e.g. trigger.dev PrivateLink VPCE IP) so cert hostname verification matches the ElastiCache cert's CN + /** Explicit file-storage backend; unset preserves Azure → S3 → GCS → local precedence. */ + STORAGE_PROVIDER: z.enum(['local', 's3', 'azure', 'gcs']).optional(), + /** Explicit PDF OCR backend; legacy installs infer it from configured credentials. */ + OCR_PROVIDER: z.enum(['local', 'mistral', 'azure-mistral']).optional(), // Payment & Billing STRIPE_SECRET_KEY: z.string().min(1).optional(), // Stripe secret key for payment processing diff --git a/apps/sim/lib/core/config/redis.test.ts b/apps/sim/lib/core/config/redis.test.ts index 82a834fc93a..69662e173b0 100644 --- a/apps/sim/lib/core/config/redis.test.ts +++ b/apps/sim/lib/core/config/redis.test.ts @@ -1,7 +1,11 @@ -import { createEnvMock, createMockRedis } from '@sim/testing' +import { createMockRedis } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { MockRedisConstructor } = vi.hoisted(() => ({ +const { mockEnv, MockRedisConstructor } = vi.hoisted(() => ({ + mockEnv: { + REDIS_URL: 'redis://localhost:6379' as string | undefined, + REDIS_TLS_SERVERNAME: undefined as string | undefined, + }, MockRedisConstructor: vi.fn(), })) @@ -15,7 +19,7 @@ MockRedisConstructor.mockImplementation( ) vi.unmock('@/lib/core/config/redis') -vi.mock('@/lib/core/config/env', () => createEnvMock({ REDIS_URL: 'redis://localhost:6379' })) +vi.mock('@/lib/core/config/env', () => ({ env: mockEnv })) vi.mock('ioredis', () => ({ default: MockRedisConstructor, })) @@ -33,6 +37,8 @@ describe('redis config', () => { vi.clearAllMocks() vi.useFakeTimers() resetForTesting() + mockEnv.REDIS_URL = 'redis://localhost:6379' + mockEnv.REDIS_TLS_SERVERNAME = undefined MockRedisConstructor.mockImplementation( class { constructor() { @@ -193,17 +199,40 @@ describe('redis config', () => { expect(extended).toBe(false) }) - it('returns true as a no-op when Redis is unavailable', async () => { - vi.resetModules() - vi.doMock('@/lib/core/config/env', () => - createEnvMock({ REDIS_URL: undefined as unknown as string }) - ) - const { extendLock: extendLockNoRedis } = await import('@/lib/core/config/redis') + it('returns true as a no-op when the cache capability selects the database', async () => { + mockEnv.REDIS_URL = undefined - const extended = await extendLockNoRedis(lockKey, value, ttlSeconds) + const extended = await extendLock(lockKey, value, ttlSeconds) expect(extended).toBe(true) - vi.doUnmock('@/lib/core/config/env') + }) + }) + + describe('capability validation', () => { + it('rejects a non-Redis URL before constructing a client', () => { + mockEnv.REDIS_URL = 'https://cache.example.com' + + expect(() => getRedisClient()).toThrow(/valid redis:\/\/ or rediss:\/\/ URL/) + expect(MockRedisConstructor).not.toHaveBeenCalled() + }) + + it('requires TLS servername for a rediss IP before constructing a client', () => { + mockEnv.REDIS_URL = 'rediss://10.0.0.1:6379' + + expect(() => getRedisClient()).toThrow(/REDIS_TLS_SERVERNAME is required/) + expect(MockRedisConstructor).not.toHaveBeenCalled() + }) + + it('passes the configured TLS servername to Redis', () => { + mockEnv.REDIS_URL = 'rediss://10.0.0.1:6379' + mockEnv.REDIS_TLS_SERVERNAME = 'cache.example.com' + + getRedisClient() + + expect(MockRedisConstructor).toHaveBeenCalledWith( + mockEnv.REDIS_URL, + expect.objectContaining({ tls: { servername: 'cache.example.com' } }) + ) }) }) diff --git a/apps/sim/lib/core/config/redis.ts b/apps/sim/lib/core/config/redis.ts index 820c0215c1e..d5bed2d4954 100644 --- a/apps/sim/lib/core/config/redis.ts +++ b/apps/sim/lib/core/config/redis.ts @@ -3,11 +3,10 @@ import { toError } from '@sim/utils/errors' import { randomFloat } from '@sim/utils/random' import Redis, { type RedisOptions } from 'ioredis' import { env } from '@/lib/core/config/env' +import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server' const logger = createLogger('Redis') -const redisUrl = env.REDIS_URL - /** * When REDIS_URL targets a bare IP over `rediss://` (e.g. trigger.dev's * PrivateLink VPCE IP), default TLS hostname verification fails — the cert @@ -77,6 +76,16 @@ const state = g._redisState const PING_INTERVAL_MS = 15_000 const MAX_PING_FAILURES = 2 +export function getConfiguredRedisUrl(): string | null { + if (getConfiguredCacheProvider() === 'database') return null + + const redisUrl = env.REDIS_URL + if (!redisUrl) { + throw new Error('Cache capability selected Redis but REDIS_URL is missing') + } + return redisUrl +} + /** * Register a callback that fires when the PING health check forces a reconnect. * Useful for resetting cached adapters that hold a stale Redis reference. @@ -140,6 +149,7 @@ function startPingHealthCheck(redis: Redis): void { */ export function getRedisClient(): Redis | null { if (typeof window !== 'undefined') return null + const redisUrl = getConfiguredRedisUrl() if (!redisUrl) return null if (state.client) return state.client diff --git a/apps/sim/lib/core/idempotency/service.ts b/apps/sim/lib/core/idempotency/service.ts index c44a1e6a442..d74121c2c56 100644 --- a/apps/sim/lib/core/idempotency/service.ts +++ b/apps/sim/lib/core/idempotency/service.ts @@ -81,8 +81,7 @@ const POLL_INTERVAL_MS = 1000 * * Storage is determined once based on configuration: * - If `forceStorage` is set → that backend unconditionally - * - Else if `REDIS_URL` is set → Redis - * - Else → PostgreSQL + * - Else use the provider selected by the cache capability */ export class IdempotencyService { private config: Required> diff --git a/apps/sim/lib/core/storage/storage.ts b/apps/sim/lib/core/storage/storage.ts index 7896ae30ac6..d05cde2a13a 100644 --- a/apps/sim/lib/core/storage/storage.ts +++ b/apps/sim/lib/core/storage/storage.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server' import { getRedisClient } from '@/lib/core/config/redis' const logger = createLogger('Storage') @@ -11,8 +12,8 @@ let cachedStorageMethod: StorageMethod | null = null * Determine the storage method once based on configuration. * This decision is made at first call and cached for the lifetime of the process. * - * - If REDIS_URL is configured and client initializes → 'redis' - * - If REDIS_URL is not configured → 'database' + * - If the cache capability selects Redis and the client initializes → 'redis' + * - If the cache capability selects the built-in provider → 'database' * * Transient failures do NOT change the storage method. * If Redis is configured but fails, operations will fail (not fallback to DB). @@ -22,9 +23,9 @@ export function getStorageMethod(): StorageMethod { return cachedStorageMethod } - const redis = getRedisClient() - - if (redis) { + if (getConfiguredCacheProvider() === 'redis') { + const redis = getRedisClient() + if (!redis) throw new Error('REDIS_URL is configured but the Redis client is unavailable') cachedStorageMethod = 'redis' logger.info('Storage method: Redis') } else { diff --git a/apps/sim/lib/events/pubsub.ts b/apps/sim/lib/events/pubsub.ts index e8a36b5522e..dd372ab488c 100644 --- a/apps/sim/lib/events/pubsub.ts +++ b/apps/sim/lib/events/pubsub.ts @@ -9,8 +9,7 @@ import { EventEmitter } from 'events' import { createLogger } from '@sim/logger' import { noop } from '@sim/utils/helpers' import Redis, { type RedisOptions } from 'ioredis' -import { env } from '@/lib/core/config/env' -import { getRedisConnectionDefaults } from '@/lib/core/config/redis' +import { getConfiguredRedisUrl, getRedisConnectionDefaults } from '@/lib/core/config/redis' const logger = createLogger('PubSub') @@ -138,7 +137,7 @@ class LocalPubSubChannel implements PubSubChannel { } export function createPubSubChannel(config: PubSubChannelConfig): PubSubChannel { - const redisUrl = env.REDIS_URL + const redisUrl = getConfiguredRedisUrl() if (!redisUrl) return new LocalPubSubChannel(config) // Resolve config-derived defaults outside the try so a missing @@ -146,11 +145,6 @@ export function createPubSubChannel(config: PubSubChannelConfig): PubSubChann // to the in-process EventEmitter — that would break cross-replica pub/sub. const connectionDefaults = getRedisConnectionDefaults(redisUrl) - try { - logger.info(`${config.label}: Using Redis`) - return new RedisPubSubChannel(redisUrl, connectionDefaults, config) - } catch (err) { - logger.error(`Failed to create Redis ${config.label}, falling back to local:`, err) - return new LocalPubSubChannel(config) - } + logger.info(`${config.label}: Using Redis`) + return new RedisPubSubChannel(redisUrl, connectionDefaults, config) } diff --git a/apps/sim/lib/execution/event-buffer.test.ts b/apps/sim/lib/execution/event-buffer.test.ts index 041fda1c85e..718d1c4e214 100644 --- a/apps/sim/lib/execution/event-buffer.test.ts +++ b/apps/sim/lib/execution/event-buffer.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { redisConfigMockFns, resetRedisConfigMock } from '@sim/testing' +import { redisConfigMockFns, resetEnvMock, resetRedisConfigMock, setEnv } from '@sim/testing' import { sleep } from '@sim/utils/helpers' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' import type { ExecutionEventEntry } from '@/lib/execution/event-buffer' @@ -26,7 +26,10 @@ const { mockRedis, persistedEntries } = vi.hoisted(() => { const mockGetRedisClient = redisConfigMockFns.mockGetRedisClient -afterAll(resetRedisConfigMock) +afterAll(() => { + resetEnvMock() + resetRedisConfigMock() +}) import { createExecutionEventWriter, @@ -77,6 +80,7 @@ function countOccurrences(haystack: string, needle: string): number { describe('execution event buffer', () => { beforeEach(() => { vi.clearAllMocks() + setEnv({ REDIS_URL: 'redis://localhost:6379' }) persistedEntries.length = 0 mockGetRedisClient.mockReturnValue(mockRedis) mockRedis.get.mockResolvedValue(null) diff --git a/apps/sim/lib/execution/event-buffer.ts b/apps/sim/lib/execution/event-buffer.ts index ebc7d2ec628..3e75a7263c8 100644 --- a/apps/sim/lib/execution/event-buffer.ts +++ b/apps/sim/lib/execution/event-buffer.ts @@ -1,7 +1,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { randomInt } from '@sim/utils/random' -import { env } from '@/lib/core/config/env' +import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server' import { getRedisClient } from '@/lib/core/config/redis' import { LARGE_VALUE_THRESHOLD_BYTES } from '@/lib/execution/payloads/large-value-ref' import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' @@ -364,7 +364,7 @@ async function compactEventForBuffer( const memoryExecutionStreams = new Map() function canUseMemoryEventBuffer(): boolean { - return typeof window === 'undefined' && !env.REDIS_URL + return typeof window === 'undefined' && getConfiguredCacheProvider() === 'database' } function pruneExpiredMemoryStreams(now = Date.now()): void { diff --git a/apps/sim/lib/execution/isolated-vm.ts b/apps/sim/lib/execution/isolated-vm.ts index febb140ba81..22917a05835 100644 --- a/apps/sim/lib/execution/isolated-vm.ts +++ b/apps/sim/lib/execution/isolated-vm.ts @@ -7,6 +7,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { filterUndefined } from '@sim/utils/object' import { randomFloat } from '@sim/utils/random' import { env } from '@/lib/core/config/env' +import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server' import { getRedisClient } from '@/lib/core/config/redis' import { type SecureFetchOptions, @@ -350,8 +351,7 @@ async function tryAcquireDistributedLease( leaseId: string, timeoutMs: number ): Promise { - // Redis not configured: explicit local-mode fallback is allowed. - if (!env.REDIS_URL) return 'acquired' + if (getConfiguredCacheProvider() === 'database') return 'acquired' const redis = getRedisClient() if (!redis) { diff --git a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts index 1fbb85b74be..63dabf80bbe 100644 --- a/apps/sim/lib/execution/remote-sandbox/conformance.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/conformance.test.ts @@ -37,6 +37,7 @@ const { mockEnv: { SANDBOX_PROVIDER: 'e2b' as string | undefined, PI_SANDBOX_LIFETIME_MS: undefined as string | undefined, + E2B_ENABLED: 'true', E2B_API_KEY: 'test-key', MOTHERSHIP_E2B_TEMPLATE_ID: 'mothership-shell', MOTHERSHIP_E2B_DOC_TEMPLATE_ID: 'mothership-docs', diff --git a/apps/sim/lib/execution/remote-sandbox/pi-lifetime.test.ts b/apps/sim/lib/execution/remote-sandbox/pi-lifetime.test.ts index bc155e3361d..1bfee0c4c1d 100644 --- a/apps/sim/lib/execution/remote-sandbox/pi-lifetime.test.ts +++ b/apps/sim/lib/execution/remote-sandbox/pi-lifetime.test.ts @@ -3,99 +3,108 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -/** - * The resolver reads configuration at import, so each case re-imports the module - * with its own mocked environment rather than mutating shared state. - */ -async function resolveWith(options: { - provider?: string - lifetimeMs?: string -}): Promise<{ lifetime: number | undefined; min: number; max: number }> { - vi.resetModules() - vi.doMock('@/lib/core/config/env', () => ({ - env: { - PI_SANDBOX_LIFETIME_MS: options.lifetimeMs, - SANDBOX_PROVIDER: options.provider, - }, - })) - - const mod = await import('@/lib/execution/remote-sandbox/pi-lifetime') +const { mockEnv } = vi.hoisted(() => ({ + mockEnv: { + PI_SANDBOX_LIFETIME_MS: undefined as string | undefined, + SANDBOX_PROVIDER: undefined as string | undefined, + }, +})) + +vi.mock('@/lib/core/config/env', () => ({ env: mockEnv })) + +import { createTimeoutAbortController } from '@/lib/core/execution-limits' +import { + PI_SANDBOX_MAX_LIFETIME_MS, + PI_SANDBOX_MIN_LIFETIME_MS, + resolvePiRunLifetimeMs, + resolvePiSandboxLifetimeMs, +} from '@/lib/execution/remote-sandbox/pi-lifetime' + +function resolveWith(options: { provider?: string; lifetimeMs?: string }): { + lifetime: number | undefined + min: number + max: number +} { + mockEnv.PI_SANDBOX_LIFETIME_MS = options.lifetimeMs + mockEnv.SANDBOX_PROVIDER = options.provider return { - lifetime: mod.resolvePiSandboxLifetimeMs(), - min: mod.PI_SANDBOX_MIN_LIFETIME_MS, - max: mod.PI_SANDBOX_MAX_LIFETIME_MS, + lifetime: resolvePiSandboxLifetimeMs(), + min: PI_SANDBOX_MIN_LIFETIME_MS, + max: PI_SANDBOX_MAX_LIFETIME_MS, } } beforeEach(() => { - vi.resetModules() + mockEnv.PI_SANDBOX_LIFETIME_MS = undefined + mockEnv.SANDBOX_PROVIDER = undefined }) describe('resolvePiSandboxLifetimeMs', () => { - it('defaults to the sub-hour cap on E2B', async () => { - const { lifetime, max } = await resolveWith({}) + it('defaults to the sub-hour cap on E2B', () => { + const { lifetime, max } = resolveWith({}) expect(lifetime).toBe(max) }) - it('matches provider selection by treating an empty provider as E2B', async () => { - const { lifetime, max } = await resolveWith({ provider: '' }) + it('matches provider selection by treating an empty provider as E2B', () => { + const { lifetime, max } = resolveWith({ provider: '' }) expect(lifetime).toBe(max) }) - it('has no lifetime to report when the provider stops on inactivity', async () => { + it('has no lifetime to report when the provider stops on inactivity', () => { // Daytona has no absolute lifetime, so reporting E2B's would cut the agent // turn to fit a ceiling that does not apply — the regression this prevents. - const { lifetime } = await resolveWith({ provider: 'daytona' }) + const { lifetime } = resolveWith({ provider: 'daytona' }) + + expect(lifetime).toBeUndefined() + }) + + it('ignores a configured lifetime entirely on that provider', () => { + const { lifetime } = resolveWith({ provider: 'daytona', lifetimeMs: '600000' }) expect(lifetime).toBeUndefined() }) - it('ignores a configured lifetime entirely on that provider', async () => { - const { lifetime } = await resolveWith({ provider: 'daytona', lifetimeMs: '600000' }) + it('uses tolerant capability inspection for an unknown provider', () => { + const { lifetime } = resolveWith({ provider: 'modal' }) expect(lifetime).toBeUndefined() }) - it('lets a configured value lower the lifetime', async () => { - const { lifetime, min, max } = await resolveWith({ lifetimeMs: String(45 * 60 * 1000) }) + it('lets a configured value lower the lifetime', () => { + const { lifetime, min, max } = resolveWith({ lifetimeMs: String(45 * 60 * 1000) }) expect(lifetime).toBe(45 * 60 * 1000) expect(lifetime!).toBeGreaterThan(min) expect(lifetime!).toBeLessThan(max) }) - it('refuses to be raised above the cap', async () => { + it('refuses to be raised above the cap', () => { // A Hobby key rejects a create above one hour, so an over-large override // would otherwise fail every Pi run rather than lengthening one. - const { lifetime, max } = await resolveWith({ lifetimeMs: String(6 * 60 * 60 * 1000) }) + const { lifetime, max } = resolveWith({ lifetimeMs: String(6 * 60 * 60 * 1000) }) expect(lifetime).toBe(max) }) - it('raises a lifetime too short for a run to finish in', async () => { + it('raises a lifetime too short for a run to finish in', () => { // Ten minutes is consumed by the clone reserve alone, leaving the turn and // the push to race a sandbox that may already be reaped. - const { lifetime, min } = await resolveWith({ lifetimeMs: String(10 * 60 * 1000) }) + const { lifetime, min } = resolveWith({ lifetimeMs: String(10 * 60 * 1000) }) expect(lifetime).toBe(min) }) - it.each(['', 'soon', '0', '-1'])('falls back to the cap for %o', async (value) => { - const { lifetime, max } = await resolveWith({ lifetimeMs: value }) + it.each(['', 'soon', '0', '-1'])('falls back to the cap for %o', (value) => { + const { lifetime, max } = resolveWith({ lifetimeMs: value }) expect(lifetime).toBe(max) }) }) describe('resolvePiRunLifetimeMs', () => { - it('keeps the provider ceiling when the execution is untimed', async () => { - const { createTimeoutAbortController } = await import('@/lib/core/execution-limits') - const { resolvePiRunLifetimeMs, PI_SANDBOX_MAX_LIFETIME_MS } = await import( - '@/lib/execution/remote-sandbox/pi-lifetime' - ) - + it('keeps the provider ceiling when the execution is untimed', () => { // No timeout means no deadline was recorded, so there is nothing to narrow // to — the ceiling is the only bound available. const untimed = createTimeoutAbortController() @@ -104,12 +113,7 @@ describe('resolvePiRunLifetimeMs', () => { expect(resolvePiRunLifetimeMs()).toBe(PI_SANDBOX_MAX_LIFETIME_MS) }) - it('narrows to the deadline of a run shorter than the ceiling', async () => { - const { createTimeoutAbortController } = await import('@/lib/core/execution-limits') - const { resolvePiRunLifetimeMs, PI_SANDBOX_MAX_LIFETIME_MS } = await import( - '@/lib/execution/remote-sandbox/pi-lifetime' - ) - + it('narrows to the deadline of a run shorter than the ceiling', () => { // A free-plan sync run gets five minutes. Handing its sandbox the sub-hour // ceiling is what left an orphan billing for an hour after a five-minute run. const timeout = createTimeoutAbortController(5 * 60 * 1000) @@ -121,12 +125,7 @@ describe('resolvePiRunLifetimeMs', () => { timeout.cleanup() }) - it('keeps the ceiling when the run outlives it', async () => { - const { createTimeoutAbortController } = await import('@/lib/core/execution-limits') - const { resolvePiRunLifetimeMs, PI_SANDBOX_MAX_LIFETIME_MS } = await import( - '@/lib/execution/remote-sandbox/pi-lifetime' - ) - + it('keeps the ceiling when the run outlives it', () => { // The deadline must be strictly past the ceiling for the ceiling to win. // Passing exactly `PI_SANDBOX_MAX_LIFETIME_MS` made this a coin flip: the // remaining budget is `deadline - Date.now()`, so it decays below the ceiling @@ -140,27 +139,17 @@ describe('resolvePiRunLifetimeMs', () => { timeout.cleanup() }) - it('keeps the ceiling for a signal that carries no deadline', async () => { - const { resolvePiRunLifetimeMs, PI_SANDBOX_MAX_LIFETIME_MS } = await import( - '@/lib/execution/remote-sandbox/pi-lifetime' - ) - + it('keeps the ceiling for a signal that carries no deadline', () => { // A derived or foreign signal reports `undefined` remaining, which means // "unknown", not "expired" — narrowing to zero there would kill every run. expect(resolvePiRunLifetimeMs(new AbortController().signal)).toBe(PI_SANDBOX_MAX_LIFETIME_MS) }) - it('has no lifetime to narrow on a provider without one', async () => { - vi.resetModules() - vi.doMock('@/lib/core/config/env', () => ({ - env: { SANDBOX_PROVIDER: 'daytona' }, - })) - const { createTimeoutAbortController } = await import('@/lib/core/execution-limits') - const { resolvePiRunLifetimeMs } = await import('@/lib/execution/remote-sandbox/pi-lifetime') - + it('has no lifetime to narrow on a provider without one', () => { // Daytona stops on inactivity, so imposing the run's deadline as an absolute // lifetime would cut a turn to fit a limit that does not apply to it. const timeout = createTimeoutAbortController(5 * 60 * 1000) + mockEnv.SANDBOX_PROVIDER = 'daytona' expect(resolvePiRunLifetimeMs(timeout.signal)).toBeUndefined() timeout.cleanup() diff --git a/apps/sim/lib/execution/remote-sandbox/pi-lifetime.ts b/apps/sim/lib/execution/remote-sandbox/pi-lifetime.ts index 9ba59554840..3a351af8412 100644 --- a/apps/sim/lib/execution/remote-sandbox/pi-lifetime.ts +++ b/apps/sim/lib/execution/remote-sandbox/pi-lifetime.ts @@ -6,17 +6,18 @@ import { createLogger } from '@sim/logger' import { env } from '@/lib/core/config/env' +import { inspectCapability, SANDBOX_CAPABILITY } from '@/lib/core/config/env-capabilities' import { getMaxExecutionTimeout, getRemainingExecutionMs } from '@/lib/core/execution-limits' const logger = createLogger('PiSandboxLifetime') /** - * Read from `env` rather than the `env-flags` gate, and normalized the same way - * `remote-sandbox/index.ts` normalizes it, so this module keeps the independence - * its header describes: no provider adapters, no barrel, no config gate. + * Uses tolerant capability inspection because this module only needs to know + * whether an E2B lifetime applies. Strict credential validation remains at the + * point where the selected sandbox provider is created. */ function isLifetimeProvider(): boolean { - return (env.SANDBOX_PROVIDER || 'e2b').toLowerCase() === 'e2b' + return inspectCapability(SANDBOX_CAPABILITY, env).providerId === 'e2b' } /** diff --git a/apps/sim/lib/execution/remote-sandbox/provider.ts b/apps/sim/lib/execution/remote-sandbox/provider.ts index 1d662194dfd..f29a310499d 100644 --- a/apps/sim/lib/execution/remote-sandbox/provider.ts +++ b/apps/sim/lib/execution/remote-sandbox/provider.ts @@ -1,4 +1,4 @@ -import { env } from '@/lib/core/config/env' +import { getConfiguredSandboxProviderId } from '@/lib/core/config/env-capabilities.server' import { daytonaProvider } from '@/lib/execution/remote-sandbox/daytona' import { e2bProvider } from '@/lib/execution/remote-sandbox/e2b' import type { SandboxProvider, SandboxProviderId } from '@/lib/execution/remote-sandbox/types' @@ -13,11 +13,9 @@ const PROVIDERS: Record = { daytona: daytonaProvider, } -const DEFAULT_PROVIDER: SandboxProviderId = 'e2b' - /** * Resolves which provider serves this execution from the `SANDBOX_PROVIDER` env - * var (defaulting to {@link DEFAULT_PROVIDER}). + * var (defaulting to E2B). * * Selection is deliberately resolved ONCE, before the sandbox is created, and is * never revisited mid-execution: user code has side effects (HTTP calls, S3 @@ -26,16 +24,6 @@ const DEFAULT_PROVIDER: SandboxProviderId = 'e2b' * `SANDBOX_PROVIDER` and redeploy; in-flight executions are unaffected. */ export function resolveProvider(): SandboxProvider { - // Normalize casing identically to env-flags' availability gate — otherwise a - // value like `Daytona` would pass the gate (which lowercases) but miss this - // lowercase-keyed map and throw at create time. - const configured = env.SANDBOX_PROVIDER?.toLowerCase() - if (!configured) return PROVIDERS[DEFAULT_PROVIDER] - const provider = PROVIDERS[configured as SandboxProviderId] - if (!provider) { - throw new Error( - `Unknown SANDBOX_PROVIDER "${env.SANDBOX_PROVIDER}" (expected one of: ${Object.keys(PROVIDERS).join(', ')})` - ) - } - return provider + const configured = getConfiguredSandboxProviderId() + return PROVIDERS[configured] } diff --git a/apps/sim/lib/integrations/availability.server.test.ts b/apps/sim/lib/integrations/availability.server.test.ts new file mode 100644 index 00000000000..d3a5a6eff19 --- /dev/null +++ b/apps/sim/lib/integrations/availability.server.test.ts @@ -0,0 +1,226 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/config/env', () => ({ env: {} })) + +import { + OAUTH_CLIENT_CAPABILITIES, + resolveOAuthClientCapabilityId, +} from '@/lib/core/config/env-capabilities' +import { + getIntegrationTypesForOAuthServiceId, + type IntegrationAvailability, + isOAuthServiceAllowedByIntegrationTypes, + resolveIntegrationAvailability, + resolveIntegrationAvailabilityStateForVisibility, +} from '@/lib/integrations/availability' +import { + isIntegrationDeploymentAvailable, + isIntegrationDeploymentAvailableForVisibility, +} from '@/lib/integrations/availability.server' +import integrationsJson from '@/lib/integrations/integrations.json' +import { SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID } from '@/lib/integrations/service-account-metadata' +import type { Integration } from '@/lib/integrations/types' +import { getServiceConfigByServiceId } from '@/lib/oauth/utils' + +const integrations = integrationsJson.integrations as readonly Integration[] + +function availabilityFor( + type: string, + values: Parameters[0] = {} +): IntegrationAvailability { + const availability = resolveIntegrationAvailability(values).find((item) => item.type === type) + if (!availability) throw new Error(`Missing integration availability for ${type}`) + return availability +} + +describe('integration availability', () => { + it('marks a configured OAuth integration ready', () => { + expect( + availabilityFor('slack', { + SLACK_CLIENT_ID: 'client', + SLACK_CLIENT_SECRET: 'secret', + }) + ).toMatchObject({ + name: 'Slack', + slug: 'slack', + state: 'ready', + oauthAvailable: true, + serviceAccountAvailable: false, + missingFields: [], + setupCommand: 'bun run setup integration slack', + }) + }) + + it('marks an integration with an ungated service-account path as limited', () => { + expect(availabilityFor('notion_v2')).toMatchObject({ + state: 'limited', + oauthAvailable: false, + serviceAccountAvailable: true, + missingFields: ['NOTION_CLIENT_ID', 'NOTION_CLIENT_SECRET'], + setupCommand: 'bun run setup integration notion', + }) + }) + + it('keeps an ungated service-account path available when OAuth is partial', () => { + expect(availabilityFor('notion_v2', { NOTION_CLIENT_ID: 'client' })).toMatchObject({ + state: 'limited', + oauthAvailable: false, + serviceAccountAvailable: true, + missingFields: ['NOTION_CLIENT_SECRET'], + }) + }) + + it('marks an unconfigured OAuth-only integration unavailable', () => { + expect(availabilityFor('x')).toMatchObject({ + state: 'unavailable', + oauthAvailable: false, + setupCommand: 'bun run setup integration x', + }) + }) + + it('reports a partially configured OAuth client and its missing fields', () => { + expect(availabilityFor('slack', { SLACK_CLIENT_ID: 'client' })).toMatchObject({ + state: 'misconfigured', + oauthAvailable: false, + serviceAccountAvailable: false, + missingFields: ['SLACK_CLIENT_SECRET'], + setupCommand: 'bun run setup integration slack', + }) + }) + + it('projects a revealed preview service-account path as limited', () => { + const unavailableSlack = availabilityFor('slack') + const misconfiguredSlack = availabilityFor('slack', { SLACK_CLIENT_ID: 'client' }) + const revealed = { + revealed: new Set(['slack_v2']), + disabled: new Set(), + previewTagged: new Set(['slack_v2']), + } + + expect(resolveIntegrationAvailabilityStateForVisibility(unavailableSlack, null)).toBe( + 'unavailable' + ) + expect(resolveIntegrationAvailabilityStateForVisibility(unavailableSlack, revealed)).toBe( + 'limited' + ) + expect(resolveIntegrationAvailabilityStateForVisibility(misconfiguredSlack, revealed)).toBe( + 'limited' + ) + }) + + it('keeps preview service accounts unavailable when their block is kill-switched', () => { + const unavailableSlack = availabilityFor('slack') + const disabled = { + revealed: new Set(['slack_v2']), + disabled: new Set(['slack_v2']), + previewTagged: new Set(['slack_v2']), + } + + expect(resolveIntegrationAvailabilityStateForVisibility(unavailableSlack, disabled)).toBe( + 'unavailable' + ) + expect(resolveIntegrationAvailabilityStateForVisibility(availabilityFor('x'), disabled)).toBe( + 'unavailable' + ) + }) + + it('projects base and versioned deployment availability through explicit visibility', () => { + const revealed = { + revealed: new Set(['slack_v2']), + disabled: new Set(), + previewTagged: new Set(['slack_v2']), + } + const disabled = { + ...revealed, + disabled: new Set(['slack_v2']), + } + + expect(isIntegrationDeploymentAvailable('slack')).toBe(false) + expect(isIntegrationDeploymentAvailable('slack_v2')).toBe(false) + expect(isIntegrationDeploymentAvailable('slack-v2')).toBe(false) + expect(isIntegrationDeploymentAvailableForVisibility('slack', null)).toBe(false) + expect(isIntegrationDeploymentAvailableForVisibility('slack_v2', null)).toBe(false) + expect(isIntegrationDeploymentAvailableForVisibility('slack-v2', null)).toBe(false) + expect(isIntegrationDeploymentAvailableForVisibility('slack', revealed)).toBe(true) + expect(isIntegrationDeploymentAvailableForVisibility('slack_v2', revealed)).toBe(true) + expect(isIntegrationDeploymentAvailableForVisibility('slack-v2', revealed)).toBe(true) + expect(isIntegrationDeploymentAvailableForVisibility('x', revealed)).toBe(false) + expect(isIntegrationDeploymentAvailableForVisibility('slack', disabled)).toBe(false) + expect(isIntegrationDeploymentAvailableForVisibility('slack_v2', disabled)).toBe(false) + expect(isIntegrationDeploymentAvailableForVisibility('slack-v2', disabled)).toBe(false) + }) + + it('requires the deployment Trello API key for OAuth and pasted member tokens', () => { + expect(availabilityFor('trello')).toMatchObject({ + state: 'unavailable', + serviceAccountAvailable: false, + missingFields: ['TRELLO_API_KEY'], + setupCommand: 'bun run setup integration trello', + }) + expect(availabilityFor('trello', { TRELLO_API_KEY: 'trello-key' })).toMatchObject({ + state: 'ready', + oauthAvailable: true, + serviceAccountAvailable: true, + missingFields: [], + }) + }) + + it('maps OAuth service ids to the integration allowlist without loading registries', () => { + expect(getIntegrationTypesForOAuthServiceId('gmail')).toContain('gmail_v2') + expect(isOAuthServiceAllowedByIntegrationTypes('gmail', new Set(['slack']))).toBe(false) + expect(isOAuthServiceAllowedByIntegrationTypes('slack', new Set(['slack']))).toBe(true) + expect(isOAuthServiceAllowedByIntegrationTypes('spotify', null)).toBe(true) + }) + + it('returns every visible integration and only emits accepted setup commands', () => { + const availability = resolveIntegrationAvailability({}) + expect(availability).toHaveLength(integrations.length) + + for (const integration of availability) { + if (!integration.setupCommand) continue + const capabilityId = integration.setupCommand.replace('bun run setup integration ', '') + expect(Object.hasOwn(OAUTH_CLIENT_CAPABILITIES, capabilityId)).toBe(true) + } + }) + + it('keeps service-account metadata in parity with canonical OAuth services', () => { + const oauthServiceIds = [ + ...new Set( + integrations.flatMap((integration) => + integration.authType === 'oauth' && integration.oauthServiceId + ? [integration.oauthServiceId] + : [] + ) + ), + ] + const expectedServiceAccountIds: Record = {} + + for (const oauthServiceId of oauthServiceIds) { + const canonical = getServiceConfigByServiceId(oauthServiceId) + if (!canonical) throw new Error(`Missing canonical OAuth service ${oauthServiceId}`) + const projected = SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID[oauthServiceId] + expect(projected?.providerId, oauthServiceId).toBe(canonical.serviceAccountProviderId) + if (canonical.serviceAccountProviderId) { + expectedServiceAccountIds[oauthServiceId] = canonical.serviceAccountProviderId + } + } + + expect( + Object.fromEntries( + Object.entries(SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID).map( + ([serviceId, metadata]) => [serviceId, metadata.providerId] + ) + ) + ).toEqual(expectedServiceAccountIds) + expect(SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID.slack.deploymentRequirement).toBe( + 'preview-gated' + ) + expect(SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID.trello.deploymentRequirement).toBe( + 'oauth-client' + ) + expect(resolveOAuthClientCapabilityId('trello')).toBe('trello') + }) +}) diff --git a/apps/sim/lib/integrations/availability.server.ts b/apps/sim/lib/integrations/availability.server.ts new file mode 100644 index 00000000000..8de88f590ed --- /dev/null +++ b/apps/sim/lib/integrations/availability.server.ts @@ -0,0 +1,96 @@ +import { stripVersionSuffix } from '@sim/utils/string' +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' +import { env } from '@/lib/core/config/env' +import { + inspectOAuthClientCapability, + resolveOAuthClientCapabilityId, +} from '@/lib/core/config/env-capabilities' +import { + type IntegrationAvailability, + resolveIntegrationAvailability, + resolveIntegrationAvailabilityStateForVisibility, +} from '@/lib/integrations/availability' + +export type { + IntegrationAvailability, + IntegrationAvailabilityState, +} from '@/lib/integrations/availability' + +let unavailableIntegrationTypes: ReadonlySet | null = null +let integrationAvailabilityByType: ReadonlyMap | null = null +const oauthServiceAvailability = new Map() + +export function getIntegrationAvailability() { + return resolveIntegrationAvailability(env) +} + +export function getUnavailableIntegrationTypes(): ReadonlySet { + if (!unavailableIntegrationTypes) { + unavailableIntegrationTypes = new Set( + getIntegrationAvailability() + .filter( + (integration) => + integration.state === 'unavailable' || integration.state === 'misconfigured' + ) + .map((integration) => integration.type.toLowerCase()) + ) + } + return unavailableIntegrationTypes +} + +function getIntegrationAvailabilityByType(): ReadonlyMap { + if (!integrationAvailabilityByType) { + integrationAvailabilityByType = new Map( + getIntegrationAvailability().map((availability) => [ + availability.type.toLowerCase(), + availability, + ]) + ) + } + return integrationAvailabilityByType +} + +function getIntegrationAvailabilityForBlockType( + blockType: string +): IntegrationAvailability | undefined { + const normalized = blockType.toLowerCase().replace(/-/g, '_') + const availabilityByType = getIntegrationAvailabilityByType() + return ( + availabilityByType.get(normalized) ?? availabilityByType.get(stripVersionSuffix(normalized)) + ) +} + +export function isIntegrationDeploymentAvailable(blockType: string): boolean { + const availability = getIntegrationAvailabilityForBlockType(blockType) + return ( + !availability || + (availability.state !== 'unavailable' && availability.state !== 'misconfigured') + ) +} + +/** + * Whether an integration can be used in this deployment for the current + * viewer. The cached catalog remains viewer-agnostic; preview service-account + * alternatives are projected against explicitly supplied block visibility. + */ +export function isIntegrationDeploymentAvailableForVisibility( + blockType: string, + visibility: BlockVisibilityState | null +): boolean { + const availability = getIntegrationAvailabilityForBlockType(blockType) + if (!availability) return true + const state = resolveIntegrationAvailabilityStateForVisibility(availability, visibility) + return state !== 'unavailable' && state !== 'misconfigured' +} + +export function isOAuthServiceDeploymentAvailable(serviceId: string): boolean { + const normalized = serviceId.toLowerCase() + const cached = oauthServiceAvailability.get(normalized) + if (cached !== undefined) return cached + const capabilityId = resolveOAuthClientCapabilityId(normalized) + const available = capabilityId + ? inspectOAuthClientCapability(capabilityId, env).state === 'ready' + : true + oauthServiceAvailability.set(normalized, available) + return available +} diff --git a/apps/sim/lib/integrations/availability.ts b/apps/sim/lib/integrations/availability.ts new file mode 100644 index 00000000000..7c1669ddc3c --- /dev/null +++ b/apps/sim/lib/integrations/availability.ts @@ -0,0 +1,175 @@ +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' +import type { EnvCapabilityValues } from '@/lib/core/config/env-capabilities' +import { + inspectOAuthClientCapability, + resolveOAuthClientCapabilityId, +} from '@/lib/core/config/env-capabilities' +import { getServiceAccountGatingBlockType } from '@/lib/credentials/service-account-provider-ids' +import integrationsJson from '@/lib/integrations/integrations.json' +import { getServiceAccountMetadata } from '@/lib/integrations/service-account-metadata' +import { isHiddenUnder } from '@/blocks/visibility/context' + +export type IntegrationAvailabilityState = 'ready' | 'limited' | 'unavailable' | 'misconfigured' + +export interface IntegrationAvailability { + type: string + slug: string + name: string + state: IntegrationAvailabilityState + oauthAvailable: boolean + serviceAccountAvailable: boolean + missingFields: readonly string[] + setupCommand?: string +} + +interface DeploymentIntegration { + type: string + slug: string + name: string + authType: 'oauth' | 'api-key' | 'none' + oauthServiceId?: string +} + +const integrations = integrationsJson.integrations as readonly DeploymentIntegration[] +const deploymentGatedIntegrationTypes = new Set( + integrations + .filter((integration) => integration.authType === 'oauth') + .map((integration) => integration.type.toLowerCase()) +) +const integrationTypesByOAuthServiceId = new Map() +const previewServiceAccountGatesByIntegrationType = new Map() +for (const integration of integrations) { + if (integration.authType !== 'oauth' || !integration.oauthServiceId) continue + const serviceId = integration.oauthServiceId.toLowerCase() + const current = integrationTypesByOAuthServiceId.get(serviceId) ?? [] + const integrationType = integration.type.toLowerCase() + integrationTypesByOAuthServiceId.set(serviceId, [...current, integrationType]) + + const serviceAccount = getServiceAccountMetadata(serviceId) + if (serviceAccount?.deploymentRequirement !== 'preview-gated') continue + const gatingBlockType = getServiceAccountGatingBlockType(serviceAccount.providerId) + if (!gatingBlockType) { + throw new Error( + `Preview-gated service account ${serviceAccount.providerId} has no gating block type` + ) + } + previewServiceAccountGatesByIntegrationType.set(integrationType, gatingBlockType) +} + +export function isDeploymentGatedIntegrationType(blockType: string): boolean { + return deploymentGatedIntegrationTypes.has(blockType.toLowerCase()) +} + +/** Returns the generated integration block types authenticated by one OAuth service entry. */ +export function getIntegrationTypesForOAuthServiceId(serviceId: string): readonly string[] { + return integrationTypesByOAuthServiceId.get(serviceId.toLowerCase()) ?? [] +} + +/** Applies an integration allowlist to an OAuth service without loading executable registries. */ +export function isOAuthServiceAllowedByIntegrationTypes( + serviceId: string, + allowedIntegrationTypes: ReadonlySet | null +): boolean { + if (allowedIntegrationTypes === null) return true + const integrationTypes = getIntegrationTypesForOAuthServiceId(serviceId) + return ( + integrationTypes.length === 0 || + integrationTypes.some((blockType) => allowedIntegrationTypes.has(blockType)) + ) +} + +interface IntegrationAvailabilitySummary { + type: string + state: IntegrationAvailabilityState + oauthAvailable: boolean +} + +/** + * Projects deployment availability through the current viewer's block gate. + * A revealed preview service-account path makes an OAuth-unavailable + * integration limited rather than unavailable; the OAuth path itself remains + * disabled. The shared hidden predicate keeps preview and kill-switch behavior + * identical to every other block discovery surface. + */ +export function resolveIntegrationAvailabilityStateForVisibility( + availability: IntegrationAvailabilitySummary, + visibility: BlockVisibilityState | null +): IntegrationAvailabilityState { + const gatingBlockType = previewServiceAccountGatesByIntegrationType.get( + availability.type.toLowerCase() + ) + if (!gatingBlockType || isHiddenUnder(visibility, { type: gatingBlockType, preview: true })) { + return availability.state + } + return availability.oauthAvailable ? 'ready' : 'limited' +} + +function resolveOAuthIntegrationAvailability( + integration: DeploymentIntegration, + values: EnvCapabilityValues +): IntegrationAvailability { + const { oauthServiceId } = integration + if (!oauthServiceId) { + throw new Error(`OAuth integration ${integration.slug} is missing oauthServiceId`) + } + + const capabilityId = resolveOAuthClientCapabilityId(oauthServiceId) + const serviceAccount = getServiceAccountMetadata(oauthServiceId) + + if (!capabilityId) { + throw new Error( + `OAuth integration ${integration.slug} has no OAuth client capability definition` + ) + } + + const oauth = inspectOAuthClientCapability(capabilityId, values) + const setupCommand = `bun run setup integration ${capabilityId}` + const serviceAccountAvailable = Boolean( + serviceAccount && + serviceAccount.deploymentRequirement !== 'preview-gated' && + (serviceAccount.deploymentRequirement !== 'oauth-client' || oauth.state === 'ready') + ) + const state: IntegrationAvailabilityState = + oauth.state === 'ready' + ? 'ready' + : serviceAccountAvailable + ? 'limited' + : oauth.state === 'partial' || oauth.state === 'invalid' + ? 'misconfigured' + : 'unavailable' + + return { + type: integration.type, + slug: integration.slug, + name: integration.name, + state, + oauthAvailable: oauth.state === 'ready', + serviceAccountAvailable, + missingFields: oauth.missingFields, + setupCommand, + } +} + +/** + * Resolves deployment availability for every integration in the generated + * catalog using only caller-supplied environment values and pure metadata. + */ +export function resolveIntegrationAvailability( + values: EnvCapabilityValues +): readonly IntegrationAvailability[] { + return integrations.map((integration) => { + if (integration.authType === 'oauth') { + return resolveOAuthIntegrationAvailability(integration, values) + } + + return { + type: integration.type, + slug: integration.slug, + name: integration.name, + state: 'ready', + oauthAvailable: false, + serviceAccountAvailable: false, + missingFields: [], + } + }) +} diff --git a/apps/sim/lib/integrations/credential-visibility.server.test.ts b/apps/sim/lib/integrations/credential-visibility.server.test.ts new file mode 100644 index 00000000000..502adecdc3b --- /dev/null +++ b/apps/sim/lib/integrations/credential-visibility.server.test.ts @@ -0,0 +1,177 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { IntegrationAvailability } from '@/lib/integrations/availability' +import type { OAuthServiceMetadata } from '@/lib/oauth/types' + +const { getBlockMock, getIntegrationAvailabilityMock } = vi.hoisted(() => ({ + getBlockMock: vi.fn(), + getIntegrationAvailabilityMock: vi.fn(), +})) + +vi.mock('@/blocks/registry', () => ({ getBlock: getBlockMock })) +vi.mock('@/lib/integrations/availability.server', () => ({ + getIntegrationAvailability: getIntegrationAvailabilityMock, + isOAuthServiceDeploymentAvailable: vi.fn(() => true), +})) + +import { createIntegrationCredentialVisibility } from '@/lib/integrations/credential-visibility.server' + +const SERVICES: readonly OAuthServiceMetadata[] = [ + { + serviceId: 'notion', + providerId: 'notion', + serviceAccountProviderId: 'notion-service-account', + name: 'Notion', + description: 'Notion workspace', + baseProvider: 'notion', + authType: 'oauth', + }, + { + serviceId: 'slack', + providerId: 'slack', + serviceAccountProviderId: 'slack-custom-bot', + name: 'Slack', + description: 'Slack workspace', + baseProvider: 'slack', + authType: 'oauth', + }, +] + +function availability( + type: string, + state: IntegrationAvailability['state'], + options: Pick +): IntegrationAvailability { + return { + type, + slug: type, + name: type, + state, + missingFields: [], + ...options, + } +} + +describe('integration credential visibility', () => { + beforeEach(() => { + vi.clearAllMocks() + getBlockMock.mockImplementation((type: string) => ({ + type, + ...(type === 'slack_v2' ? { preview: true } : {}), + })) + getIntegrationAvailabilityMock.mockReturnValue([ + availability('notion_v2', 'limited', { + oauthAvailable: false, + serviceAccountAvailable: true, + }), + availability('slack', 'unavailable', { + oauthAvailable: false, + serviceAccountAvailable: false, + }), + ]) + }) + + it('applies the integration allowlist to OAuth and service-account credentials', () => { + const visibility = createIntegrationCredentialVisibility({ + allowedIntegrationTypes: new Set(['slack']), + blockVisibility: null, + oauthServices: SERVICES, + }) + + expect(visibility.isCredentialVisible({ providerId: 'notion', type: 'oauth' })).toBe(false) + expect( + visibility.isCredentialVisible({ + providerId: 'notion-service-account', + type: 'service_account', + }) + ).toBe(false) + }) + + it('keeps an independent service-account fallback when OAuth is unavailable', () => { + const visibility = createIntegrationCredentialVisibility({ + allowedIntegrationTypes: new Set(['notion_v2']), + blockVisibility: null, + oauthServices: SERVICES, + }) + + expect(visibility.isCredentialVisible({ providerId: 'notion', type: 'oauth' })).toBe(false) + expect( + visibility.isCredentialVisible({ + providerId: 'notion-service-account', + type: 'service_account', + }) + ).toBe(true) + }) + + it('requires the Slack preview reveal for custom-bot credentials', () => { + const hidden = createIntegrationCredentialVisibility({ + allowedIntegrationTypes: new Set(['slack']), + blockVisibility: null, + oauthServices: SERVICES, + }) + const revealed = createIntegrationCredentialVisibility({ + allowedIntegrationTypes: new Set(['slack']), + blockVisibility: { + revealed: new Set(['slack_v2']), + disabled: new Set(), + previewTagged: new Set(['slack_v2']), + }, + oauthServices: SERVICES, + }) + + const credential = { providerId: 'slack-custom-bot', type: 'service_account' } as const + expect(hidden.isCredentialVisible(credential)).toBe(false) + expect(revealed.isCredentialVisible(credential)).toBe(true) + }) + + it('projects partial OAuth state through a revealed service-account preview', () => { + getIntegrationAvailabilityMock.mockReturnValue([ + availability('slack', 'misconfigured', { + oauthAvailable: false, + serviceAccountAvailable: false, + }), + ]) + const visibility = createIntegrationCredentialVisibility({ + allowedIntegrationTypes: new Set(['slack']), + blockVisibility: { + revealed: new Set(['slack_v2']), + disabled: new Set(), + previewTagged: new Set(['slack_v2']), + }, + oauthServices: SERVICES, + }) + const disabled = createIntegrationCredentialVisibility({ + allowedIntegrationTypes: new Set(['slack']), + blockVisibility: { + revealed: new Set(['slack_v2']), + disabled: new Set(['slack_v2']), + previewTagged: new Set(['slack_v2']), + }, + oauthServices: SERVICES, + }) + + const credential = { + providerId: 'slack-custom-bot', + type: 'service_account', + } as const + expect(visibility.isCredentialVisible(credential)).toBe(true) + expect(disabled.isCredentialVisible(credential)).toBe(false) + }) + + it('leaves non-integration credentials visible', () => { + const visibility = createIntegrationCredentialVisibility({ + allowedIntegrationTypes: new Set(), + blockVisibility: null, + oauthServices: SERVICES, + }) + + expect( + visibility.isCredentialVisible({ + providerId: 'claude-platform', + type: 'service_account', + }) + ).toBe(true) + }) +}) diff --git a/apps/sim/lib/integrations/credential-visibility.server.ts b/apps/sim/lib/integrations/credential-visibility.server.ts new file mode 100644 index 00000000000..7608fc059f9 --- /dev/null +++ b/apps/sim/lib/integrations/credential-visibility.server.ts @@ -0,0 +1,142 @@ +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' +import { getServiceAccountGatingBlockType } from '@/lib/credentials/service-account-provider-ids' +import { + getIntegrationTypesForOAuthServiceId, + isOAuthServiceAllowedByIntegrationTypes, + resolveIntegrationAvailabilityStateForVisibility, +} from '@/lib/integrations/availability' +import { + getIntegrationAvailability, + isOAuthServiceDeploymentAvailable, +} from '@/lib/integrations/availability.server' +import type { OAuthServiceMetadata } from '@/lib/oauth/types' +import { getAllOAuthServices } from '@/lib/oauth/utils' +import { getBlock } from '@/blocks/registry' +import { isHiddenUnder } from '@/blocks/visibility/context' + +export interface IntegrationCredentialIdentity { + providerId: string + type?: 'oauth' | 'service_account' +} + +interface IntegrationCredentialVisibilityOptions { + allowedIntegrationTypes: ReadonlySet | null + blockVisibility: BlockVisibilityState | null + oauthServices?: readonly OAuthServiceMetadata[] +} + +export interface IntegrationCredentialVisibility { + isCredentialVisible: (credential: IntegrationCredentialIdentity) => boolean + isOAuthServiceVisible: (service: OAuthServiceMetadata) => boolean +} + +/** + * Builds the server-side projection shared by Copilot credential discovery and + * the workspace VFS. Unknown provider ids are not integration credentials and + * remain visible; mapped OAuth and service-account ids must have at least one + * owning integration that is allowed, deployment-ready, and visible to the + * current block-visibility projection. + */ +export function createIntegrationCredentialVisibility({ + allowedIntegrationTypes, + blockVisibility, + oauthServices = getAllOAuthServices(), +}: IntegrationCredentialVisibilityOptions): IntegrationCredentialVisibility { + const oauthOwners = oauthServices.filter((service) => service.authType === 'oauth') + const oauthOwnersByProviderId = new Map() + const serviceAccountOwnersByProviderId = new Map() + const availabilityByType = new Map( + getIntegrationAvailability().map((availability) => [ + availability.type.toLowerCase(), + availability, + ]) + ) + + const addOwner = ( + ownersByProviderId: Map, + providerId: string, + service: OAuthServiceMetadata + ) => { + const owners = ownersByProviderId.get(providerId) + if (owners) owners.push(service) + else ownersByProviderId.set(providerId, [service]) + } + + for (const service of oauthOwners) { + addOwner(oauthOwnersByProviderId, service.providerId, service) + if (service.serviceAccountProviderId) { + addOwner(serviceAccountOwnersByProviderId, service.serviceAccountProviderId, service) + } + } + + const isServiceAllowed = (service: OAuthServiceMetadata) => + isOAuthServiceAllowedByIntegrationTypes(service.serviceId, allowedIntegrationTypes) + + const visibleAvailability = (service: OAuthServiceMetadata) => { + return getIntegrationTypesForOAuthServiceId(service.serviceId).flatMap((blockType) => { + const block = getBlock(blockType) + if (!block || block.hideFromToolbar || isHiddenUnder(blockVisibility, block)) return [] + const availability = availabilityByType.get(blockType.toLowerCase()) + return availability ? [availability] : [] + }) + } + + const isOAuthServiceVisible = (service: OAuthServiceMetadata): boolean => { + if (service.authType !== 'oauth' || !isServiceAllowed(service)) return false + const integrationTypes = getIntegrationTypesForOAuthServiceId(service.serviceId) + if (integrationTypes.length === 0) { + return isOAuthServiceDeploymentAvailable(service.providerId) + } + return visibleAvailability(service).some( + (availability) => availability.state === 'ready' && availability.oauthAvailable + ) + } + + const isServiceAccountVisible = ( + providerId: string, + owners: readonly OAuthServiceMetadata[] + ): boolean => { + const gatingBlockType = getServiceAccountGatingBlockType(providerId) + if (gatingBlockType) { + const gatingBlock = getBlock(gatingBlockType) + if (!gatingBlock || isHiddenUnder(blockVisibility, gatingBlock)) return false + return owners.some( + (service) => + isServiceAllowed(service) && + visibleAvailability(service).some((availability) => { + const state = resolveIntegrationAvailabilityStateForVisibility( + availability, + blockVisibility + ) + return state === 'ready' || state === 'limited' + }) + ) + } + + return owners.some( + (service) => + isServiceAllowed(service) && + visibleAvailability(service).some( + (availability) => + availability.serviceAccountAvailable && + (availability.state === 'ready' || availability.state === 'limited') + ) + ) + } + + const isCredentialVisible = ({ providerId, type }: IntegrationCredentialIdentity): boolean => { + if (type !== 'service_account') { + const owners = oauthOwnersByProviderId.get(providerId) + if (owners) return owners.some(isOAuthServiceVisible) + } + + if (type !== 'oauth') { + const owners = serviceAccountOwnersByProviderId.get(providerId) + if (owners) return isServiceAccountVisible(providerId, owners) + } + + return true + } + + return { isCredentialVisible, isOAuthServiceVisible } +} diff --git a/apps/sim/lib/integrations/service-account-metadata.ts b/apps/sim/lib/integrations/service-account-metadata.ts new file mode 100644 index 00000000000..a1a90346662 --- /dev/null +++ b/apps/sim/lib/integrations/service-account-metadata.ts @@ -0,0 +1,61 @@ +/** + * Lightweight deployment metadata for OAuth services that also accept a + * user-supplied service-account credential. + * + * This projection deliberately contains no icons, scopes, or OAuth runtime + * configuration so deployment tooling can inspect integration availability + * without loading the executable integration graph. Its parity with the + * canonical OAuth service configuration is enforced by an invariant test. + */ + +export interface ServiceAccountMetadata { + providerId: string + deploymentRequirement?: 'preview-gated' | 'oauth-client' +} + +export const SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID: Readonly< + Record +> = { + airtable: { providerId: 'airtable-service-account' }, + asana: { providerId: 'asana-service-account' }, + attio: { providerId: 'attio-service-account' }, + box: { providerId: 'box-service-account' }, + calcom: { providerId: 'calcom-service-account' }, + clickup: { providerId: 'clickup-service-account' }, + confluence: { providerId: 'atlassian-service-account' }, + gmail: { providerId: 'google-service-account' }, + 'google-bigquery': { providerId: 'google-service-account' }, + 'google-calendar': { providerId: 'google-service-account' }, + 'google-contacts': { providerId: 'google-service-account' }, + 'google-docs': { providerId: 'google-service-account' }, + 'google-drive': { providerId: 'google-service-account' }, + 'google-forms': { providerId: 'google-service-account' }, + 'google-groups': { providerId: 'google-service-account' }, + 'google-meet': { providerId: 'google-service-account' }, + 'google-sheets': { providerId: 'google-service-account' }, + 'google-tasks': { providerId: 'google-service-account' }, + 'google-vault': { providerId: 'google-service-account' }, + hubspot: { providerId: 'hubspot-service-account' }, + jira: { providerId: 'atlassian-service-account' }, + linear: { providerId: 'linear-service-account' }, + monday: { providerId: 'monday-service-account' }, + notion: { providerId: 'notion-service-account' }, + pipedrive: { providerId: 'pipedrive-service-account' }, + salesforce: { providerId: 'salesforce-service-account' }, + shopify: { providerId: 'shopify-service-account' }, + slack: { providerId: 'slack-custom-bot', deploymentRequirement: 'preview-gated' }, + trello: { providerId: 'trello-service-account', deploymentRequirement: 'oauth-client' }, + wealthbox: { providerId: 'wealthbox-service-account' }, + webflow: { providerId: 'webflow-service-account' }, + 'zoho-desk': { providerId: 'zoho-desk-service-account' }, + zoom: { providerId: 'zoom-service-account' }, +} as const + +export function getServiceAccountMetadata( + oauthServiceId: string +): ServiceAccountMetadata | undefined { + if (!Object.hasOwn(SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID, oauthServiceId)) { + return undefined + } + return SERVICE_ACCOUNT_METADATA_BY_OAUTH_SERVICE_ID[oauthServiceId] +} diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts index 2c1eb839d9f..0f61f83f376 100644 --- a/apps/sim/lib/knowledge/documents/document-processor.ts +++ b/apps/sim/lib/knowledge/documents/document-processor.ts @@ -15,6 +15,7 @@ import { } from '@/lib/chunkers' import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types' import { env, envNumber } from '@/lib/core/config/env' +import { OCR_CAPABILITY, requireCapability } from '@/lib/core/config/env-capabilities' import { parseBuffer } from '@/lib/file-parsers' import type { FileParseMetadata } from '@/lib/file-parsers/types' import { resolveParserExtension } from '@/lib/knowledge/documents/parser-extension' @@ -287,19 +288,23 @@ async function parseDocument( metadata?: FileParseMetadata }> { const isPDF = mimeType === 'application/pdf' - const hasAzureMistralOCR = - env.OCR_AZURE_API_KEY && env.OCR_AZURE_ENDPOINT && env.OCR_AZURE_MODEL_NAME - const mistralApiKey = await getMistralApiKey(workspaceId) - const hasMistralOCR = !!mistralApiKey - if (isPDF && (hasAzureMistralOCR || hasMistralOCR)) { - if (hasAzureMistralOCR) { + if (isPDF) { + const ocrProvider = requireCapability(OCR_CAPABILITY, { + OCR_PROVIDER: env.OCR_PROVIDER, + OCR_AZURE_API_KEY: env.OCR_AZURE_API_KEY, + OCR_AZURE_ENDPOINT: env.OCR_AZURE_ENDPOINT, + OCR_AZURE_MODEL_NAME: env.OCR_AZURE_MODEL_NAME, + MISTRAL_API_KEY: mistralApiKey, + }).providerId + + if (ocrProvider === 'azure-mistral') { logger.info(`Using Azure Mistral OCR: ${filename}`) return parseWithAzureMistralOCR(fileUrl, filename, mimeType, userId) } - if (hasMistralOCR) { + if (ocrProvider === 'mistral') { logger.info(`Using Mistral OCR: ${filename}`) return parseWithMistralOCR(fileUrl, filename, mimeType, userId, workspaceId, mistralApiKey) } @@ -499,11 +504,9 @@ async function parseWithAzureMistralOCR( if (mimeType === 'application/pdf') { const pageCount = await getPdfPageCount(fileBuffer) if (pageCount > MISTRAL_MAX_PAGES) { - logger.info( - `PDF has ${pageCount} pages, exceeds Azure OCR limit of ${MISTRAL_MAX_PAGES}. ` + - `Falling back to file parser.` + throw new Error( + `PDF has ${pageCount} pages, exceeding the Azure OCR limit of ${MISTRAL_MAX_PAGES}` ) - return parseWithFileParser(fileUrl, filename, mimeType, userId) } logger.info(`Azure Mistral OCR: PDF page count for ${filename}: ${pageCount}`) } @@ -545,9 +548,7 @@ async function parseWithAzureMistralOCR( logger.error(`Azure Mistral OCR failed for ${filename}:`, { message: toError(error).message, }) - - logger.info(`Falling back to file parser: ${filename}`) - return parseWithFileParser(fileUrl, filename, mimeType, userId) + throw error } } @@ -605,9 +606,7 @@ async function parseWithMistralOCR( logger.error(`Mistral OCR failed for ${filename}:`, { message: toError(error).message, }) - - logger.info(`Falling back to file parser: ${filename}`) - return parseWithFileParser(fileUrl, filename, mimeType, userId) + throw error } } diff --git a/apps/sim/lib/messaging/email/mailer.ts b/apps/sim/lib/messaging/email/mailer.ts index 6b2dbbc84a7..6b879b421cc 100644 --- a/apps/sim/lib/messaging/email/mailer.ts +++ b/apps/sim/lib/messaging/email/mailer.ts @@ -2,7 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { getAccessControlConfig, isEmailBlockedByAccessControl } from '@/lib/auth/access-control' import { processEmailData, shouldSkipForUnsubscribe } from '@/lib/messaging/email/prepare' -import { activeProviders } from '@/lib/messaging/email/providers' +import { activeProviders, emailFallback } from '@/lib/messaging/email/providers' import type { BatchEmailOptions, BatchSendEmailResult, @@ -103,20 +103,15 @@ export async function sendEmail(options: EmailOptions): Promise } async function dispatchWithFallback(data: ProcessedEmailData): Promise { - let lastError: unknown - for (const provider of activeProviders) { - try { - return await provider.send(data) - } catch (error) { - lastError = error - logger.warn(`${provider.name} failed, trying next provider`, error) + try { + return await emailFallback.execute((provider) => provider.send(data)) + } catch (error) { + logger.error('All email providers failed', error) + return { + success: false, + message: getErrorMessage(error, 'All email providers failed'), } } - logger.error('All email providers failed', lastError) - return { - success: false, - message: `All email providers failed: ${getErrorMessage(lastError, 'unknown error')}`, - } } interface PreparedBatchEntry { diff --git a/apps/sim/lib/messaging/email/providers/index.ts b/apps/sim/lib/messaging/email/providers/index.ts index e383efc1e85..19428079e6a 100644 --- a/apps/sim/lib/messaging/email/providers/index.ts +++ b/apps/sim/lib/messaging/email/providers/index.ts @@ -1,4 +1,6 @@ import { createLogger } from '@sim/logger' +import { EMAIL_CAPABILITY, type FallbackFactories } from '@/lib/core/config/env-capabilities' +import { wireServerFallback } from '@/lib/core/config/env-capabilities.server' import { createAzureProvider } from '@/lib/messaging/email/providers/azure' import { createGmailProvider } from '@/lib/messaging/email/providers/gmail' import { createResendProvider } from '@/lib/messaging/email/providers/resend' @@ -8,23 +10,20 @@ import type { MailProvider } from '@/lib/messaging/email/types' const logger = createLogger('MailProviders') -const factories = [ - createResendProvider, - createSesProvider, - createSmtpProvider, - createAzureProvider, - createGmailProvider, -] as const +const factories = { + resend: createResendProvider, + ses: createSesProvider, + smtp: createSmtpProvider, + azure: createAzureProvider, + gmail: createGmailProvider, +} satisfies FallbackFactories -function safeCreate(factory: () => MailProvider | null): MailProvider | null { - try { - return factory() - } catch (error) { - logger.error('Mail provider factory threw at startup; skipping', error) - return null - } -} +export const emailFallback = wireServerFallback({ + definition: EMAIL_CAPABILITY, + factories, + onFailure(providerId, error) { + logger.warn(`${providerId} failed, trying next provider`, error) + }, +}) -export const activeProviders: readonly MailProvider[] = factories - .map((factory) => safeCreate(factory)) - .filter((provider): provider is MailProvider => provider !== null) +export const activeProviders: readonly MailProvider[] = emailFallback.providers diff --git a/apps/sim/lib/oauth/oauth.test.ts b/apps/sim/lib/oauth/oauth.test.ts index 6211943de41..9db573f208c 100644 --- a/apps/sim/lib/oauth/oauth.test.ts +++ b/apps/sim/lib/oauth/oauth.test.ts @@ -55,6 +55,9 @@ beforeAll(() => { WORDPRESS_CLIENT_SECRET: 'wordpress_client_secret', SPOTIFY_CLIENT_ID: 'spotify_client_id', SPOTIFY_CLIENT_SECRET: 'spotify_client_secret', + CALCOM_CLIENT_ID: 'calcom_client_id', + MONDAY_CLIENT_ID: 'monday_client_id', + MONDAY_CLIENT_SECRET: undefined, }) }) @@ -296,6 +299,26 @@ describe('OAuth Token Refresh', () => { expect(bodyParams.get('client_id')).toBeNull() }) + it.concurrent('should preserve Cal.com bearer refresh authentication', async () => { + const mockFetch = createMockFetch(defaultOAuthResponse) + const refreshToken = 'test_refresh_token' + + await withMockFetch(mockFetch, () => refreshOAuthToken('calcom', refreshToken)) + + const [endpoint, requestOptions] = mockFetch.mock.calls[0] as [ + string, + { headers: Record; body: string }, + ] + const bodyParams = new URLSearchParams(requestOptions.body) + + expect(endpoint).toBe('https://app.cal.com/api/auth/oauth/refreshToken') + expect(requestOptions.headers.Authorization).toBe(`Bearer ${refreshToken}`) + expect(bodyParams.get('grant_type')).toBe('refresh_token') + expect(bodyParams.get('client_id')).toBe('calcom_client_id') + expect(bodyParams.get('client_secret')).toBeNull() + expect(bodyParams.get('refresh_token')).toBeNull() + }) + it.concurrent('should send Notion request with Basic Auth header and JSON body', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) const refreshToken = 'test_refresh_token' @@ -351,6 +374,21 @@ describe('OAuth Token Refresh', () => { }) describe('Error Handling', () => { + it.concurrent('should return the canonical error for partial OAuth configuration', async () => { + const mockFetch = createMockFetch(defaultOAuthResponse) + + const result = await withMockFetch(mockFetch, () => + refreshOAuthToken('monday', 'test_refresh_token') + ) + + expect(result).toEqual({ + ok: false, + message: + 'OAuth client monday is partially configured — missing MONDAY_CLIENT_SECRET. Run bun run setup integration monday.', + }) + expect(mockFetch).not.toHaveBeenCalled() + }) + it.concurrent('should return failure for unsupported provider', async () => { const mockFetch = createMockFetch(defaultOAuthResponse) const refreshToken = 'test_refresh_token' diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index 2cd452358d0..116d8a5cde1 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -59,6 +59,11 @@ import { ZoomIcon, } from '@/components/icons' import { env } from '@/lib/core/config/env' +import { + type OAuthClientCapabilityField, + type OAuthClientCapabilityId, + requireOAuthClientCapability, +} from '@/lib/core/config/env-capabilities' import { isSlackExtendedScopesEnabled } from '@/lib/core/config/env-flags' import { DEFAULT_MAX_ERROR_BODY_BYTES, @@ -1249,22 +1254,28 @@ interface ProviderAuthConfig { clientIdParamName?: string } +function getConfiguredClientCredentials( + providerId: TCapabilityId, + clientIdField: NoInfer>, + clientSecretField?: NoInfer> +): Pick { + const { values } = requireOAuthClientCapability(providerId, env) + return { + clientId: values[clientIdField], + clientSecret: clientSecretField ? values[clientSecretField] : '', + } +} + /** * Get OAuth provider configuration for token refresh */ function getProviderAuthConfig(provider: string): ProviderAuthConfig { - const getCredentials = (clientId: string | undefined, clientSecret: string | undefined) => { - if (!clientId || !clientSecret) { - throw new Error(`Missing client credentials for provider: ${provider}`) - } - return { clientId, clientSecret } - } - switch (provider) { case 'google': { - const { clientId, clientSecret } = getCredentials( - env.GOOGLE_CLIENT_ID, - env.GOOGLE_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'google', + 'GOOGLE_CLIENT_ID', + 'GOOGLE_CLIENT_SECRET' ) return { tokenEndpoint: 'https://oauth2.googleapis.com/token', @@ -1274,7 +1285,11 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'x': { - const { clientId, clientSecret } = getCredentials(env.X_CLIENT_ID, env.X_CLIENT_SECRET) + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'x', + 'X_CLIENT_ID', + 'X_CLIENT_SECRET' + ) return { tokenEndpoint: 'https://api.x.com/2/oauth2/token', clientId, @@ -1284,9 +1299,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'tiktok': { - const { clientId, clientSecret } = getCredentials( - env.TIKTOK_CLIENT_ID, - env.TIKTOK_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'tiktok', + 'TIKTOK_CLIENT_ID', + 'TIKTOK_CLIENT_SECRET' ) return { tokenEndpoint: 'https://open.tiktokapis.com/v2/oauth/token/', @@ -1299,9 +1315,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'confluence': { - const { clientId, clientSecret } = getCredentials( - env.CONFLUENCE_CLIENT_ID, - env.CONFLUENCE_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'confluence', + 'CONFLUENCE_CLIENT_ID', + 'CONFLUENCE_CLIENT_SECRET' ) return { tokenEndpoint: 'https://auth.atlassian.com/oauth/token', @@ -1312,7 +1329,11 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'jira': { - const { clientId, clientSecret } = getCredentials(env.JIRA_CLIENT_ID, env.JIRA_CLIENT_SECRET) + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'jira', + 'JIRA_CLIENT_ID', + 'JIRA_CLIENT_SECRET' + ) return { tokenEndpoint: 'https://auth.atlassian.com/oauth/token', clientId, @@ -1322,14 +1343,14 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'calcom': { - const clientId = env.CALCOM_CLIENT_ID - if (!clientId) { - throw new Error('Missing CALCOM_CLIENT_ID') - } + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'calcom', + 'CALCOM_CLIENT_ID' + ) return { tokenEndpoint: 'https://app.cal.com/api/auth/oauth/refreshToken', clientId, - clientSecret: '', + clientSecret, useBasicAuth: false, supportsRefreshTokenRotation: true, // Cal.com requires refresh token in Authorization header, not body @@ -1337,9 +1358,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'airtable': { - const { clientId, clientSecret } = getCredentials( - env.AIRTABLE_CLIENT_ID, - env.AIRTABLE_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'airtable', + 'AIRTABLE_CLIENT_ID', + 'AIRTABLE_CLIENT_SECRET' ) return { tokenEndpoint: 'https://airtable.com/oauth2/v1/token', @@ -1350,9 +1372,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'notion': { - const { clientId, clientSecret } = getCredentials( - env.NOTION_CLIENT_ID, - env.NOTION_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'notion', + 'NOTION_CLIENT_ID', + 'NOTION_CLIENT_SECRET' ) return { tokenEndpoint: 'https://api.notion.com/v1/oauth/token', @@ -1367,9 +1390,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { case 'outlook': case 'onedrive': case 'sharepoint': { - const { clientId, clientSecret } = getCredentials( - env.MICROSOFT_CLIENT_ID, - env.MICROSOFT_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'microsoft', + 'MICROSOFT_CLIENT_ID', + 'MICROSOFT_CLIENT_SECRET' ) return { tokenEndpoint: 'https://login.microsoftonline.com/common/oauth2/v2.0/token', @@ -1380,9 +1404,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'clickup': { - const { clientId, clientSecret } = getCredentials( - env.CLICKUP_CLIENT_ID, - env.CLICKUP_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'clickup', + 'CLICKUP_CLIENT_ID', + 'CLICKUP_CLIENT_SECRET' ) return { tokenEndpoint: 'https://api.clickup.com/api/v2/oauth/token', @@ -1393,9 +1418,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'linear': { - const { clientId, clientSecret } = getCredentials( - env.LINEAR_CLIENT_ID, - env.LINEAR_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'linear', + 'LINEAR_CLIENT_ID', + 'LINEAR_CLIENT_SECRET' ) return { tokenEndpoint: 'https://api.linear.app/oauth/token', @@ -1406,9 +1432,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'attio': { - const { clientId, clientSecret } = getCredentials( - env.ATTIO_CLIENT_ID, - env.ATTIO_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'attio', + 'ATTIO_CLIENT_ID', + 'ATTIO_CLIENT_SECRET' ) return { tokenEndpoint: 'https://app.attio.com/oauth/token', @@ -1418,7 +1445,11 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'box': { - const { clientId, clientSecret } = getCredentials(env.BOX_CLIENT_ID, env.BOX_CLIENT_SECRET) + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'box', + 'BOX_CLIENT_ID', + 'BOX_CLIENT_SECRET' + ) return { tokenEndpoint: 'https://api.box.com/oauth2/token', clientId, @@ -1427,9 +1458,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'docusign': { - const { clientId, clientSecret } = getCredentials( - env.DOCUSIGN_CLIENT_ID, - env.DOCUSIGN_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'docusign', + 'DOCUSIGN_CLIENT_ID', + 'DOCUSIGN_CLIENT_SECRET' ) return { tokenEndpoint: 'https://account-d.docusign.com/oauth/token', @@ -1440,9 +1472,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'dropbox': { - const { clientId, clientSecret } = getCredentials( - env.DROPBOX_CLIENT_ID, - env.DROPBOX_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'dropbox', + 'DROPBOX_CLIENT_ID', + 'DROPBOX_CLIENT_SECRET' ) return { tokenEndpoint: 'https://api.dropboxapi.com/oauth2/token', @@ -1453,9 +1486,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'slack': { - const { clientId, clientSecret } = getCredentials( - env.SLACK_CLIENT_ID, - env.SLACK_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'slack', + 'SLACK_CLIENT_ID', + 'SLACK_CLIENT_SECRET' ) return { tokenEndpoint: 'https://slack.com/api/oauth.v2.access', @@ -1466,9 +1500,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'reddit': { - const { clientId, clientSecret } = getCredentials( - env.REDDIT_CLIENT_ID, - env.REDDIT_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'reddit', + 'REDDIT_CLIENT_ID', + 'REDDIT_CLIENT_SECRET' ) return { tokenEndpoint: 'https://www.reddit.com/api/v1/access_token', @@ -1481,9 +1516,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'wealthbox': { - const { clientId, clientSecret } = getCredentials( - env.WEALTHBOX_CLIENT_ID, - env.WEALTHBOX_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'wealthbox', + 'WEALTHBOX_CLIENT_ID', + 'WEALTHBOX_CLIENT_SECRET' ) return { tokenEndpoint: 'https://app.crmworkspace.com/oauth/token', @@ -1494,9 +1530,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'webflow': { - const { clientId, clientSecret } = getCredentials( - env.WEBFLOW_CLIENT_ID, - env.WEBFLOW_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'webflow', + 'WEBFLOW_CLIENT_ID', + 'WEBFLOW_CLIENT_SECRET' ) return { tokenEndpoint: 'https://api.webflow.com/oauth/access_token', @@ -1507,9 +1544,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'asana': { - const { clientId, clientSecret } = getCredentials( - env.ASANA_CLIENT_ID, - env.ASANA_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'asana', + 'ASANA_CLIENT_ID', + 'ASANA_CLIENT_SECRET' ) return { tokenEndpoint: 'https://app.asana.com/-/oauth_token', @@ -1520,9 +1558,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'pipedrive': { - const { clientId, clientSecret } = getCredentials( - env.PIPEDRIVE_CLIENT_ID, - env.PIPEDRIVE_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'pipedrive', + 'PIPEDRIVE_CLIENT_ID', + 'PIPEDRIVE_CLIENT_SECRET' ) return { tokenEndpoint: 'https://oauth.pipedrive.com/oauth/token', @@ -1533,9 +1572,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'hubspot': { - const { clientId, clientSecret } = getCredentials( - env.HUBSPOT_CLIENT_ID, - env.HUBSPOT_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'hubspot', + 'HUBSPOT_CLIENT_ID', + 'HUBSPOT_CLIENT_SECRET' ) return { tokenEndpoint: 'https://api.hubapi.com/oauth/v1/token', @@ -1546,9 +1586,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'linkedin': { - const { clientId, clientSecret } = getCredentials( - env.LINKEDIN_CLIENT_ID, - env.LINKEDIN_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'linkedin', + 'LINKEDIN_CLIENT_ID', + 'LINKEDIN_CLIENT_SECRET' ) return { tokenEndpoint: 'https://www.linkedin.com/oauth/v2/accessToken', @@ -1559,9 +1600,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'instagram': { - const { clientId, clientSecret } = getCredentials( - env.INSTAGRAM_CLIENT_ID, - env.INSTAGRAM_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'instagram', + 'INSTAGRAM_CLIENT_ID', + 'INSTAGRAM_CLIENT_SECRET' ) return { tokenEndpoint: 'https://graph.instagram.com/refresh_access_token', @@ -1573,9 +1615,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'salesforce': { - const { clientId, clientSecret } = getCredentials( - env.SALESFORCE_CLIENT_ID, - env.SALESFORCE_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'salesforce', + 'SALESFORCE_CLIENT_ID', + 'SALESFORCE_CLIENT_SECRET' ) return { tokenEndpoint: 'https://login.salesforce.com/services/oauth2/token', @@ -1588,9 +1631,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { case 'shopify': { // Shopify access tokens don't expire and don't support refresh tokens // This configuration is provided for completeness but won't be used for token refresh - const { clientId, clientSecret } = getCredentials( - env.SHOPIFY_CLIENT_ID, - env.SHOPIFY_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'shopify', + 'SHOPIFY_CLIENT_ID', + 'SHOPIFY_CLIENT_SECRET' ) return { tokenEndpoint: 'https://accounts.shopify.com/oauth/token', @@ -1601,7 +1645,11 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'zoom': { - const { clientId, clientSecret } = getCredentials(env.ZOOM_CLIENT_ID, env.ZOOM_CLIENT_SECRET) + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'zoom', + 'ZOOM_CLIENT_ID', + 'ZOOM_CLIENT_SECRET' + ) return { tokenEndpoint: 'https://zoom.us/oauth/token', clientId, @@ -1613,9 +1661,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { case 'wordpress': { // WordPress.com does NOT support refresh tokens // Users will need to re-authorize when tokens expire (~2 weeks) - const { clientId, clientSecret } = getCredentials( - env.WORDPRESS_CLIENT_ID, - env.WORDPRESS_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'wordpress', + 'WORDPRESS_CLIENT_ID', + 'WORDPRESS_CLIENT_SECRET' ) return { tokenEndpoint: 'https://public-api.wordpress.com/oauth2/token', @@ -1626,9 +1675,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'spotify': { - const { clientId, clientSecret } = getCredentials( - env.SPOTIFY_CLIENT_ID, - env.SPOTIFY_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'spotify', + 'SPOTIFY_CLIENT_ID', + 'SPOTIFY_CLIENT_SECRET' ) return { tokenEndpoint: 'https://accounts.spotify.com/api/token', @@ -1639,9 +1689,10 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { } } case 'monday': { - const { clientId, clientSecret } = getCredentials( - env.MONDAY_CLIENT_ID, - env.MONDAY_CLIENT_SECRET + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'monday', + 'MONDAY_CLIENT_ID', + 'MONDAY_CLIENT_SECRET' ) return { tokenEndpoint: 'https://auth.monday.com/oauth2/token', @@ -1657,7 +1708,11 @@ function getProviderAuthConfig(provider: string): ProviderAuthConfig { // The refresh must target the accounts server; a US/multi-DC-enabled client // uses accounts.zoho.com. Data residency for API calls is honored separately // via the persisted Desk base URL derived from the token response api_domain. - const { clientId, clientSecret } = getCredentials(env.ZOHO_CLIENT_ID, env.ZOHO_CLIENT_SECRET) + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'zoho-desk', + 'ZOHO_CLIENT_ID', + 'ZOHO_CLIENT_SECRET' + ) return { tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', clientId, diff --git a/apps/sim/lib/oauth/types.ts b/apps/sim/lib/oauth/types.ts index 4a036e17dbe..0e71b4c4ba2 100644 --- a/apps/sim/lib/oauth/types.ts +++ b/apps/sim/lib/oauth/types.ts @@ -168,10 +168,13 @@ export interface OAuthServiceConfig { * Service metadata without React components - safe for server-side use */ export interface OAuthServiceMetadata { + serviceId: string providerId: string + serviceAccountProviderId?: string name: string description: string baseProvider: string + authType: OAuthAuthType } export interface Credential { diff --git a/apps/sim/lib/oauth/utils.test.ts b/apps/sim/lib/oauth/utils.test.ts index acc0945270e..cc758eb90f2 100644 --- a/apps/sim/lib/oauth/utils.test.ts +++ b/apps/sim/lib/oauth/utils.test.ts @@ -25,14 +25,18 @@ describe('getAllOAuthServices', () => { services.forEach((service) => { expect(service).toHaveProperty('providerId') + expect(service).toHaveProperty('serviceId') expect(service).toHaveProperty('name') expect(service).toHaveProperty('description') expect(service).toHaveProperty('baseProvider') + expect(service).toHaveProperty('authType') expect(typeof service.providerId).toBe('string') + expect(typeof service.serviceId).toBe('string') expect(typeof service.name).toBe('string') expect(typeof service.description).toBe('string') expect(typeof service.baseProvider).toBe('string') + expect(['oauth', 'service_account']).toContain(service.authType) }) }) @@ -87,9 +91,24 @@ describe('getAllOAuthServices', () => { services.forEach((service) => { const metadata: OAuthServiceMetadata = service expect(metadata.providerId).toBeDefined() + expect(metadata.serviceId).toBeDefined() expect(metadata.name).toBeDefined() expect(metadata.description).toBeDefined() expect(metadata.baseProvider).toBeDefined() + expect(metadata.authType).toBeDefined() + }) + }) + + it.concurrent('preserves service-account auth metadata', () => { + const services = getAllOAuthServices() + + expect(services.find((service) => service.providerId === 'claude-platform')).toMatchObject({ + serviceId: 'claude-platform', + authType: 'service_account', + }) + expect(services.find((service) => service.providerId === 'google-email')).toMatchObject({ + serviceId: 'gmail', + authType: 'oauth', }) }) }) diff --git a/apps/sim/lib/oauth/utils.ts b/apps/sim/lib/oauth/utils.ts index b0df084e0c4..21ccaeadbd8 100644 --- a/apps/sim/lib/oauth/utils.ts +++ b/apps/sim/lib/oauth/utils.ts @@ -473,12 +473,15 @@ export function getAllOAuthServices(): OAuthServiceMetadata[] { const services: OAuthServiceMetadata[] = [] for (const [baseProviderId, provider] of Object.entries(OAUTH_PROVIDERS)) { - for (const service of Object.values(provider.services)) { + for (const [serviceId, service] of Object.entries(provider.services)) { services.push({ + serviceId, providerId: service.providerId, + serviceAccountProviderId: service.serviceAccountProviderId, name: service.name, description: service.description, baseProvider: baseProviderId, + authType: service.authType ?? 'oauth', }) } } diff --git a/apps/sim/lib/permission-groups/integration-allowlist.test.ts b/apps/sim/lib/permission-groups/integration-allowlist.test.ts new file mode 100644 index 00000000000..314bdf9c86e --- /dev/null +++ b/apps/sim/lib/permission-groups/integration-allowlist.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest' +import { intersectIntegrationAllowlists } from '@/lib/permission-groups/integration-allowlist' + +describe('intersectIntegrationAllowlists', () => { + it('uses the configured list when the other policy is unrestricted', () => { + expect(intersectIntegrationAllowlists(null, ['Slack'])).toEqual(['slack']) + expect(intersectIntegrationAllowlists(['Notion'], null)).toEqual(['notion']) + expect(intersectIntegrationAllowlists(null, null)).toBeNull() + }) + + it('keeps only integrations allowed by both policies', () => { + expect(intersectIntegrationAllowlists(['Slack', 'Notion'], ['notion', 'gmail'])).toEqual([ + 'notion', + ]) + }) + + it('preserves an explicit deny-all list', () => { + expect(intersectIntegrationAllowlists([], null)).toEqual([]) + expect(intersectIntegrationAllowlists(['slack'], [])).toEqual([]) + }) +}) diff --git a/apps/sim/lib/permission-groups/integration-allowlist.ts b/apps/sim/lib/permission-groups/integration-allowlist.ts new file mode 100644 index 00000000000..3656ee60f6d --- /dev/null +++ b/apps/sim/lib/permission-groups/integration-allowlist.ts @@ -0,0 +1,17 @@ +/** + * Intersects integration allowlists from independent policy layers. + * `null` means unrestricted, while an empty array denies every integration. + */ +export function intersectIntegrationAllowlists( + first: readonly string[] | null, + second: readonly string[] | null +): string[] | null { + const normalizedFirst = first?.map((integration) => integration.toLowerCase()) ?? null + const normalizedSecond = second?.map((integration) => integration.toLowerCase()) ?? null + + if (normalizedFirst === null) return normalizedSecond + if (normalizedSecond === null) return normalizedFirst + + const secondSet = new Set(normalizedSecond) + return normalizedFirst.filter((integration) => secondSet.has(integration)) +} diff --git a/apps/sim/lib/realtime/event-log.test.ts b/apps/sim/lib/realtime/event-log.test.ts index 20b8ab44cf8..b8159b108cf 100644 --- a/apps/sim/lib/realtime/event-log.test.ts +++ b/apps/sim/lib/realtime/event-log.test.ts @@ -3,7 +3,14 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -vi.mock('@/lib/core/config/env', () => ({ env: { REDIS_URL: undefined } })) +const { mockEnv } = vi.hoisted(() => ({ + mockEnv: { + REDIS_URL: undefined as string | undefined, + REDIS_TLS_SERVERNAME: undefined as string | undefined, + }, +})) + +vi.mock('@/lib/core/config/env', () => ({ env: mockEnv })) vi.mock('@/lib/core/config/redis', () => ({ getRedisClient: () => null })) import { @@ -32,7 +39,11 @@ function serializerFor(streamId: string, value: string) { } describe('event-log (memory fallback)', () => { - beforeEach(() => resetEventLogMemoryForTesting()) + beforeEach(() => { + mockEnv.REDIS_URL = undefined + mockEnv.REDIS_TLS_SERVERNAME = undefined + resetEventLogMemoryForTesting() + }) it('assigns monotonically increasing event ids', async () => { const first = await appendEvent(config, 's1', serializerFor('s1', 'a')) @@ -83,4 +94,22 @@ describe('event-log (memory fallback)', () => { const result = await readEventsSince(config, 'missing', 5) expect(result.status).toBe('pruned') }) + + it('does not use memory when Redis is selected but its client is unavailable', async () => { + mockEnv.REDIS_URL = 'redis://localhost:6379' + + await expect(appendEvent(config, 's1', serializerFor('s1', 'a'))).resolves.toBeNull() + await expect(readEventsSince(config, 's1', 0)).resolves.toEqual({ + status: 'unavailable', + error: 'Redis client unavailable', + }) + }) + + it('fails fast instead of using memory for an invalid Redis configuration', async () => { + mockEnv.REDIS_URL = 'https://cache.example.com' + + await expect(appendEvent(config, 's1', serializerFor('s1', 'a'))).rejects.toThrow( + /valid redis:\/\/ or rediss:\/\/ URL/ + ) + }) }) diff --git a/apps/sim/lib/realtime/event-log.ts b/apps/sim/lib/realtime/event-log.ts index f7b470c33a2..5916c4b6820 100644 --- a/apps/sim/lib/realtime/event-log.ts +++ b/apps/sim/lib/realtime/event-log.ts @@ -16,7 +16,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { env } from '@/lib/core/config/env' +import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server' import { getRedisClient } from '@/lib/core/config/redis' const logger = createLogger('EventLog') @@ -111,7 +111,7 @@ function memoryKey(config: EventLogConfig, streamId: string) { } function canUseMemoryBuffer(): boolean { - return typeof window === 'undefined' && !env.REDIS_URL + return typeof window === 'undefined' && getConfiguredCacheProvider() === 'database' } function pruneExpiredMemoryStreams(now = Date.now()): void { diff --git a/apps/sim/lib/uploads/config.ts b/apps/sim/lib/uploads/config.ts index 10cb9a2eff7..e134d57ae2e 100644 --- a/apps/sim/lib/uploads/config.ts +++ b/apps/sim/lib/uploads/config.ts @@ -1,20 +1,17 @@ import { env, envBoolean } from '@/lib/core/config/env' +import { getConfiguredStorageProviderId } from '@/lib/core/config/env-capabilities.server' import type { StorageConfig, StorageContext } from '@/lib/uploads/shared/types' export type { StorageConfig, StorageContext } from '@/lib/uploads/shared/types' export const UPLOAD_DIR = '/uploads' -const hasS3Config = !!(env.S3_BUCKET_NAME && env.AWS_REGION) -export const hasBlobConfig = !!( - env.AZURE_STORAGE_CONTAINER_NAME && - ((env.AZURE_ACCOUNT_NAME && env.AZURE_ACCOUNT_KEY) || env.AZURE_CONNECTION_STRING) -) -const hasGcsConfig = !!env.GCS_BUCKET_NAME +const storageProvider = getConfiguredStorageProviderId() -export const USE_BLOB_STORAGE = hasBlobConfig -export const USE_S3_STORAGE = hasS3Config && !USE_BLOB_STORAGE -export const USE_GCS_STORAGE = hasGcsConfig && !USE_BLOB_STORAGE && !USE_S3_STORAGE +export const hasBlobConfig = storageProvider === 'azure' +export const USE_BLOB_STORAGE = storageProvider === 'azure' +export const USE_S3_STORAGE = storageProvider === 's3' +export const USE_GCS_STORAGE = storageProvider === 'gcs' export const S3_CONFIG = { bucket: env.S3_BUCKET_NAME || '', diff --git a/helm/sim/Chart.yaml b/helm/sim/Chart.yaml index dc46c83540b..371e645be86 100644 --- a/helm/sim/Chart.yaml +++ b/helm/sim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: sim description: A Helm chart for Sim - the open-source AI workspace where teams build, deploy, and manage AI agents type: application -version: 1.5.0 +version: 1.5.1 appVersion: "v0.7.44" kubeVersion: ">=1.25.0-0" home: https://sim.ai diff --git a/helm/sim/examples/values-aws.yaml b/helm/sim/examples/values-aws.yaml index aed3afe592d..011b9d92f0a 100644 --- a/helm/sim/examples/values-aws.yaml +++ b/helm/sim/examples/values-aws.yaml @@ -94,6 +94,7 @@ app: # AWS S3 Cloud Storage Configuration (RECOMMENDED for production) # Create S3 buckets in your AWS account and configure IAM permissions + STORAGE_PROVIDER: "s3" AWS_REGION: "us-west-2" AWS_ACCESS_KEY_ID: "" # AWS access key (or use IRSA for EKS) AWS_SECRET_ACCESS_KEY: "" # AWS secret key (or use IRSA for EKS) diff --git a/helm/sim/examples/values-azure.yaml b/helm/sim/examples/values-azure.yaml index ccd7c19c47f..99e58645f39 100644 --- a/helm/sim/examples/values-azure.yaml +++ b/helm/sim/examples/values-azure.yaml @@ -111,6 +111,7 @@ app: # Azure Blob Storage Configuration (RECOMMENDED for production) # Create a storage account and containers in your Azure subscription + STORAGE_PROVIDER: "azure" AZURE_ACCOUNT_NAME: "simstudiostorageacct" # Azure storage account name AZURE_ACCOUNT_KEY: "" # Storage account access key # Or use connection string instead of account name/key: diff --git a/helm/sim/examples/values-gcp.yaml b/helm/sim/examples/values-gcp.yaml index 590d95e1deb..52523221820 100644 --- a/helm/sim/examples/values-gcp.yaml +++ b/helm/sim/examples/values-gcp.yaml @@ -92,6 +92,7 @@ app: # without a key file additionally requires roles/iam.serviceAccountTokenCreator # on the service account itself (or set GCS_CREDENTIALS_JSON to inline # service-account JSON with a private key). + STORAGE_PROVIDER: "gcs" GCS_PROJECT_ID: "your-project-id" GCS_BUCKET_NAME: "myorg-sim-workspace-files" # Workspace files (enables GCS) GCS_KB_BUCKET_NAME: "myorg-sim-knowledge-base" # Knowledge base documents diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 98ce4199450..43686a3715f 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -151,6 +151,7 @@ app: WAND_OPENAI_MODEL_NAME: "" # Wand generation model deployment name (works with both regular OpenAI and Azure OpenAI) # Azure Mistral OCR Configuration (leave empty if not using Azure-hosted OCR for document processing) + OCR_PROVIDER: "" # One of: local, mistral, azure-mistral. Empty preserves legacy credential inference OCR_AZURE_ENDPOINT: "" # Azure Mistral OCR service endpoint OCR_AZURE_MODEL_NAME: "" # Azure Mistral OCR model name OCR_AZURE_API_KEY: "" # Azure Mistral OCR API key @@ -269,8 +270,11 @@ app: # hidden in the Knowledge block UI. NEXT_PUBLIC_COHERE_CONFIGURED: "" # Set to "true" to hide the Cohere API Key field on the Knowledge block + # Optional file storage override. One of: local, s3, azure, gcs. + # Empty preserves legacy Azure → S3 → GCS → local precedence. + STORAGE_PROVIDER: "" + # AWS S3 Cloud Storage Configuration (optional - for file storage) - # If configured, files will be stored in S3 instead of local storage AWS_REGION: "" # AWS region (e.g., "us-east-1") AWS_ACCESS_KEY_ID: "" # AWS access key ID AWS_SECRET_ACCESS_KEY: "" # AWS secret access key @@ -286,8 +290,6 @@ app: S3_FORCE_PATH_STYLE: "" # Set to "true" for path-style addressing (MinIO/Ceph RGW). Leave empty for AWS S3 and R2 # Azure Blob Storage Configuration (optional - for file storage) - # If configured, files will be stored in Azure Blob instead of local storage - # Note: Azure Blob takes precedence over S3 if both are configured AZURE_ACCOUNT_NAME: "" # Azure storage account name AZURE_ACCOUNT_KEY: "" # Azure storage account key AZURE_CONNECTION_STRING: "" # Azure connection string (alternative to account name/key) @@ -301,8 +303,6 @@ app: AZURE_STORAGE_WORKSPACE_LOGOS_CONTAINER_NAME: "" # Azure container for workspace logos # Google Cloud Storage Configuration (optional - for file storage) - # If configured, files will be stored in GCS instead of local storage - # Note: used when neither Azure Blob nor S3 is configured GCS_PROJECT_ID: "" # GCP project ID (optional — inferred from credentials/ADC when unset) GCS_CREDENTIALS_JSON: "" # Inline service-account JSON. Leave empty to use Workload Identity / Application Default Credentials GCS_BUCKET_NAME: "" # GCS bucket for workspace files (all other GCS buckets fall back to it) diff --git a/package.json b/package.json index 6dabbe3baf3..19112cb17ce 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "packages/*" ], "bin": { - "sim": "./scripts/setup/index.ts" + "sim": "./scripts/setup/launcher.ts" }, "scripts": { "build": "turbo run build", @@ -17,7 +17,8 @@ "dev:sockets": "cd apps/realtime && bun run dev", "dev:full": "bunx concurrently -n \"App,Realtime\" -c \"cyan,magenta\" \"cd apps/sim && bun run dev\" \"cd apps/realtime && bun run dev\"", "dev:full:capped": "bunx concurrently -n \"App,Realtime\" -c \"cyan,magenta\" \"cd apps/sim && bun run dev:capped\" \"cd apps/realtime && bun run dev\"", - "test": "turbo run test", + "test": "bun run test:setup && turbo run test", + "test:setup": "bun test scripts/setup", "format": "turbo run format", "format:check": "turbo run format:check", "lint": "turbo run lint", @@ -47,6 +48,7 @@ "billing-protocol-contract:check": "bun run scripts/sync-billing-protocol-contract.ts --check", "tool-metadata:generate": "bun run scripts/sync-tool-metadata.ts", "tool-metadata:check": "bun run scripts/sync-tool-metadata.ts --check", + "integration-catalog:check": "bun run scripts/check-integration-catalog.ts", "mship-tools:generate": "bun run scripts/sync-tool-catalog.ts", "mship-tools:check": "bun run scripts/sync-tool-catalog.ts --check", "trace-spans-contract:generate": "bun run scripts/sync-trace-spans-contract.ts", @@ -67,9 +69,9 @@ "library:covers:check": "bun run scripts/generate-library-covers.tsx --check", "skills:sync": "bun run scripts/sync-skills.ts", "skills:check": "bun run scripts/sync-skills.ts --check", - "setup": "bun install && bun run scripts/setup/index.ts setup", - "sim": "bun install && bun run scripts/setup/index.ts", - "doctor": "bun run scripts/setup/index.ts doctor", + "setup": "bun run scripts/setup/launcher.ts setup", + "sim": "bun run scripts/setup/launcher.ts", + "doctor": "bun run scripts/setup/launcher.ts doctor", "agent-stream-docs:generate": "bun run scripts/sync-agent-stream-docs.ts", "agent-stream-docs:check": "bun run scripts/sync-agent-stream-docs.ts --check", "prepare": "bun husky", diff --git a/packages/testing/src/mocks/env.mock.test.ts b/packages/testing/src/mocks/env.mock.test.ts index 0270c23220e..67744f8c1a1 100644 --- a/packages/testing/src/mocks/env.mock.test.ts +++ b/packages/testing/src/mocks/env.mock.test.ts @@ -36,6 +36,16 @@ describe('env mock', () => { expect(envMock.getEnv('SOME_UNPINNED_TEST_VAR')).toBe('from-process-env') }) + it('does not inherit process.env for pinned capability defaults', () => { + vi.stubEnv('REDIS_URL', 'redis://localhost:6379') + vi.stubEnv('STORAGE_PROVIDER', 's3') + resetEnvMock() + expect(envMock.env.REDIS_URL).toBeUndefined() + expect(envMock.getEnv('REDIS_URL')).toBeUndefined() + expect(envMock.env.STORAGE_PROVIDER).toBe('local') + expect(envMock.getEnv('STORAGE_PROVIDER')).toBe('local') + }) + it('pins explicitly-undefined overrides without process.env fallback', () => { vi.stubEnv('NEXT_PUBLIC_APP_URL', 'https://shadowed.example.com') setEnv({ NEXT_PUBLIC_APP_URL: undefined }) diff --git a/packages/testing/src/mocks/env.mock.ts b/packages/testing/src/mocks/env.mock.ts index 26368b020f1..d4b5a274483 100644 --- a/packages/testing/src/mocks/env.mock.ts +++ b/packages/testing/src/mocks/env.mock.ts @@ -25,6 +25,12 @@ export const defaultMockEnv = { EMAIL_DOMAIN: 'test.sim.ai', PERSONAL_EMAIL_FROM: 'Test ', + // Cache + REDIS_URL: undefined, + + // Storage + STORAGE_PROVIDER: 'local', + // URLs NEXT_PUBLIC_APP_URL: 'https://test.sim.ai', } diff --git a/packages/testing/src/mocks/redis-config.mock.test.ts b/packages/testing/src/mocks/redis-config.mock.test.ts index 65f86acf315..0db6bbdd54a 100644 --- a/packages/testing/src/mocks/redis-config.mock.test.ts +++ b/packages/testing/src/mocks/redis-config.mock.test.ts @@ -7,6 +7,7 @@ describe('redis-config mock', () => { }) it('defaults to the Redis-unavailable behavior of the real module', async () => { + expect(redisConfigMock.getConfiguredRedisUrl()).toBeNull() expect(redisConfigMock.getRedisClient()).toBeNull() await expect(redisConfigMock.acquireLock('k', 'v', 10)).resolves.toBe(true) await expect(redisConfigMock.releaseLock('k', 'v')).resolves.toBe(true) @@ -24,12 +25,15 @@ describe('redis-config mock', () => { it('resetRedisConfigMock restores defaults after overrides', async () => { const fakeClient = { ping: () => 'PONG' } + redisConfigMockFns.mockGetConfiguredRedisUrl.mockReturnValue('redis://localhost:6379') redisConfigMockFns.mockGetRedisClient.mockReturnValue(fakeClient) redisConfigMockFns.mockAcquireLock.mockResolvedValue(false) + expect(redisConfigMock.getConfiguredRedisUrl()).toBe('redis://localhost:6379') expect(redisConfigMock.getRedisClient()).toBe(fakeClient) await expect(redisConfigMock.acquireLock('k', 'v', 10)).resolves.toBe(false) resetRedisConfigMock() + expect(redisConfigMock.getConfiguredRedisUrl()).toBeNull() expect(redisConfigMock.getRedisClient()).toBeNull() await expect(redisConfigMock.acquireLock('k', 'v', 10)).resolves.toBe(true) }) diff --git a/packages/testing/src/mocks/redis-config.mock.ts b/packages/testing/src/mocks/redis-config.mock.ts index 48d4fc9e01e..82c9a4d88d2 100644 --- a/packages/testing/src/mocks/redis-config.mock.ts +++ b/packages/testing/src/mocks/redis-config.mock.ts @@ -45,8 +45,9 @@ function getRedisConnectionDefaultsImpl(url?: string): { /** * Controllable mock functions for `@/lib/core/config/redis`. - * Default: `getRedisClient` returns `null` (tests that need a client override - * it), matching the real module's behavior when `REDIS_URL` is unset. + * Default: `getConfiguredRedisUrl` and `getRedisClient` return `null` (tests + * that need Redis override them), matching the real module's database-cache + * behavior. * `acquireLock`/`releaseLock`/`extendLock` default to succeeding (`true`), * matching the real module's Redis-unavailable no-op path. * {@link resetRedisConfigMock} restores the default behaviors. @@ -59,6 +60,7 @@ function getRedisConnectionDefaultsImpl(url?: string): { * ``` */ export const redisConfigMockFns = { + mockGetConfiguredRedisUrl: vi.fn().mockReturnValue(null), mockGetRedisClient: vi.fn().mockReturnValue(null), mockGetRedisConnectionDefaults: vi.fn(getRedisConnectionDefaultsImpl), mockOnRedisReconnect: vi.fn(), @@ -73,6 +75,7 @@ export const redisConfigMockFns = { * Restores every redis-config mock function to its default behavior. */ export function resetRedisConfigMock(): void { + redisConfigMockFns.mockGetConfiguredRedisUrl.mockReset().mockReturnValue(null) redisConfigMockFns.mockGetRedisClient.mockReset().mockReturnValue(null) redisConfigMockFns.mockGetRedisConnectionDefaults .mockReset() @@ -95,6 +98,7 @@ export function resetRedisConfigMock(): void { * ``` */ export const redisConfigMock = { + getConfiguredRedisUrl: redisConfigMockFns.mockGetConfiguredRedisUrl, getRedisClient: redisConfigMockFns.mockGetRedisClient, getRedisConnectionDefaults: redisConfigMockFns.mockGetRedisConnectionDefaults, onRedisReconnect: redisConfigMockFns.mockOnRedisReconnect, diff --git a/scripts/check-integration-catalog.ts b/scripts/check-integration-catalog.ts new file mode 100644 index 00000000000..2524f70c010 --- /dev/null +++ b/scripts/check-integration-catalog.ts @@ -0,0 +1,138 @@ +#!/usr/bin/env bun +import { stripVersionSuffix } from '@sim/utils/string' +/** + * Verifies the registry-free integration catalog matches the executable block + * registry fields that deployment availability depends on. + */ +import { BLOCK_REGISTRY } from '../apps/sim/blocks/registry-maps' +import { AuthMode, type BlockConfig } from '../apps/sim/blocks/types' +import integrationsJson from '../apps/sim/lib/integrations/integrations.json' + +type CatalogAuthType = 'oauth' | 'api-key' | 'none' + +interface CatalogEntry { + type: string + slug: string + name: string + category: string + integrationType: string + authType: CatalogAuthType + oauthServiceId?: string +} + +function resolveAuthType(block: BlockConfig): CatalogAuthType { + if (block.authMode === AuthMode.OAuth) return 'oauth' + if (block.authMode === AuthMode.ApiKey || block.authMode === AuthMode.BotToken) return 'api-key' + if (block.subBlocks.some((subBlock) => subBlock.type === 'oauth-input')) return 'oauth' + if ( + block.subBlocks.some((subBlock) => ['apiKey', 'api_key', 'accessToken'].includes(subBlock.id)) + ) { + return 'api-key' + } + return 'none' +} + +function resolveOAuthServiceId(block: BlockConfig): string | undefined { + const serviceIds = new Set( + block.subBlocks + .filter((subBlock) => subBlock.type === 'oauth-input') + .map((subBlock) => subBlock.serviceId) + .filter((serviceId): serviceId is string => Boolean(serviceId)) + ) + if (serviceIds.size > 1) { + throw new Error( + `Integration block "${block.type}" declares more than one OAuth service ID: ${[...serviceIds].join(', ')}` + ) + } + return serviceIds.values().next().value +} + +function expectedEntry(block: BlockConfig): CatalogEntry { + if (!block.integrationType) { + throw new Error(`Integration block "${block.type}" is missing integrationType`) + } + const authType = resolveAuthType(block) + const oauthServiceId = authType === 'oauth' ? resolveOAuthServiceId(block) : undefined + if (authType === 'oauth' && !oauthServiceId) { + throw new Error(`OAuth integration block "${block.type}" is missing an OAuth service ID`) + } + return { + type: block.type, + slug: block.name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''), + name: block.name, + category: block.category, + integrationType: block.integrationType, + authType, + ...(oauthServiceId ? { oauthServiceId } : {}), + } +} + +function verifyIntegrationCatalog(): void { + const expected = Object.values(BLOCK_REGISTRY).filter( + (block) => block.category === 'tools' && !block.hideFromToolbar && !block.preview + ) + const expectedBaseTypes = new Map() + for (const block of expected) { + const baseType = stripVersionSuffix(block.type) + const existing = expectedBaseTypes.get(baseType) + if (existing) { + throw new Error( + `Visible integration blocks "${existing}" and "${block.type}" share base type "${baseType}"` + ) + } + expectedBaseTypes.set(baseType, block.type) + } + + const actual = integrationsJson.integrations as readonly CatalogEntry[] + const expectedByType = new Map(expected.map((block) => [block.type, expectedEntry(block)])) + const actualByType = new Map() + const actualSlugs = new Set() + for (const entry of actual) { + if (actualByType.has(entry.type)) { + throw new Error(`Generated integration catalog contains duplicate type "${entry.type}"`) + } + if (actualSlugs.has(entry.slug)) { + throw new Error(`Generated integration catalog contains duplicate slug "${entry.slug}"`) + } + actualByType.set(entry.type, entry) + actualSlugs.add(entry.slug) + } + + const issues: string[] = [] + for (const [type, entry] of expectedByType) { + const generated = actualByType.get(type) + if (!generated) { + issues.push(`missing generated entry for "${type}"`) + continue + } + for (const field of [ + 'name', + 'slug', + 'category', + 'integrationType', + 'authType', + 'oauthServiceId', + ] as const) { + if (generated[field] !== entry[field]) { + issues.push(`"${type}" has stale ${field}`) + } + } + } + for (const type of actualByType.keys()) { + if (!expectedByType.has(type)) issues.push(`unexpected generated entry for "${type}"`) + } + + if (issues.length > 0) { + throw new Error( + `Generated integration catalog is stale:\n- ${issues.join('\n- ')}\nRun \`bun run scripts/generate-docs.ts\` and commit the generated catalog.` + ) + } + process.stdout.write( + `Integration deployment metadata is in sync (${actual.length} integrations).\n` + ) +} + +verifyIntegrationCatalog() diff --git a/scripts/setup/capability-config.test.ts b/scripts/setup/capability-config.test.ts new file mode 100644 index 00000000000..9e9a37f8d99 --- /dev/null +++ b/scripts/setup/capability-config.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from 'bun:test' +import { + defineCapability, + ENV_CAPABILITIES, + envField, + OAUTH_CLIENT_CAPABILITIES, +} from '../../apps/sim/lib/core/config/env-capabilities.ts' +import { + CAPABILITY_SETUPS, + defineCapabilitySetup, + EMAIL_SETUP, + getOAuthClientSetupFields, + STORAGE_SETUP, +} from './capability-config.ts' +import { getCapabilitySetupOptions } from './capability-setup.ts' + +describe('capability setup configuration', () => { + it('maps every runtime capability and provider exactly once', () => { + expect(CAPABILITY_SETUPS.map((setup) => setup.definition.id)).toEqual( + ENV_CAPABILITIES.map((capability) => capability.id) + ) + for (const setup of CAPABILITY_SETUPS) { + expect(Object.keys(setup.providers).sort()).toEqual( + setup.definition.providers.map((provider) => provider.id).sort() + ) + } + }) + + it('fails fast when CLI prompts omit a runtime-owned provider field', () => { + const definition = defineCapability({ + strategy: 'fallback', + id: 'sample', + label: 'Sample', + providers: [ + { + id: 'remote', + label: 'Remote', + activation: { mode: 'any-present', keys: ['REMOTE_KEY'] }, + requires: envField('REMOTE_KEY'), + }, + ], + } as const) + + expect(() => + defineCapabilitySetup(definition, { + label: 'Sample', + message: 'Sample provider?', + actions: {}, + providers: { remote: { prompts: [] } }, + optionOrder: ['remote'], + } as never) + ).toThrow(/missing: REMOTE_KEY/) + + expect(() => + defineCapabilitySetup(definition, { + label: 'Sample', + message: 'Sample provider?', + actions: {}, + providers: { + remote: { + env: { MISSPELLED_REMOTE_KEY: 'true' }, + prompts: [{ type: 'field', key: 'REMOTE_KEY', input: 'secret' }], + }, + }, + optionOrder: ['remote'], + }) + ).toThrow(/unknown: MISSPELLED_REMOTE_KEY/) + }) + + it('keeps presets out of the generic provider choices', () => { + expect(getCapabilitySetupOptions(EMAIL_SETUP).map((option) => option.id)).not.toContain( + 'mailhog' + ) + expect(getCapabilitySetupOptions(STORAGE_SETUP).map((option) => option.id)).not.toContain( + 's3-compatible' + ) + }) + + it('maps every OAuth runtime field to a CLI input mode in runtime order', () => { + for (const id of Object.keys(OAUTH_CLIENT_CAPABILITIES) as Array< + keyof typeof OAUTH_CLIENT_CAPABILITIES + >) { + expect(getOAuthClientSetupFields(id).map((field) => field.key)).toEqual( + OAUTH_CLIENT_CAPABILITIES[id] + ) + } + }) +}) diff --git a/scripts/setup/capability-config.ts b/scripts/setup/capability-config.ts new file mode 100644 index 00000000000..fdb0f5330d8 --- /dev/null +++ b/scripts/setup/capability-config.ts @@ -0,0 +1,923 @@ +/** + * CLI-only labels, hints, prompts, and actions for runtime deployment capabilities. + * Provider IDs and environment fields are checked against the application catalog at load time. + * + * @packageDocumentation + */ +import { + ASYNC_JOBS_CAPABILITY, + CACHE_CAPABILITY, + type CapabilityDefinition, + EMAIL_CAPABILITY, + ENV_CAPABILITIES, + type EnvCapabilityValues, + getCapabilityFields, + hasEnvCapabilityValue, + OAUTH_CLIENT_CAPABILITIES, + type OAuthClientCapabilityField, + type OAuthClientCapabilityId, + OCR_CAPABILITY, + SANDBOX_CAPABILITY, + STORAGE_CAPABILITY, +} from '../../apps/sim/lib/core/config/env-capabilities.ts' + +export type SetupHint = + | string + | { + development: string + containerized: string + } + +export type SetupCondition = + | { kind: 'present'; key: string } + | { kind: 'truthy'; key: string } + | { kind: 'equals'; key: string; value: string } + | { kind: 'all'; conditions: readonly SetupCondition[] } + | { kind: 'any'; conditions: readonly SetupCondition[] } + | { kind: 'not'; condition: SetupCondition } + +export type SetupPromptCondition = SetupCondition | { kind: 'provider-missing-field'; key: string } + +export interface SetupFieldPrompt { + type: 'field' + key: string + input: 'text' | 'secret' + message?: string + hint?: string + required?: boolean + defaultValue?: string + validate?: boolean + when?: SetupPromptCondition +} + +export interface SetupConfirmPrompt { + type: 'confirm' + key: string + message: string + defaultValue?: boolean + when?: SetupPromptCondition +} + +export interface SetupChoiceOption { + id: string + label: string + hint?: string + currentWhen?: SetupCondition + env?: Readonly> + prompts?: readonly SetupPrompt[] +} + +export interface SetupChoicePrompt { + type: 'choice' + id: string + message: string + options: readonly SetupChoiceOption[] + when?: SetupPromptCondition +} + +export type SetupPrompt = SetupFieldPrompt | SetupConfirmPrompt | SetupChoicePrompt + +type ProviderId = TDefinition['providers'][number]['id'] + +export interface ProviderSetupDefinition { + hint?: SetupHint + env?: Readonly> + prompts: readonly SetupPrompt[] + currentWhen?: SetupCondition +} + +type ProviderSetupMap = { + [TId in ProviderId]: ProviderSetupDefinition +} + +export interface SetupActionDefinition { + label: string + hint?: SetupHint + env?: Readonly> + currentWhen?: SetupCondition +} + +type DefaultOptionId = TDefinition extends { + defaultProvider: { kind: 'built-in'; id: infer TId extends string } +} + ? TId + : never + +export interface CapabilitySetupDefinition< + TDefinition extends CapabilityDefinition = CapabilityDefinition, + TActions extends Readonly> = Readonly< + Record + >, +> { + definition: TDefinition + label: string + message: string + providers: ProviderSetupMap + actions: TActions + defaultOption?: { hint?: SetupHint } + optionOrder: readonly (ProviderId | DefaultOptionId | keyof TActions)[] +} + +function promptKeys(prompts: readonly SetupPrompt[] = []): string[] { + return prompts.flatMap((prompt) => { + if (prompt.type === 'field' || prompt.type === 'confirm') return [prompt.key] + return prompt.options.flatMap((option) => [ + ...Object.keys(option.env ?? {}), + ...promptKeys(option.prompts), + ]) + }) +} + +function conditionKeys(condition: SetupPromptCondition | undefined): string[] { + if (!condition) return [] + if ( + condition.kind === 'present' || + condition.kind === 'truthy' || + condition.kind === 'equals' || + condition.kind === 'provider-missing-field' + ) { + return [condition.key] + } + if (condition.kind === 'all' || condition.kind === 'any') { + return condition.conditions.flatMap(conditionKeys) + } + return conditionKeys(condition.condition) +} + +function promptConditionKeys(prompts: readonly SetupPrompt[] = []): string[] { + return prompts.flatMap((prompt) => [ + ...conditionKeys(prompt.when), + ...(prompt.type === 'choice' + ? prompt.options.flatMap((option) => [ + ...conditionKeys(option.currentWhen), + ...promptConditionKeys(option.prompts), + ]) + : []), + ]) +} + +function requirementKeys( + requirement: CapabilityDefinition['providers'][number]['requires'] +): string[] { + return requirement.type === 'field' + ? [requirement.key] + : requirement.requirements.flatMap(requirementKeys) +} + +function providerOwnedInputKeys( + provider: CapabilityDefinition['providers'][number] +): readonly string[] { + return [ + ...requirementKeys(provider.requires), + ...(provider.optionalFields ?? []).map((field) => field.key), + ...(provider.pairedFields ?? []).flat(), + ] +} + +function providerOwnedSetupKeys( + provider: CapabilityDefinition['providers'][number] +): readonly string[] { + return [ + ...providerOwnedInputKeys(provider), + ...(provider.activation.mode === 'enabled' + ? [provider.activation.key] + : provider.activation.keys), + ] +} + +export function defineCapabilitySetup< + const TDefinition extends CapabilityDefinition, + const TActions extends Readonly>, +>( + definition: TDefinition, + setup: Omit, 'definition'> +): CapabilitySetupDefinition { + const providerIds = definition.providers.map((provider) => provider.id) + const configuredProviderIds = Object.keys(setup.providers) + const missingProviders = providerIds.filter((id) => !configuredProviderIds.includes(id)) + const unknownProviders = configuredProviderIds.filter((id) => !providerIds.includes(id)) + if (missingProviders.length > 0 || unknownProviders.length > 0) { + throw new Error( + `Setup ${definition.id} provider mapping drifted (missing: ${missingProviders.join(', ') || 'none'}; unknown: ${unknownProviders.join(', ') || 'none'})` + ) + } + + for (const provider of definition.providers) { + const providerSetup = (setup.providers as Record)[provider.id] + if (!providerSetup) throw new Error(`Setup ${definition.id} has no provider ${provider.id}`) + const ownedFields = providerOwnedInputKeys(provider) + const setupFields = providerOwnedSetupKeys(provider) + const configuredFields = [ + ...promptKeys(providerSetup.prompts), + ...Object.keys(providerSetup.env ?? {}), + ] + const referencedFields = [ + ...configuredFields, + ...conditionKeys(providerSetup.currentWhen), + ...promptConditionKeys(providerSetup.prompts), + ] + const unknownFields = referencedFields.filter((key) => !setupFields.includes(key)) + const missingFields = ownedFields.filter((key) => !configuredFields.includes(key)) + if (unknownFields.length > 0 || missingFields.length > 0) { + throw new Error( + `Setup ${definition.id}/${provider.id} field mapping drifted (missing: ${missingFields.join(', ') || 'none'}; unknown: ${unknownFields.join(', ') || 'none'})` + ) + } + } + + const capabilityFields = getCapabilityFields(definition) + for (const [actionId, action] of Object.entries(setup.actions)) { + const unknownFields = [ + ...Object.keys(action.env ?? {}), + ...conditionKeys(action.currentWhen), + ].filter((key) => !capabilityFields.includes(key)) + if (unknownFields.length > 0) { + throw new Error( + `Setup ${definition.id}/${actionId} action writes unknown fields: ${unknownFields.join(', ')}` + ) + } + } + + const optionIds = [ + ...providerIds, + ...Object.keys(setup.actions), + ...(definition.strategy === 'selected' && definition.defaultProvider.kind === 'built-in' + ? [definition.defaultProvider.id] + : []), + ] + if ( + setup.optionOrder.length !== optionIds.length || + setup.optionOrder.some((id) => !optionIds.includes(String(id))) || + optionIds.some((id) => !setup.optionOrder.includes(id)) + ) { + throw new Error(`Setup ${definition.id} option order must list every option once`) + } + + return { definition, ...setup } +} + +export const EMAIL_SETUP = defineCapabilitySetup(EMAIL_CAPABILITY, { + label: 'Email delivery', + message: 'Email sending?', + actions: { + none: { + label: 'None', + hint: 'emails are logged to the console — fine for local', + }, + }, + providers: { + resend: { + hint: 'paste an API key', + prompts: [ + { + type: 'field', + key: 'RESEND_API_KEY', + input: 'secret', + required: true, + }, + ], + }, + ses: { + hint: 'uses the AWS SDK credential chain', + prompts: [ + { + type: 'field', + key: 'AWS_SES_REGION', + input: 'text', + required: true, + defaultValue: 'us-east-1', + }, + ], + }, + smtp: { + hint: 'any SMTP relay', + prompts: [ + { type: 'field', key: 'SMTP_HOST', input: 'text', required: true }, + { + type: 'field', + key: 'SMTP_PORT', + input: 'text', + required: true, + defaultValue: '587', + validate: true, + }, + { + type: 'field', + key: 'SMTP_USER', + input: 'text', + hint: 'leave empty for an unauthenticated relay', + }, + { + type: 'field', + key: 'SMTP_PASS', + input: 'secret', + required: true, + when: { kind: 'present', key: 'SMTP_USER' }, + }, + ], + }, + azure: { + hint: 'connection string', + prompts: [ + { + type: 'field', + key: 'AZURE_ACS_CONNECTION_STRING', + input: 'secret', + required: true, + }, + ], + }, + gmail: { + hint: 'Workspace service account delegation', + prompts: [ + { + type: 'field', + key: 'GMAIL_CREDENTIALS_JSON', + input: 'secret', + required: true, + validate: true, + }, + { type: 'field', key: 'GMAIL_SENDER', input: 'text', required: true }, + ], + }, + }, + optionOrder: ['none', 'resend', 'ses', 'smtp', 'azure', 'gmail'], +}) + +export const STORAGE_SETUP = defineCapabilitySetup(STORAGE_CAPABILITY, { + label: 'File storage', + message: 'File storage?', + actions: {}, + defaultOption: { + hint: { + development: + 'fine for local dev (external-fetch flows like Instagram publish need cloud storage)', + containerized: 'files live in the container — LOST on restart; evaluation only', + }, + }, + providers: { + azure: { + hint: 'connection string or account name + key', + prompts: [ + { + type: 'field', + key: 'AZURE_STORAGE_CONTAINER_NAME', + input: 'text', + required: true, + defaultValue: 'sim-files', + }, + { + type: 'choice', + id: 'azure-credentials', + message: 'Azure credentials?', + options: [ + { + id: 'connection-string', + label: 'Connection string', + currentWhen: { kind: 'present', key: 'AZURE_CONNECTION_STRING' }, + prompts: [ + { + type: 'field', + key: 'AZURE_CONNECTION_STRING', + input: 'secret', + required: true, + }, + ], + }, + { + id: 'account-key', + label: 'Account name and key', + currentWhen: { + kind: 'any', + conditions: [ + { kind: 'present', key: 'AZURE_ACCOUNT_NAME' }, + { kind: 'present', key: 'AZURE_ACCOUNT_KEY' }, + ], + }, + prompts: [ + { + type: 'field', + key: 'AZURE_ACCOUNT_NAME', + input: 'text', + required: true, + }, + { + type: 'field', + key: 'AZURE_ACCOUNT_KEY', + input: 'secret', + required: true, + }, + ], + }, + ], + }, + ], + }, + s3: { + hint: 'AWS S3 or an S3-compatible endpoint', + env: { S3_FORCE_PATH_STYLE: 'false' }, + prompts: [ + { + type: 'field', + key: 'S3_ENDPOINT', + input: 'text', + hint: 'optional for R2, MinIO, B2, or another S3-compatible service', + validate: true, + }, + { + type: 'confirm', + key: 'S3_FORCE_PATH_STYLE', + message: 'Force path-style addressing? (required for MinIO/Ceph, not for R2)', + when: { kind: 'present', key: 'S3_ENDPOINT' }, + }, + { type: 'field', key: 'AWS_REGION', input: 'text', required: true }, + { type: 'field', key: 'S3_BUCKET_NAME', input: 'text', required: true }, + { + type: 'choice', + id: 'aws-credentials', + message: 'AWS credentials?', + options: [ + { + id: 'chain', + label: 'Default credential chain', + hint: 'IAM role, IRSA, profile, or other SDK source', + currentWhen: { + kind: 'all', + conditions: [ + { kind: 'present', key: 'S3_BUCKET_NAME' }, + { + kind: 'not', + condition: { + kind: 'any', + conditions: [ + { kind: 'present', key: 'AWS_ACCESS_KEY_ID' }, + { kind: 'present', key: 'AWS_SECRET_ACCESS_KEY' }, + ], + }, + }, + ], + }, + }, + { + id: 'static', + label: 'Access key and secret', + hint: 'stored in AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY', + currentWhen: { + kind: 'any', + conditions: [ + { kind: 'present', key: 'AWS_ACCESS_KEY_ID' }, + { kind: 'present', key: 'AWS_SECRET_ACCESS_KEY' }, + ], + }, + prompts: [ + { + type: 'field', + key: 'AWS_ACCESS_KEY_ID', + input: 'secret', + required: true, + }, + { + type: 'field', + key: 'AWS_SECRET_ACCESS_KEY', + input: 'secret', + required: true, + }, + ], + }, + ], + }, + ], + }, + gcs: { + hint: 'bucket; credentials via ADC by default', + prompts: [ + { + type: 'field', + key: 'GCS_BUCKET_NAME', + input: 'text', + required: true, + }, + { + type: 'field', + key: 'GCS_PROJECT_ID', + input: 'text', + hint: 'optional; inferred from credentials or ADC when empty', + }, + { + type: 'choice', + id: 'gcs-credentials', + message: 'Google Cloud credentials?', + options: [ + { + id: 'adc', + label: 'Application Default Credentials', + hint: 'Workload Identity or GOOGLE_APPLICATION_CREDENTIALS', + currentWhen: { + kind: 'all', + conditions: [ + { kind: 'present', key: 'GCS_BUCKET_NAME' }, + { + kind: 'not', + condition: { kind: 'present', key: 'GCS_CREDENTIALS_JSON' }, + }, + ], + }, + }, + { + id: 'json', + label: 'Inline service account JSON', + hint: 'stored in GCS_CREDENTIALS_JSON', + currentWhen: { kind: 'present', key: 'GCS_CREDENTIALS_JSON' }, + prompts: [ + { + type: 'field', + key: 'GCS_CREDENTIALS_JSON', + input: 'secret', + required: true, + validate: true, + }, + ], + }, + ], + }, + ], + }, + }, + optionOrder: ['local', 's3', 'azure', 'gcs'], +}) + +export const SANDBOX_SETUP = defineCapabilitySetup(SANDBOX_CAPABILITY, { + label: 'Remote sandboxes', + message: 'Remote sandbox provider?', + actions: { + disabled: { + label: 'Disabled', + hint: 'local JavaScript execution only', + env: { + NEXT_PUBLIC_E2B_ENABLED: 'false', + NEXT_PUBLIC_SANDBOX_ENABLED: 'false', + }, + currentWhen: { + kind: 'all', + conditions: [ + { kind: 'not', condition: { kind: 'truthy', key: 'E2B_ENABLED' } }, + { + kind: 'not', + condition: { kind: 'present', key: 'DAYTONA_API_KEY' }, + }, + { + kind: 'not', + condition: { kind: 'present', key: 'DAYTONA_SHELL_SNAPSHOT_ID' }, + }, + ], + }, + }, + }, + providers: { + e2b: { + hint: 'remote code interpreter sandboxes', + env: { + NEXT_PUBLIC_E2B_ENABLED: 'true', + NEXT_PUBLIC_SANDBOX_ENABLED: 'true', + }, + prompts: [{ type: 'field', key: 'E2B_API_KEY', input: 'secret', required: true }], + currentWhen: { kind: 'truthy', key: 'E2B_ENABLED' }, + }, + daytona: { + hint: 'remote Daytona sandboxes', + env: { + NEXT_PUBLIC_E2B_ENABLED: 'false', + NEXT_PUBLIC_SANDBOX_ENABLED: 'true', + }, + prompts: [ + { + type: 'field', + key: 'DAYTONA_API_KEY', + input: 'secret', + required: true, + }, + { + type: 'field', + key: 'DAYTONA_SHELL_SNAPSHOT_ID', + input: 'text', + required: true, + validate: true, + }, + ], + }, + }, + optionOrder: ['disabled', 'e2b', 'daytona'], +}) + +export const JOBS_SETUP = defineCapabilitySetup(ASYNC_JOBS_CAPABILITY, { + label: 'Async jobs', + message: 'Async job provider?', + actions: {}, + defaultOption: { hint: 'built-in default' }, + providers: { + 'trigger-dev': { + hint: 'external background jobs', + prompts: [ + { + type: 'field', + key: 'TRIGGER_PROJECT_ID', + input: 'text', + required: true, + }, + { + type: 'field', + key: 'TRIGGER_SECRET_KEY', + input: 'secret', + required: true, + }, + ], + }, + }, + optionOrder: ['database', 'trigger-dev'], +}) + +export const CACHE_SETUP = defineCapabilitySetup(CACHE_CAPABILITY, { + label: 'Redis cache', + message: 'Cache and realtime coordination?', + actions: {}, + defaultOption: { hint: 'built-in default' }, + providers: { + redis: { + hint: 'recommended for multiple replicas', + prompts: [ + { + type: 'field', + key: 'REDIS_URL', + input: 'text', + required: true, + defaultValue: 'redis://localhost:6379', + validate: true, + }, + { + type: 'field', + key: 'REDIS_TLS_SERVERNAME', + input: 'text', + required: true, + when: { kind: 'provider-missing-field', key: 'REDIS_TLS_SERVERNAME' }, + }, + ], + }, + }, + optionOrder: ['database', 'redis'], +}) + +export const KNOWLEDGE_SETUP = defineCapabilitySetup(OCR_CAPABILITY, { + label: 'Knowledge and OCR', + message: 'PDF OCR provider?', + actions: {}, + defaultOption: { hint: 'built-in default' }, + providers: { + 'azure-mistral': { + hint: 'Azure model deployment', + prompts: [ + { + type: 'field', + key: 'OCR_AZURE_ENDPOINT', + input: 'text', + required: true, + validate: true, + }, + { + type: 'field', + key: 'OCR_AZURE_MODEL_NAME', + input: 'text', + required: true, + }, + { + type: 'field', + key: 'OCR_AZURE_API_KEY', + input: 'secret', + required: true, + }, + ], + }, + mistral: { + hint: 'Mistral API key', + prompts: [ + { + type: 'field', + key: 'MISTRAL_API_KEY', + input: 'secret', + required: true, + }, + ], + }, + }, + optionOrder: ['local', 'mistral', 'azure-mistral'], +}) + +export const CAPABILITY_SETUPS = [ + EMAIL_SETUP, + STORAGE_SETUP, + SANDBOX_SETUP, + JOBS_SETUP, + CACHE_SETUP, + KNOWLEDGE_SETUP, +] as const + +const configuredCapabilityIds = new Set(CAPABILITY_SETUPS.map((setup) => setup.definition.id)) +const missingCapabilitySetups = ENV_CAPABILITIES.filter( + (capability) => !configuredCapabilityIds.has(capability.id) +) +if (missingCapabilitySetups.length > 0) { + throw new Error( + `Missing CLI setup for capabilities: ${missingCapabilitySetups.map((capability) => capability.id).join(', ')}` + ) +} + +export type CapabilitySetupId = (typeof CAPABILITY_SETUPS)[number]['definition']['id'] +export type SetupFeatureId = CapabilitySetupId | 'llm' | 'integration' + +export const SETUP_FEATURES: readonly { id: SetupFeatureId; label: string }[] = [ + ...CAPABILITY_SETUPS.map((setup) => ({ + id: setup.definition.id, + label: setup.label, + })), + { id: 'llm', label: 'LLM API keys' }, + { id: 'integration', label: 'OAuth integration' }, +] + +export function getCapabilitySetup(id: string): CapabilitySetupDefinition | null { + return CAPABILITY_SETUPS.find((setup) => setup.definition.id === id) ?? null +} + +export function getSetupCommand(id: string): string { + return `bun run setup ${id}` +} + +type OAuthClientSetupFields = { + [TId in OAuthClientCapabilityId]: Record< + OAuthClientCapabilityField, + { input: 'text' | 'secret' } + > +} + +export const OAUTH_CLIENT_SETUP_FIELDS = { + google: { + GOOGLE_CLIENT_ID: { input: 'text' }, + GOOGLE_CLIENT_SECRET: { input: 'secret' }, + }, + x: { X_CLIENT_ID: { input: 'text' }, X_CLIENT_SECRET: { input: 'secret' } }, + tiktok: { + TIKTOK_CLIENT_ID: { input: 'text' }, + TIKTOK_CLIENT_SECRET: { input: 'secret' }, + }, + confluence: { + CONFLUENCE_CLIENT_ID: { input: 'text' }, + CONFLUENCE_CLIENT_SECRET: { input: 'secret' }, + }, + jira: { + JIRA_CLIENT_ID: { input: 'text' }, + JIRA_CLIENT_SECRET: { input: 'secret' }, + }, + calcom: { CALCOM_CLIENT_ID: { input: 'text' } }, + airtable: { + AIRTABLE_CLIENT_ID: { input: 'text' }, + AIRTABLE_CLIENT_SECRET: { input: 'secret' }, + }, + notion: { + NOTION_CLIENT_ID: { input: 'text' }, + NOTION_CLIENT_SECRET: { input: 'secret' }, + }, + microsoft: { + MICROSOFT_CLIENT_ID: { input: 'text' }, + MICROSOFT_CLIENT_SECRET: { input: 'secret' }, + }, + clickup: { + CLICKUP_CLIENT_ID: { input: 'text' }, + CLICKUP_CLIENT_SECRET: { input: 'secret' }, + }, + linear: { + LINEAR_CLIENT_ID: { input: 'text' }, + LINEAR_CLIENT_SECRET: { input: 'secret' }, + }, + attio: { + ATTIO_CLIENT_ID: { input: 'text' }, + ATTIO_CLIENT_SECRET: { input: 'secret' }, + }, + box: { + BOX_CLIENT_ID: { input: 'text' }, + BOX_CLIENT_SECRET: { input: 'secret' }, + }, + docusign: { + DOCUSIGN_CLIENT_ID: { input: 'text' }, + DOCUSIGN_CLIENT_SECRET: { input: 'secret' }, + }, + dropbox: { + DROPBOX_CLIENT_ID: { input: 'text' }, + DROPBOX_CLIENT_SECRET: { input: 'secret' }, + }, + slack: { + SLACK_CLIENT_ID: { input: 'text' }, + SLACK_CLIENT_SECRET: { input: 'secret' }, + }, + reddit: { + REDDIT_CLIENT_ID: { input: 'text' }, + REDDIT_CLIENT_SECRET: { input: 'secret' }, + }, + wealthbox: { + WEALTHBOX_CLIENT_ID: { input: 'text' }, + WEALTHBOX_CLIENT_SECRET: { input: 'secret' }, + }, + webflow: { + WEBFLOW_CLIENT_ID: { input: 'text' }, + WEBFLOW_CLIENT_SECRET: { input: 'secret' }, + }, + asana: { + ASANA_CLIENT_ID: { input: 'text' }, + ASANA_CLIENT_SECRET: { input: 'secret' }, + }, + pipedrive: { + PIPEDRIVE_CLIENT_ID: { input: 'text' }, + PIPEDRIVE_CLIENT_SECRET: { input: 'secret' }, + }, + hubspot: { + HUBSPOT_CLIENT_ID: { input: 'text' }, + HUBSPOT_CLIENT_SECRET: { input: 'secret' }, + }, + linkedin: { + LINKEDIN_CLIENT_ID: { input: 'text' }, + LINKEDIN_CLIENT_SECRET: { input: 'secret' }, + }, + instagram: { + INSTAGRAM_CLIENT_ID: { input: 'text' }, + INSTAGRAM_CLIENT_SECRET: { input: 'secret' }, + }, + salesforce: { + SALESFORCE_CLIENT_ID: { input: 'text' }, + SALESFORCE_CLIENT_SECRET: { input: 'secret' }, + }, + shopify: { + SHOPIFY_CLIENT_ID: { input: 'text' }, + SHOPIFY_CLIENT_SECRET: { input: 'secret' }, + }, + zoom: { + ZOOM_CLIENT_ID: { input: 'text' }, + ZOOM_CLIENT_SECRET: { input: 'secret' }, + }, + wordpress: { + WORDPRESS_CLIENT_ID: { input: 'text' }, + WORDPRESS_CLIENT_SECRET: { input: 'secret' }, + }, + spotify: { + SPOTIFY_CLIENT_ID: { input: 'text' }, + SPOTIFY_CLIENT_SECRET: { input: 'secret' }, + }, + monday: { + MONDAY_CLIENT_ID: { input: 'text' }, + MONDAY_CLIENT_SECRET: { input: 'secret' }, + }, + trello: { TRELLO_API_KEY: { input: 'secret' } }, + 'zoho-desk': { + ZOHO_CLIENT_ID: { input: 'text' }, + ZOHO_CLIENT_SECRET: { input: 'secret' }, + }, +} satisfies OAuthClientSetupFields + +export function getOAuthClientSetupFields( + id: OAuthClientCapabilityId +): readonly { key: string; input: 'text' | 'secret' }[] { + const runtimeFields = OAUTH_CLIENT_CAPABILITIES[id] as readonly string[] + const setupFields = OAUTH_CLIENT_SETUP_FIELDS[id] as Record + const unknownFields = Object.keys(setupFields).filter((key) => !runtimeFields.includes(key)) + const missingFields = runtimeFields.filter((key) => !Object.hasOwn(setupFields, key)) + if (unknownFields.length > 0 || missingFields.length > 0) { + throw new Error( + `OAuth setup ${id} field mapping drifted (missing: ${missingFields.join(', ') || 'none'}; unknown: ${unknownFields.join(', ') || 'none'})` + ) + } + return runtimeFields.map((key) => ({ key, input: setupFields[key].input })) +} + +export function matchesSetupCondition( + condition: SetupCondition, + values: EnvCapabilityValues +): boolean { + if (condition.kind === 'present') return hasEnvCapabilityValue(values, condition.key) + if (condition.kind === 'truthy') { + const value = + values instanceof Map + ? values.get(condition.key) + : (values as Readonly>)[condition.key] + return value === true || value === 1 || String(value).toLowerCase() === 'true' + } + if (condition.kind === 'equals') { + const value = + values instanceof Map + ? values.get(condition.key) + : (values as Readonly>)[condition.key] + return hasEnvCapabilityValue(values, condition.key) && String(value).trim() === condition.value + } + if (condition.kind === 'all') { + return condition.conditions.every((child) => matchesSetupCondition(child, values)) + } + if (condition.kind === 'any') { + return condition.conditions.some((child) => matchesSetupCondition(child, values)) + } + return !matchesSetupCondition(condition.condition, values) +} diff --git a/scripts/setup/capability-setup.test.ts b/scripts/setup/capability-setup.test.ts new file mode 100644 index 00000000000..4b4f8c857a8 --- /dev/null +++ b/scripts/setup/capability-setup.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'bun:test' +import type { SetupFieldPrompt } from './capability-config.ts' +import { formatCapabilitySetupFieldMessage, markCurrentlyUsed } from './capability-setup.ts' + +describe('capability setup presentation', () => { + it('marks the effective option as currently used without replacing its hint', () => { + expect(markCurrentlyUsed('paste an API key', true)).toBe('paste an API key · Currently used') + expect(markCurrentlyUsed(undefined, true)).toBe('Currently used') + expect(markCurrentlyUsed('paste an API key', false)).toBe('paste an API key') + }) + + it('marks existing fields and explains how secrets are preserved', () => { + const prompt: SetupFieldPrompt = { + type: 'field', + key: 'RESEND_API_KEY', + input: 'secret', + } + + expect(formatCapabilitySetupFieldMessage(prompt, true, true)).toBe( + 'RESEND_API_KEY (Currently used); leave empty to keep it' + ) + expect(formatCapabilitySetupFieldMessage(prompt, false, true)).toBe('RESEND_API_KEY') + }) +}) diff --git a/scripts/setup/capability-setup.ts b/scripts/setup/capability-setup.ts new file mode 100644 index 00000000000..bad7fa26e43 --- /dev/null +++ b/scripts/setup/capability-setup.ts @@ -0,0 +1,508 @@ +/** + * Generic prompt rendering and environment transitions for the CLI setup catalog. + * + * @packageDocumentation + */ +import { + EnvCapabilityConfigurationError, + type EnvCapabilityValue, + type EnvCapabilityValues, + getProviderFields, + hasEnvCapabilityValue, + inspectCapability, + isTruthyEnvCapabilityValue, + validateCapabilityFieldInput, +} from '../../apps/sim/lib/core/config/env-capabilities.ts' +import { + type CapabilitySetupDefinition, + matchesSetupCondition, + type SetupCondition, + type SetupHint, + type SetupPrompt, +} from './capability-config.ts' +import * as p from './prompter.ts' + +export interface CapabilitySetupContext { + containerized: boolean +} + +export interface EnvCapabilitySetupTransition { + values: Record + remove: readonly string[] +} + +interface ResolvedSetupOption { + id: string + label: string + hint?: SetupHint + kind: 'provider' | 'default' | 'action' + providerId?: string + env: Readonly> + prompts: readonly SetupPrompt[] + currentWhen?: SetupCondition +} + +interface PromptState { + setup: CapabilitySetupDefinition + optionId: string + currentValues: ReadonlyMap + values: Record +} + +/** Stages a capability transition into a larger setup run without losing prompt context. */ +export function stageCapabilitySetupTransition( + currentValues: Map, + values: Record, + remove: Set, + transition: EnvCapabilitySetupTransition +): void { + for (const key of transition.remove) { + currentValues.delete(key) + Reflect.deleteProperty(values, key) + remove.add(key) + } + for (const [key, value] of Object.entries(transition.values)) { + currentValues.set(key, value) + values[key] = value + remove.delete(key) + } +} + +function resolveHint( + hint: SetupHint | undefined, + context: CapabilitySetupContext +): string | undefined { + if (!hint || typeof hint === 'string') return hint + return context.containerized ? hint.containerized : hint.development +} + +/** Adds the standard status marker without discarding an option's explanatory hint. */ +export function markCurrentlyUsed(hint: string | undefined, current: boolean): string | undefined { + if (!current) return hint + return hint ? `${hint} · Currently used` : 'Currently used' +} + +function promptKeys(prompts: readonly SetupPrompt[] = []): string[] { + return prompts.flatMap((prompt) => { + if (prompt.type === 'field' || prompt.type === 'confirm') return [prompt.key] + return prompt.options.flatMap((option) => [ + ...Object.keys(option.env ?? {}), + ...promptKeys(option.prompts), + ]) + }) +} + +function providerSetupFields( + setup: CapabilitySetupDefinition, + providerId: string +): readonly string[] { + const provider = setup.definition.providers.find((candidate) => candidate.id === providerId) + if (!provider) throw new Error(`Capability ${setup.definition.id} has no provider ${providerId}`) + const providerSetup = setup.providers[providerId] + if (!providerSetup) throw new Error(`Setup ${setup.definition.id} has no provider ${providerId}`) + return [ + ...(provider.activation.mode === 'enabled' ? [provider.activation.key] : []), + ...Object.keys(providerSetup.env ?? {}), + ...promptKeys(providerSetup.prompts), + ] +} + +/** Returns fields the setup flow writes or prompts for, including inferred selectors and flags. */ +export function getCapabilitySetupFields(setup: CapabilitySetupDefinition): readonly string[] { + return [ + ...new Set([ + ...(setup.definition.strategy === 'selected' && setup.definition.selectorKey + ? [setup.definition.selectorKey] + : []), + ...setup.definition.providers.flatMap((provider) => + provider.activation.mode === 'enabled' ? [provider.activation.key] : [] + ), + ...Object.entries(setup.providers).flatMap(([providerId]) => + providerSetupFields(setup, providerId) + ), + ...Object.values(setup.actions).flatMap((action) => Object.keys(action.env ?? {})), + ]), + ] +} + +/** Expands the runtime providers and CLI-only actions into renderable options. */ +export function getCapabilitySetupOptions( + setup: CapabilitySetupDefinition +): readonly ResolvedSetupOption[] { + const definition = setup.definition + const options: ResolvedSetupOption[] = definition.providers.map((provider) => { + const providerSetup = setup.providers[provider.id] + if (!providerSetup) throw new Error(`Setup ${definition.id} has no provider ${provider.id}`) + return { + id: provider.id, + label: provider.label, + hint: providerSetup.hint, + kind: 'provider', + providerId: provider.id, + env: providerSetup.env ?? {}, + prompts: providerSetup.prompts, + currentWhen: providerSetup.currentWhen, + } + }) + + if (definition.strategy === 'selected' && definition.defaultProvider.kind === 'built-in') { + options.push({ + id: definition.defaultProvider.id, + label: definition.defaultProvider.label, + hint: setup.defaultOption?.hint, + kind: 'default', + env: {}, + prompts: [], + }) + } + + for (const [id, action] of Object.entries(setup.actions)) { + options.push({ + id, + label: action.label, + hint: action.hint, + kind: 'action', + env: action.env ?? {}, + prompts: [], + currentWhen: action.currentWhen, + }) + } + + const byId = new Map(options.map((option) => [option.id, option])) + return setup.optionOrder.map((id) => { + const option = byId.get(String(id)) + if (!option) throw new Error(`Setup ${definition.id} option order references ${String(id)}`) + return option + }) +} + +/** Resolves the setup option representing the effective current configuration. */ +export function resolveCurrentCapabilitySetupOptionId( + setup: CapabilitySetupDefinition, + values: EnvCapabilityValues +): string { + const options = getCapabilitySetupOptions(setup) + const explicitAction = options.find( + (option) => + option.kind === 'action' && + option.currentWhen && + matchesSetupCondition(option.currentWhen, values) + ) + if (explicitAction) return explicitAction.id + + const inspection = inspectCapability(setup.definition, values) + const selectedProviderId = + inspection.strategy === 'selected' + ? (inspection.providerId ?? inspection.providers.find((provider) => provider.active)?.id) + : (inspection.providerIds[0] ?? inspection.providers.find((provider) => provider.active)?.id) + if (selectedProviderId) { + const provider = options.find( + (option) => + option.kind === 'provider' && + option.providerId === selectedProviderId && + (!option.currentWhen || matchesSetupCondition(option.currentWhen, values)) + ) + if (provider) return provider.id + } + + if ( + setup.definition.strategy === 'selected' && + setup.definition.defaultProvider.kind === 'built-in' + ) { + return setup.definition.defaultProvider.id + } + + const firstAction = options.find((option) => option.kind === 'action') + if (firstAction) return firstAction.id + throw new Error( + `Capability ${setup.definition.id} has no setup option for its current configuration` + ) +} + +/** Applies selector and activation inference to the CLI-entered values. */ +export function getCapabilitySetupDraftValues( + setup: CapabilitySetupDefinition, + optionId: string, + promptedValues: Readonly> +): Record { + const definition = setup.definition + const option = getCapabilitySetupOptions(setup).find((candidate) => candidate.id === optionId) + if (!option) throw new Error(`Capability ${definition.id} has no setup option ${optionId}`) + + const values: Record = { ...option.env } + if (definition.strategy === 'selected' && definition.selectorKey) { + if (option.providerId) values[definition.selectorKey] = option.providerId + else if (option.kind === 'default' && option.id === definition.defaultProvider.id) { + values[definition.selectorKey] = option.id + } + } + for (const provider of definition.providers) { + if (provider.activation.mode !== 'enabled') continue + values[provider.activation.key] = provider.id === option.providerId ? 'true' : 'false' + } + Object.assign(values, promptedValues) + return values +} + +function copyValues(values: EnvCapabilityValues): Record { + return values instanceof Map + ? Object.fromEntries(values) + : { ...(values as Readonly>) } +} + +function fieldsToReplace( + setup: CapabilitySetupDefinition, + option: ResolvedSetupOption +): readonly string[] { + if (setup.definition.strategy === 'fallback' && option.providerId) { + return providerSetupFields(setup, option.providerId) + } + const selectedProviderFields = new Set( + setup.definition.providers + .filter((provider) => provider.id === option.providerId) + .flatMap(getProviderFields) + ) + const inactiveProviderFields = setup.definition.providers + .filter((provider) => provider.id !== option.providerId) + .flatMap(getProviderFields) + .filter((field) => !selectedProviderFields.has(field)) + return [...new Set([...getCapabilitySetupFields(setup), ...inactiveProviderFields])] +} + +function proposedCapabilitySetupValues( + setup: CapabilitySetupDefinition, + option: ResolvedSetupOption, + values: Readonly>, + currentValues: EnvCapabilityValues +): Record { + const proposed = copyValues(currentValues) + for (const key of fieldsToReplace(setup, option)) Reflect.deleteProperty(proposed, key) + Object.assign(proposed, values) + return proposed +} + +/** Returns provider requirements discovered after earlier prompts have been answered. */ +export function getCapabilitySetupProviderMissingFields( + setup: CapabilitySetupDefinition, + optionId: string, + promptedValues: Readonly>, + currentValues: EnvCapabilityValues +): readonly string[] { + const option = getCapabilitySetupOptions(setup).find((candidate) => candidate.id === optionId) + if (!option?.providerId) return [] + const values = getCapabilitySetupDraftValues(setup, optionId, promptedValues) + const inspection = inspectCapability( + setup.definition, + proposedCapabilitySetupValues(setup, option, values, currentValues) + ) + return ( + inspection.providers.find((provider) => provider.id === option.providerId)?.missingFields ?? [] + ) +} + +/** Builds and validates the complete environment transition for one setup option. */ +export function buildCapabilitySetupTransition( + setup: CapabilitySetupDefinition, + optionId: string, + promptedValues: Readonly>, + currentValues: EnvCapabilityValues +): EnvCapabilitySetupTransition { + const definition = setup.definition + const option = getCapabilitySetupOptions(setup).find((candidate) => candidate.id === optionId) + if (!option) throw new Error(`Capability ${definition.id} has no setup option ${optionId}`) + + const values = getCapabilitySetupDraftValues(setup, optionId, promptedValues) + const ownedFields = getCapabilitySetupFields(setup) + const unexpected = Object.keys(values).filter((key) => !ownedFields.includes(key)) + if (unexpected.length > 0) { + throw new Error( + `Capability ${definition.id} setup produced unowned fields: ${unexpected.join(', ')}` + ) + } + + const replacedFields = fieldsToReplace(setup, option) + const remove = replacedFields.filter((key) => !Object.hasOwn(values, key)) + const proposed = proposedCapabilitySetupValues(setup, option, values, currentValues) + const inspection = inspectCapability(definition, proposed) + if (inspection.error) throw inspection.error + + if (option.providerId) { + const provider = inspection.providers.find((candidate) => candidate.id === option.providerId) + if (!provider || provider.state !== 'ready') { + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} setup option ${option.id} did not configure ${option.providerId}` + ) + } + if (inspection.strategy === 'selected' && inspection.providerId !== option.providerId) { + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} setup selected ${inspection.providerId ?? 'nothing'} instead of ${option.providerId}` + ) + } + } else { + const activeProvider = inspection.providers.find((provider) => provider.active) + if (activeProvider) { + throw new EnvCapabilityConfigurationError( + definition.id, + `${definition.label} setup option ${option.id} left ${activeProvider.id} active` + ) + } + } + + return { values, remove } +} + +function proposedPromptValues(state: PromptState): EnvCapabilityValues { + const option = getCapabilitySetupOptions(state.setup).find( + (candidate) => candidate.id === state.optionId + ) + if (!option) { + throw new Error(`Capability ${state.setup.definition.id} has no setup option ${state.optionId}`) + } + return proposedCapabilitySetupValues( + state.setup, + option, + getCapabilitySetupDraftValues(state.setup, state.optionId, state.values), + state.currentValues + ) +} + +function shouldRenderPrompt(prompt: SetupPrompt, state: PromptState): boolean { + if (!prompt.when) return true + if (prompt.when.kind === 'provider-missing-field') { + return getCapabilitySetupProviderMissingFields( + state.setup, + state.optionId, + state.values, + state.currentValues + ).includes(prompt.when.key) + } + return matchesSetupCondition(prompt.when, proposedPromptValues(state)) +} + +/** Formats current-value status without ever rendering the configured value. */ +export function formatCapabilitySetupFieldMessage( + prompt: Extract, + current: boolean, + keepExisting = false +): string { + const label = prompt.message ?? prompt.key + const status = current ? ' (Currently used)' : '' + const hint = prompt.hint ? ` — ${prompt.hint}` : '' + const preservation = current && keepExisting ? '; leave empty to keep it' : '' + return `${label}${status}${hint}${preservation}` +} + +async function promptField( + prompt: Extract, + state: PromptState +): Promise { + const existing = state.currentValues.get(prompt.key) + const current = hasEnvCapabilityValue(state.currentValues, prompt.key) + const draft = getCapabilitySetupDraftValues(state.setup, state.optionId, state.values) + const initialValue = existing ?? draft[prompt.key] ?? prompt.defaultValue + const validate = (value: string): string | undefined => { + if (!value) return prompt.required && !existing ? 'required' : undefined + return prompt.validate + ? validateCapabilityFieldInput(state.setup.definition, prompt.key, value) + : undefined + } + + let value: string + if (prompt.input === 'secret') { + value = await p.password({ + message: formatCapabilitySetupFieldMessage(prompt, current, true), + validate, + }) + if (!value && existing) value = existing + } else { + value = await p.text({ + message: formatCapabilitySetupFieldMessage(prompt, current), + initialValue, + defaultValue: initialValue ? undefined : prompt.defaultValue, + validate, + }) + } + + if (value) state.values[prompt.key] = value + else Reflect.deleteProperty(state.values, prompt.key) +} + +async function renderPrompts(prompts: readonly SetupPrompt[], state: PromptState): Promise { + for (const prompt of prompts) { + if (!shouldRenderPrompt(prompt, state)) continue + if (prompt.type === 'field') { + await promptField(prompt, state) + continue + } + if (prompt.type === 'confirm') { + const proposed = proposedPromptValues(state) + const current = hasEnvCapabilityValue(state.currentValues, prompt.key) + const enabled = await p.confirm({ + message: `${prompt.message}${current ? ' (Currently used)' : ''}`, + initialValue: hasEnvCapabilityValue(proposed, prompt.key) + ? isTruthyEnvCapabilityValue(proposed, prompt.key) + : (prompt.defaultValue ?? false), + }) + state.values[prompt.key] = enabled ? 'true' : 'false' + continue + } + + const currentOption = prompt.options.find( + (option) => + option.currentWhen && matchesSetupCondition(option.currentWhen, state.currentValues) + ) + const initialOption = currentOption ?? prompt.options[0] + if (!initialOption) throw new Error(`Setup choice ${prompt.id} has no options`) + const selectedId = await p.select({ + message: prompt.message, + options: prompt.options.map((option) => ({ + value: option.id, + label: option.label, + hint: markCurrentlyUsed(option.hint, option.id === currentOption?.id), + })), + initialValue: initialOption.id, + }) + const selected = prompt.options.find((option) => option.id === selectedId) + if (!selected) { + throw new Error(`Setup choice ${prompt.id} returned unknown option ${selectedId}`) + } + Object.assign(state.values, selected.env ?? {}) + await renderPrompts(selected.prompts ?? [], state) + } +} + +/** Renders a CLI-owned capability setup and returns its validated environment transition. */ +export async function promptCapabilitySetup( + setup: CapabilitySetupDefinition, + currentValues: ReadonlyMap, + context: CapabilitySetupContext +): Promise { + const options = getCapabilitySetupOptions(setup) + const currentOptionId = resolveCurrentCapabilitySetupOptionId(setup, currentValues) + const selectedOptionId = await p.select({ + message: setup.message, + options: options.map((option) => ({ + value: option.id, + label: option.label, + hint: markCurrentlyUsed(resolveHint(option.hint, context), option.id === currentOptionId), + })), + initialValue: currentOptionId, + }) + const selected = options.find((option) => option.id === selectedOptionId) + if (!selected) { + throw new Error( + `Capability ${setup.definition.id} returned unknown setup option ${selectedOptionId}` + ) + } + + const state: PromptState = { + setup, + optionId: selected.id, + currentValues, + values: {}, + } + await renderPrompts(selected.prompts, state) + return buildCapabilitySetupTransition(setup, selected.id, state.values, currentValues) +} diff --git a/scripts/setup/capability-status.test.ts b/scripts/setup/capability-status.test.ts new file mode 100644 index 00000000000..706cbb43772 --- /dev/null +++ b/scripts/setup/capability-status.test.ts @@ -0,0 +1,271 @@ +import { describe, expect, it } from 'bun:test' +import { OAUTH_CLIENT_CAPABILITIES } from '../../apps/sim/lib/core/config/env-capabilities.ts' +import { buildEnvCapabilityStatus } from './capability-status.ts' + +describe('env capability status', () => { + it('reports built-in defaults without treating them as configured services', () => { + const status = buildEnvCapabilityStatus({}) + + expect(status.features.email).toMatchObject({ state: 'missing', providerIds: [] }) + expect(status.features.storage).toMatchObject({ state: 'default', providerId: 'local' }) + expect(status.features.sandbox).toMatchObject({ state: 'default', providerId: 'disabled' }) + expect(status.features.jobs).toMatchObject({ state: 'default', providerId: 'database' }) + expect(status.features.cache).toMatchObject({ state: 'default', providerId: 'database' }) + expect(status.features.knowledge).toMatchObject({ state: 'default', providerId: 'local' }) + expect(status.features.llm).toMatchObject({ + state: 'missing', + configuredPoolCount: 0, + configuredKeyCount: 0, + effectiveKeyCount: 0, + }) + expect(status.oauthClients.absentCount).toBe(Object.keys(OAUTH_CLIENT_CAPABILITIES).length) + }) + + it('reports an explicitly selected but disabled E2B provider as the default', () => { + const status = buildEnvCapabilityStatus({ + SANDBOX_PROVIDER: 'e2b', + E2B_ENABLED: 'false', + NEXT_PUBLIC_E2B_ENABLED: 'false', + NEXT_PUBLIC_SANDBOX_ENABLED: 'false', + }) + + expect(status.features.sandbox).toEqual({ + id: 'sandbox', + label: 'Remote sandboxes', + setupCommand: 'bun run setup sandbox', + state: 'default', + providerId: 'disabled', + }) + }) + + it('preserves declared email fallback order', () => { + const status = buildEnvCapabilityStatus({ + SMTP_HOST: 'localhost', + SMTP_PORT: '1025', + RESEND_API_KEY: 'secret-resend-key', + GMAIL_CREDENTIALS_JSON: JSON.stringify({ + client_email: 'mailer@example.com', + private_key: 'secret-private-key', + }), + GMAIL_SENDER: 'mailer@example.com', + }) + + expect(status.features.email).toMatchObject({ + state: 'configured', + providerIds: ['resend', 'smtp', 'gmail'], + }) + expect(JSON.stringify(status)).not.toContain('secret-resend-key') + expect(JSON.stringify(status)).not.toContain('secret-private-key') + }) + + it('reports selected Daytona and cloud storage providers', () => { + const status = buildEnvCapabilityStatus({ + SANDBOX_PROVIDER: 'daytona', + DAYTONA_API_KEY: 'daytona-secret', + DAYTONA_SHELL_SNAPSHOT_ID: 'mothership-shell:v1', + NEXT_PUBLIC_SANDBOX_ENABLED: 'true', + STORAGE_PROVIDER: 's3', + AWS_REGION: 'us-east-1', + S3_BUCKET_NAME: 'files', + }) + + expect(status.features.sandbox).toMatchObject({ + state: 'configured', + providerId: 'daytona', + }) + expect(status.features.storage).toMatchObject({ + state: 'configured', + providerId: 's3', + }) + }) + + it('reports Daytona as missing when its default shell snapshot is absent', () => { + const status = buildEnvCapabilityStatus({ + SANDBOX_PROVIDER: 'daytona', + DAYTONA_API_KEY: 'daytona-secret', + NEXT_PUBLIC_SANDBOX_ENABLED: 'true', + }) + + expect(status.features.sandbox).toMatchObject({ + state: 'missing', + providerId: 'daytona', + issue: { state: 'missing' }, + }) + expect(status.features.sandbox.issue?.message).toContain('DAYTONA_SHELL_SNAPSHOT_ID') + }) + + it('reports an untagged or floating Daytona shell snapshot as invalid', () => { + const status = buildEnvCapabilityStatus({ + SANDBOX_PROVIDER: 'daytona', + DAYTONA_API_KEY: 'daytona-secret', + DAYTONA_SHELL_SNAPSHOT_ID: 'mothership-shell:latest', + NEXT_PUBLIC_SANDBOX_ENABLED: 'true', + }) + + expect(status.features.sandbox).toMatchObject({ + state: 'invalid', + providerId: 'daytona', + issue: { state: 'invalid' }, + }) + expect(status.features.sandbox.issue?.message).toContain('explicit, non-floating name:tag') + }) + + it('reports remote sandbox server/browser drift as partial', () => { + const status = buildEnvCapabilityStatus({ + SANDBOX_PROVIDER: 'daytona', + DAYTONA_API_KEY: 'daytona-secret', + DAYTONA_SHELL_SNAPSHOT_ID: 'mothership-shell:v1', + }) + + expect(status.features.sandbox).toMatchObject({ + state: 'partial', + providerId: 'daytona', + issue: { state: 'partial' }, + }) + expect(status.features.sandbox.issue?.message).toContain('NEXT_PUBLIC_SANDBOX_ENABLED') + }) + + it('captures partial and invalid entries without aborting the snapshot', () => { + const status = buildEnvCapabilityStatus({ + SMTP_HOST: 'localhost', + TRIGGER_DEV_ENABLED: 'true', + TRIGGER_PROJECT_ID: 'project', + REDIS_URL: 'not-a-url', + OCR_AZURE_ENDPOINT: 'https://ocr.example.com', + }) + + expect(status.features.email).toMatchObject({ state: 'partial' }) + expect( + status.features.email.providers.find((provider) => provider.id === 'smtp') + ).toMatchObject({ + state: 'partial', + missingFields: ['SMTP_PORT'], + }) + expect(status.features.jobs).toMatchObject({ state: 'partial', providerId: 'trigger-dev' }) + expect(status.features.cache).toMatchObject({ state: 'invalid', providerId: 'redis' }) + expect(status.features.knowledge).toMatchObject({ + state: 'partial', + providerId: 'azure-mistral', + }) + expect(status.features.storage).toMatchObject({ state: 'default', providerId: 'local' }) + }) + + it('does not expose invalid selector values', () => { + const sensitiveValue = 'sensitive-selector-value' + const snapshots = [ + buildEnvCapabilityStatus({ STORAGE_PROVIDER: sensitiveValue }), + buildEnvCapabilityStatus({ SANDBOX_PROVIDER: sensitiveValue }), + buildEnvCapabilityStatus({ OCR_PROVIDER: sensitiveValue }), + ] + + for (const snapshot of snapshots) { + expect(JSON.stringify(snapshot)).not.toContain(sensitiveValue) + } + }) + + it('reports every OAuth client group as ready, partial, or absent', () => { + const status = buildEnvCapabilityStatus({ + GOOGLE_CLIENT_ID: 'google-id', + GOOGLE_CLIENT_SECRET: 'google-secret', + MICROSOFT_CLIENT_ID: 'microsoft-id', + }) + + expect(status.oauthClients.clients.google).toMatchObject({ + state: 'ready', + configuredFieldCount: 2, + requiredFieldCount: 2, + }) + expect(status.oauthClients.clients.microsoft).toMatchObject({ + state: 'partial', + configuredFieldCount: 1, + missingFields: ['MICROSOFT_CLIENT_SECRET'], + }) + expect(status.oauthClients.clients.slack).toMatchObject({ + state: 'absent', + configuredFieldCount: 0, + }) + expect(status.oauthClients.readyCount).toBe(1) + expect(status.oauthClients.partialCount).toBe(1) + expect(status.oauthClients.absentCount).toBe(Object.keys(OAUTH_CLIENT_CAPABILITIES).length - 2) + }) + + it('counts configured and effective LLM pool keys without exposing them', () => { + const status = buildEnvCapabilityStatus({ + OPENAI_API_KEY_1: 'openai-one', + OPENAI_API_KEY_3: 'openai-three', + FIREWORKS_API_KEY: 'fireworks-fallback', + FIREWORKS_API_KEY_1: 'fireworks-one', + }) + + expect(status.features.llm).toMatchObject({ + state: 'configured', + configuredPoolCount: 2, + configuredKeyCount: 4, + effectiveKeyCount: 3, + }) + expect(status.features.llm.pools.openai).toMatchObject({ + state: 'configured', + configuredKeyCount: 2, + effectiveKeyCount: 2, + fallbackKeyConfigured: false, + }) + expect(status.features.llm.pools.fireworks).toMatchObject({ + state: 'configured', + configuredKeyCount: 2, + effectiveKeyCount: 1, + fallbackKeyConfigured: true, + }) + expect(JSON.stringify(status)).not.toContain('openai-one') + expect(JSON.stringify(status)).not.toContain('fireworks-fallback') + }) + + it('counts singular runtime LLM keys as effective pool fallbacks', () => { + const status = buildEnvCapabilityStatus({ + OPENAI_API_KEY: 'openai-fallback', + GEMINI_API_KEY: 'gemini-fallback', + COHERE_API_KEY: 'cohere-fallback', + }) + + expect(status.features.llm).toMatchObject({ + state: 'configured', + configuredPoolCount: 3, + configuredKeyCount: 3, + effectiveKeyCount: 3, + }) + expect(status.features.llm.pools.openai).toMatchObject({ + state: 'configured', + fallbackKeyConfigured: true, + }) + expect(status.features.llm.pools.gemini).toMatchObject({ + state: 'configured', + fallbackKeyConfigured: true, + }) + expect(status.features.llm.pools.cohere).toMatchObject({ + state: 'configured', + fallbackKeyConfigured: true, + }) + expect(JSON.stringify(status)).not.toContain('openai-fallback') + expect(JSON.stringify(status)).not.toContain('gemini-fallback') + expect(JSON.stringify(status)).not.toContain('cohere-fallback') + }) + + it('reports non-Redis cache URLs and non-HTTP Azure OCR endpoints as invalid', () => { + const status = buildEnvCapabilityStatus({ + REDIS_URL: 'https://cache.example.com', + OCR_PROVIDER: 'azure-mistral', + OCR_AZURE_API_KEY: 'azure-key', + OCR_AZURE_ENDPOINT: 'ftp://ocr.example.com', + OCR_AZURE_MODEL_NAME: 'mistral-ocr', + }) + + expect(status.features.cache).toMatchObject({ state: 'invalid', providerId: 'redis' }) + expect(status.features.cache.issue?.message).toContain('redis:// or rediss://') + expect(status.features.knowledge).toMatchObject({ + state: 'invalid', + providerId: 'azure-mistral', + }) + expect(status.features.knowledge.issue?.message).toContain( + 'OCR_AZURE_ENDPOINT must be a valid HTTP(S) URL' + ) + }) +}) diff --git a/scripts/setup/capability-status.ts b/scripts/setup/capability-status.ts new file mode 100644 index 00000000000..5241e14c71e --- /dev/null +++ b/scripts/setup/capability-status.ts @@ -0,0 +1,481 @@ +/** + * Non-secret deployment-capability status derived from the same definitions the + * application uses at runtime. + * + * @packageDocumentation + */ +import { + ASYNC_JOBS_CAPABILITY, + CACHE_CAPABILITY, + EMAIL_CAPABILITY, + type EnvCapabilityConfigurationError, + type EnvCapabilityValues, + getCapabilityConfigurationError, + hasEnvCapabilityValue, + inspectCapability, + inspectOAuthClientCapability, + isTruthyEnvCapabilityValue, + LLM_KEY_POOLS, + OAUTH_CLIENT_CAPABILITIES, + type OAuthClientCapabilityId, + OCR_CAPABILITY, + type ProviderConfigurationState, + type ProviderInspection, + SANDBOX_CAPABILITY, + STORAGE_CAPABILITY, +} from '../../apps/sim/lib/core/config/env-capabilities.ts' +import { SETUP_FEATURES, type SetupFeatureId } from './capability-config.ts' + +export type SetupStatusFeatureId = Exclude +export type CapabilityStatusState = 'default' | 'configured' | 'missing' | 'partial' | 'invalid' + +export interface CapabilityStatusIssue { + state: Extract + message: string +} + +interface FeatureStatusBase { + id: TId + label: string + setupCommand: `bun run setup ${TId}` + state: CapabilityStatusState + issue?: CapabilityStatusIssue +} + +type EmailProviderId = (typeof EMAIL_CAPABILITY.providers)[number]['id'] +type StorageProviderId = + | (typeof STORAGE_CAPABILITY)['defaultProvider']['id'] + | (typeof STORAGE_CAPABILITY.providers)[number]['id'] +type LlmKeyPoolId = keyof typeof LLM_KEY_POOLS + +export interface EmailCapabilityStatus extends FeatureStatusBase<'email'> { + strategy: 'fallback' + providerIds: readonly EmailProviderId[] + providers: readonly ProviderInspection[] +} + +export interface StorageCapabilityStatus extends FeatureStatusBase<'storage'> { + strategy: 'selected' + providerId: StorageProviderId | null + defaultProviderId: (typeof STORAGE_CAPABILITY)['defaultProvider']['id'] + providers: readonly ProviderInspection<(typeof STORAGE_CAPABILITY.providers)[number]['id']>[] +} + +export interface SandboxCapabilityStatus extends FeatureStatusBase<'sandbox'> { + providerId: 'disabled' | 'e2b' | 'daytona' | null +} + +export interface JobsCapabilityStatus extends FeatureStatusBase<'jobs'> { + providerId: 'database' | 'trigger-dev' +} + +export interface CacheCapabilityStatus extends FeatureStatusBase<'cache'> { + providerId: 'database' | 'redis' +} + +export interface KnowledgeCapabilityStatus extends FeatureStatusBase<'knowledge'> { + providerId: 'local' | 'mistral' | 'azure-mistral' | null +} + +export interface LlmKeyPoolStatus { + id: LlmKeyPoolId + state: 'configured' | 'missing' + configuredKeyCount: number + effectiveKeyCount: number + fallbackKeyConfigured: boolean +} + +export interface LlmCapabilityStatus extends FeatureStatusBase<'llm'> { + pools: Readonly> + configuredPoolCount: number + configuredKeyCount: number + effectiveKeyCount: number +} + +interface FeatureStatusById { + email: EmailCapabilityStatus + storage: StorageCapabilityStatus + sandbox: SandboxCapabilityStatus + jobs: JobsCapabilityStatus + cache: CacheCapabilityStatus + knowledge: KnowledgeCapabilityStatus + llm: LlmCapabilityStatus +} + +export type EnvCapabilityFeatureStatuses = Pick + +export interface OAuthClientStatus { + id: OAuthClientCapabilityId + state: ProviderConfigurationState + configuredFieldCount: number + requiredFieldCount: number + missingFields: readonly string[] + setupCommand: string +} + +export interface OAuthClientStatuses { + clients: Readonly> + readyCount: number + partialCount: number + absentCount: number + invalidCount: number +} + +export interface EnvCapabilityStatusSnapshot { + features: EnvCapabilityFeatureStatuses + oauthClients: OAuthClientStatuses +} + +function featureMetadata(id: TId) { + const definition = SETUP_FEATURES.find((feature) => feature.id === id) + if (!definition) throw new Error(`Missing setup feature definition for ${id}`) + return { + id, + label: definition.label, + setupCommand: `bun run setup ${id}` as const, + } +} + +function readConfiguredString(values: EnvCapabilityValues, key: string): string | null { + if (!hasEnvCapabilityValue(values, key)) return null + const value = + values instanceof Map ? values.get(key) : (values as Readonly>)[key] + return String(value).trim().toLowerCase() +} + +function issue( + state: CapabilityStatusIssue['state'], + error: EnvCapabilityConfigurationError, + safeMessage = error.message +): CapabilityStatusIssue { + return { state, message: safeMessage } +} + +function brokenProviderState( + providers: readonly ProviderInspection[] +): Extract | null { + if (providers.some((provider) => provider.state === 'invalid')) return 'invalid' + if (providers.some((provider) => provider.state === 'partial')) return 'partial' + return null +} + +function inspectEmail(values: EnvCapabilityValues): EmailCapabilityStatus { + const inspection = inspectCapability(EMAIL_CAPABILITY, values) + const brokenState = brokenProviderState(inspection.providers) + const state = inspection.configured ? 'configured' : (brokenState ?? 'missing') + const configurationError = + inspection.error ?? getCapabilityConfigurationError(EMAIL_CAPABILITY, inspection.providers) + + return { + ...featureMetadata('email'), + strategy: 'fallback', + state, + providerIds: inspection.providerIds, + providers: inspection.providers, + ...(configurationError ? { issue: issue(brokenState ?? 'invalid', configurationError) } : {}), + } +} + +function inspectStorage(values: EnvCapabilityValues): StorageCapabilityStatus { + const inspection = inspectCapability(STORAGE_CAPABILITY, values) + + if (!inspection.error && inspection.providerId) { + const providerId = inspection.providerId as StorageProviderId + return { + ...featureMetadata('storage'), + strategy: 'selected', + state: providerId === STORAGE_CAPABILITY.defaultProvider.id ? 'default' : 'configured', + providerId, + defaultProviderId: STORAGE_CAPABILITY.defaultProvider.id, + providers: inspection.providers, + } + } + + const selectedId = readConfiguredString(values, STORAGE_CAPABILITY.selectorKey) + const selected = inspection.providers.find((provider) => provider.id === selectedId) + const state = + selected && !selected.active + ? 'missing' + : selected?.state === 'partial' + ? 'partial' + : selected?.state === 'invalid' + ? 'invalid' + : (brokenProviderState(inspection.providers) ?? 'invalid') + const error = inspection.error + if (!error) throw new Error('Storage resolution failed without a configuration error') + const providerId: StorageProviderId | null = + selectedId === STORAGE_CAPABILITY.defaultProvider.id + ? STORAGE_CAPABILITY.defaultProvider.id + : (selected?.id ?? null) + const safeMessage = + selectedId && !providerId + ? `Unknown ${STORAGE_CAPABILITY.selectorKey}. Expected one of: ${[ + STORAGE_CAPABILITY.defaultProvider.id, + ...STORAGE_CAPABILITY.providers.map((provider) => provider.id), + ].join(', ')}` + : error.message + + return { + ...featureMetadata('storage'), + strategy: 'selected', + state, + providerId, + defaultProviderId: STORAGE_CAPABILITY.defaultProvider.id, + providers: inspection.providers, + issue: issue(state, error, safeMessage), + } +} + +function inspectSandbox(values: EnvCapabilityValues): SandboxCapabilityStatus { + const inspection = inspectCapability(SANDBOX_CAPABILITY, values) + const requestedProvider = + readConfiguredString(values, SANDBOX_CAPABILITY.selectorKey) ?? + SANDBOX_CAPABILITY.defaultProvider.id + const providerId = + inspection.providerId === 'e2b' && !isTruthyEnvCapabilityValue(values, 'E2B_ENABLED') + ? 'disabled' + : inspection.providerId + + if (!inspection.error) { + const coherenceProblems: string[] = [] + if ( + isTruthyEnvCapabilityValue(values, 'E2B_ENABLED') !== + isTruthyEnvCapabilityValue(values, 'NEXT_PUBLIC_E2B_ENABLED') + ) { + coherenceProblems.push('E2B_ENABLED and NEXT_PUBLIC_E2B_ENABLED disagree') + } + const remoteAvailable = providerId !== null && providerId !== 'disabled' + if (remoteAvailable !== isTruthyEnvCapabilityValue(values, 'NEXT_PUBLIC_SANDBOX_ENABLED')) { + coherenceProblems.push('remote sandbox availability and NEXT_PUBLIC_SANDBOX_ENABLED disagree') + } + if (coherenceProblems.length > 0) { + return { + ...featureMetadata('sandbox'), + state: 'partial', + providerId, + issue: { + state: 'partial', + message: `${coherenceProblems.join('; ')}. Server and browser configuration must match.`, + }, + } + } + return { + ...featureMetadata('sandbox'), + state: providerId === 'disabled' ? 'default' : 'configured', + providerId, + } + } + + const knownProvider = requestedProvider === 'e2b' || requestedProvider === 'daytona' + const selectedProvider = inspection.providers.find( + (provider) => provider.id === requestedProvider + ) + const state = !knownProvider || selectedProvider?.state === 'invalid' ? 'invalid' : 'missing' + + return { + ...featureMetadata('sandbox'), + state, + providerId: knownProvider ? requestedProvider : null, + issue: issue( + state, + inspection.error, + knownProvider + ? inspection.error.message + : 'Unknown SANDBOX_PROVIDER. Expected one of: e2b, daytona' + ), + } +} + +function inspectJobs(values: EnvCapabilityValues): JobsCapabilityStatus { + const inspection = inspectCapability(ASYNC_JOBS_CAPABILITY, values) + if (!inspection.error && inspection.providerId) { + return { + ...featureMetadata('jobs'), + state: inspection.providerId === 'database' ? 'default' : 'configured', + providerId: inspection.providerId, + } + } + + if (!inspection.error) { + throw new Error('Async jobs inspection failed without a configuration error') + } + const configuredFieldCount = ['TRIGGER_SECRET_KEY', 'TRIGGER_PROJECT_ID'].filter((key) => + hasEnvCapabilityValue(values, key) + ).length + const state = configuredFieldCount === 0 ? 'missing' : 'partial' + return { + ...featureMetadata('jobs'), + state, + providerId: 'trigger-dev', + issue: issue(state, inspection.error), + } +} + +function inspectCache(values: EnvCapabilityValues): CacheCapabilityStatus { + const inspection = inspectCapability(CACHE_CAPABILITY, values) + if (!inspection.error && inspection.providerId) { + return { + ...featureMetadata('cache'), + state: inspection.providerId === 'database' ? 'default' : 'configured', + providerId: inspection.providerId, + } + } + + if (!inspection.error) { + throw new Error('Cache inspection failed without a configuration error') + } + const redis = inspection.providers.find((provider) => provider.id === 'redis') + const state: CapabilityStatusIssue['state'] = redis?.state === 'invalid' ? 'invalid' : 'partial' + return { + ...featureMetadata('cache'), + state, + providerId: 'redis', + issue: issue(state, inspection.error), + } +} + +function inspectKnowledge(values: EnvCapabilityValues): KnowledgeCapabilityStatus { + const inspection = inspectCapability(OCR_CAPABILITY, values) + if (!inspection.error && inspection.providerId) { + return { + ...featureMetadata('knowledge'), + state: inspection.providerId === 'local' ? 'default' : 'configured', + providerId: inspection.providerId, + } + } + + if (!inspection.error) { + throw new Error('OCR inspection failed without a configuration error') + } + const selected = readConfiguredString(values, OCR_CAPABILITY.selectorKey) + const azureFields = ['OCR_AZURE_API_KEY', 'OCR_AZURE_ENDPOINT', 'OCR_AZURE_MODEL_NAME'] as const + const azureFieldCount = azureFields.filter((key) => hasEnvCapabilityValue(values, key)).length + const knownProvider = + selected === null || + selected === 'local' || + selected === 'mistral' || + selected === 'azure-mistral' + const resolvedProvider = inspection.providerId + const selectedInspection = inspection.providers.find( + (provider) => provider.id === resolvedProvider + ) + const state = + !knownProvider || selectedInspection?.state === 'invalid' + ? 'invalid' + : selected === 'mistral' || selected === 'azure-mistral' + ? azureFieldCount === 0 && selected === 'azure-mistral' + ? 'missing' + : selected === 'mistral' && !hasEnvCapabilityValue(values, 'MISTRAL_API_KEY') + ? 'missing' + : 'partial' + : 'partial' + + return { + ...featureMetadata('knowledge'), + state, + providerId: + selected === 'local' || selected === 'mistral' || selected === 'azure-mistral' + ? selected + : selected === null + ? resolvedProvider + : null, + issue: issue( + state, + inspection.error, + knownProvider + ? inspection.error.message + : 'Unknown OCR_PROVIDER. Expected one of: local, mistral, azure-mistral' + ), + } +} + +function inspectLlm(values: EnvCapabilityValues): LlmCapabilityStatus { + const pools = {} as Record + let configuredPoolCount = 0 + let configuredKeyCount = 0 + let effectiveKeyCount = 0 + + for (const id of Object.keys(LLM_KEY_POOLS) as LlmKeyPoolId[]) { + const definition = LLM_KEY_POOLS[id] + const rotationKeyCount = definition.keys.filter((key) => + hasEnvCapabilityValue(values, key) + ).length + const fallbackKeyConfigured = + 'fallbackKey' in definition && hasEnvCapabilityValue(values, definition.fallbackKey) + const poolConfiguredKeyCount = rotationKeyCount + (fallbackKeyConfigured ? 1 : 0) + const poolEffectiveKeyCount = rotationKeyCount || (fallbackKeyConfigured ? 1 : 0) + const state = poolEffectiveKeyCount > 0 ? 'configured' : 'missing' + if (state === 'configured') configuredPoolCount += 1 + configuredKeyCount += poolConfiguredKeyCount + effectiveKeyCount += poolEffectiveKeyCount + pools[id] = { + id, + state, + configuredKeyCount: poolConfiguredKeyCount, + effectiveKeyCount: poolEffectiveKeyCount, + fallbackKeyConfigured, + } + } + + return { + ...featureMetadata('llm'), + state: configuredPoolCount > 0 ? 'configured' : 'missing', + pools, + configuredPoolCount, + configuredKeyCount, + effectiveKeyCount, + } +} + +function inspectOAuthClients(values: EnvCapabilityValues): OAuthClientStatuses { + const clients = {} as Record + let readyCount = 0 + let partialCount = 0 + let absentCount = 0 + let invalidCount = 0 + + for (const id of Object.keys(OAUTH_CLIENT_CAPABILITIES) as OAuthClientCapabilityId[]) { + const inspection = inspectOAuthClientCapability(id, values) + if (inspection.state === 'ready') readyCount += 1 + else if (inspection.state === 'partial') partialCount += 1 + else if (inspection.state === 'absent') absentCount += 1 + else invalidCount += 1 + + clients[id] = { + id, + state: inspection.state, + configuredFieldCount: OAUTH_CLIENT_CAPABILITIES[id].length - inspection.missingFields.length, + requiredFieldCount: OAUTH_CLIENT_CAPABILITIES[id].length, + missingFields: inspection.missingFields, + setupCommand: inspection.setupCommand, + } + } + + return { clients, readyCount, partialCount, absentCount, invalidCount } +} + +const FEATURE_STATUS_BUILDERS = { + email: inspectEmail, + storage: inspectStorage, + sandbox: inspectSandbox, + jobs: inspectJobs, + cache: inspectCache, + knowledge: inspectKnowledge, + llm: inspectLlm, +} satisfies Record unknown> + +/** Builds a status snapshot without returning any configured secret values. */ +export function buildEnvCapabilityStatus(values: EnvCapabilityValues): EnvCapabilityStatusSnapshot { + return { + features: { + email: FEATURE_STATUS_BUILDERS.email(values), + storage: FEATURE_STATUS_BUILDERS.storage(values), + sandbox: FEATURE_STATUS_BUILDERS.sandbox(values), + jobs: FEATURE_STATUS_BUILDERS.jobs(values), + cache: FEATURE_STATUS_BUILDERS.cache(values), + knowledge: FEATURE_STATUS_BUILDERS.knowledge(values), + llm: FEATURE_STATUS_BUILDERS.llm(values), + }, + oauthClients: inspectOAuthClients(values), + } +} diff --git a/scripts/setup/checks.ts b/scripts/setup/checks.ts index 1d1a196b598..3d50b5a2a1c 100644 --- a/scripts/setup/checks.ts +++ b/scripts/setup/checks.ts @@ -1,3 +1,19 @@ +import { + ASYNC_JOBS_CAPABILITY, + CACHE_CAPABILITY, + CORE_CONFIGURATION_KEYS, + EMAIL_CAPABILITY, + EnvCapabilityConfigurationError, + hasEnvCapabilityValue, + inspectCapability, + inspectOAuthClientCapability, + OAUTH_CLIENT_CAPABILITIES, + OCR_CAPABILITY, + requireCapability, + SANDBOX_CAPABILITY, + STORAGE_CAPABILITY, +} from '../../apps/sim/lib/core/config/env-capabilities.ts' +import { getSetupCommand } from './capability-config.ts' import { portOpen } from './detect.ts' import { type EnvFile, @@ -13,7 +29,7 @@ import { writeEnvValues, } from './env-files.ts' import { httpHealth, pgProbe, redisPing } from './probes.ts' -import { FLAG_TWINS, hasMailProvider, LOGIN_PROVIDERS } from './twins.ts' +import { FLAG_TWINS, LOGIN_PROVIDERS } from './twins.ts' export type CheckGroup = 'files' | 'schema' | 'consistency' | 'coherence' | 'live' export type CheckStatus = 'pass' | 'warn' | 'fail' | 'skip' @@ -65,15 +81,10 @@ export function loadCheckContext(live: boolean): CheckContext { return { env, layout, primary: layout === 'root' ? env.root : env.sim, live } } -const REQUIRED_KEYS: Partial> = { - sim: [ - 'DATABASE_URL', - 'BETTER_AUTH_SECRET', - 'BETTER_AUTH_URL', - 'NEXT_PUBLIC_APP_URL', - 'ENCRYPTION_KEY', - 'INTERNAL_API_SECRET', - ], +export const REQUIRED_APP_KEYS = CORE_CONFIGURATION_KEYS + +const REQUIRED_KEYS: Partial> = { + sim: REQUIRED_APP_KEYS, realtime: [ 'DATABASE_URL', 'BETTER_AUTH_URL', @@ -111,7 +122,11 @@ function checkFiles(ctx: CheckContext): Finding[] { for (const target of layoutTargets(ctx.layout)) { const file = ctx.env[target] if (file.exists) { - findings.push({ group: 'files', status: 'pass', message: `${rel(file)} exists` }) + findings.push({ + group: 'files', + status: 'pass', + message: `${rel(file)} exists`, + }) continue } const canSeed = target !== 'sim' && ctx.env.sim.exists @@ -232,7 +247,13 @@ function checkConsistency(ctx: CheckContext): Finding[] { // file has nothing to disagree with. if (ctx.layout !== 'split') { return ctx.layout === 'root' - ? [{ group: 'consistency', status: 'skip', message: 'single .env — nothing to mirror' }] + ? [ + { + group: 'consistency', + status: 'skip', + message: 'single .env — nothing to mirror', + }, + ] : [] } const findings: Finding[] = [] @@ -281,27 +302,38 @@ function checkCoherence(ctx: CheckContext): Finding[] { const findings: Finding[] = [] const sim = ctx.primary if (!sim.exists) return findings - if (isTruthy(sim.vars.get('TRIGGER_DEV_ENABLED'))) { - const missing = ['TRIGGER_SECRET_KEY', 'TRIGGER_PROJECT_ID'].filter((k) => !sim.vars.get(k)) - if (missing.length > 0) { - findings.push({ - group: 'coherence', - status: 'fail', - message: `TRIGGER_DEV_ENABLED is on but ${missing.join(' and ')} ${missing.length > 1 ? 'are' : 'is'} not set`, - fix: 'set the missing Trigger.dev vars or remove TRIGGER_DEV_ENABLED (jobs fall back to the DB queue)', - }) - } - } - const redisUrl = sim.vars.get('REDIS_URL') - if (redisUrl?.startsWith('rediss://')) { - const host = new URL(redisUrl).hostname - if (/^\d+\.\d+\.\d+\.\d+$/.test(host) && !sim.vars.get('REDIS_TLS_SERVERNAME')) { + const capabilityChecks = [ + { + command: getSetupCommand(ASYNC_JOBS_CAPABILITY.id), + resolve: () => requireCapability(ASYNC_JOBS_CAPABILITY, sim.vars), + }, + { + command: getSetupCommand(CACHE_CAPABILITY.id), + resolve: () => requireCapability(CACHE_CAPABILITY, sim.vars), + }, + { + command: getSetupCommand(SANDBOX_CAPABILITY.id), + resolve: () => { + const inspection = inspectCapability(SANDBOX_CAPABILITY, sim.vars) + if (inspection.error) throw inspection.error + return inspection + }, + }, + { + command: getSetupCommand(OCR_CAPABILITY.id), + resolve: () => requireCapability(OCR_CAPABILITY, sim.vars), + }, + ] + for (const check of capabilityChecks) { + try { + check.resolve() + } catch (error) { + if (!(error instanceof EnvCapabilityConfigurationError)) throw error findings.push({ group: 'coherence', status: 'fail', - message: - 'rediss:// with a bare IP host requires REDIS_TLS_SERVERNAME — the redis client throws without it', - fix: 'set REDIS_TLS_SERVERNAME to the certificate hostname', + message: error.message, + fix: check.command, }) } } @@ -321,50 +353,15 @@ function checkCoherence(ctx: CheckContext): Finding[] { // schema group already reports the invalid URL } } - const hasS3 = Boolean(sim.vars.get('AWS_REGION') && sim.vars.get('S3_BUCKET_NAME')) - const s3Partial = Boolean(sim.vars.get('AWS_REGION')) !== Boolean(sim.vars.get('S3_BUCKET_NAME')) - const hasAzure = Boolean( - sim.vars.get('AZURE_CONNECTION_STRING') || sim.vars.get('AZURE_ACCOUNT_NAME') - ) - const azurePartial = - Boolean(sim.vars.get('AZURE_ACCOUNT_NAME')) && - !sim.vars.get('AZURE_ACCOUNT_KEY') && - !sim.vars.get('AZURE_CONNECTION_STRING') - const hasGcs = Boolean(sim.vars.get('GCS_BUCKET_NAME')) - if (s3Partial) { + try { + requireCapability(STORAGE_CAPABILITY, sim.vars) + } catch (error) { + if (!(error instanceof EnvCapabilityConfigurationError)) throw error findings.push({ group: 'coherence', status: 'fail', - message: - 'S3 is half-configured (need BOTH AWS_REGION and S3_BUCKET_NAME) — storage silently falls back to local disk', - fix: 'set the missing var, or remove both to use local disk intentionally', - }) - } - if (azurePartial) { - findings.push({ - group: 'coherence', - status: 'fail', - message: - 'Azure storage is half-configured — AZURE_ACCOUNT_NAME needs AZURE_ACCOUNT_KEY (or use AZURE_CONNECTION_STRING)', - fix: 'set the missing credential, or remove the Azure vars', - }) - } - if (hasAzure && hasS3) { - findings.push({ - group: 'coherence', - status: 'warn', - message: - 'both Azure Blob and S3 are configured — Azure takes precedence, the S3 vars are ignored', - fix: 'remove the backend you are not using', - }) - } - if (hasGcs && (hasAzure || hasS3)) { - findings.push({ - group: 'coherence', - status: 'warn', - message: - 'GCS is configured alongside Azure/S3 — GCS is only used when neither of those is set', - fix: 'remove the backend you are not using', + message: error.message, + fix: getSetupCommand(STORAGE_CAPABILITY.id), }) } for (const { server, client } of FLAG_TWINS) { @@ -389,11 +386,13 @@ function checkCoherence(ctx: CheckContext): Finding[] { // under E2B_ENABLED or, when SANDBOX_PROVIDER=daytona, DAYTONA_API_KEY. Without // it the Function block hides its language dropdown and sandbox selector even // though the server would happily run Python. - const sandboxProvider = (sim.vars.get('SANDBOX_PROVIDER') || 'e2b').toLowerCase() + const sandboxProvider = inspectCapability(SANDBOX_CAPABILITY, sim.vars).providerId const remoteSandboxAvailable = sandboxProvider === 'daytona' - ? Boolean(sim.vars.get('DAYTONA_API_KEY')) - : isTruthy(sim.vars.get('E2B_ENABLED')) + ? hasEnvCapabilityValue(sim.vars, 'DAYTONA_API_KEY') + : sandboxProvider === 'e2b' + ? isTruthy(sim.vars.get('E2B_ENABLED')) + : false if (remoteSandboxAvailable && !isTruthy(sim.vars.get('NEXT_PUBLIC_SANDBOX_ENABLED'))) { findings.push({ group: 'coherence', @@ -421,19 +420,43 @@ function checkCoherence(ctx: CheckContext): Finding[] { }) } - if (isTruthy(sim.vars.get('EMAIL_VERIFICATION_ENABLED')) && !hasMailProvider(sim.vars)) { + const email = inspectCapability(EMAIL_CAPABILITY, sim.vars) + const emailConfigured = email.configured + if (email.error) { + findings.push({ + group: 'coherence', + status: 'fail', + message: email.error.message, + fix: getSetupCommand(EMAIL_CAPABILITY.id), + }) + } + if (isTruthy(sim.vars.get('EMAIL_VERIFICATION_ENABLED')) && !emailConfigured) { findings.push({ group: 'coherence', status: 'fail', message: - 'EMAIL_VERIFICATION_ENABLED is on but no mail provider is configured — verification emails only go to the console, locking out new users', - fix: 'configure RESEND_API_KEY / SMTP_* / AWS_SES_REGION, or turn verification off', + 'EMAIL_VERIFICATION_ENABLED is on but no mail provider is configured — the app must bypass verification to avoid locking out new users', + fix: `${getSetupCommand(EMAIL_CAPABILITY.id)}, or turn verification off`, + }) + } + + for (const providerId of Object.keys(OAUTH_CLIENT_CAPABILITIES)) { + const oauth = inspectOAuthClientCapability(providerId, sim.vars) + if (oauth.state !== 'partial' && oauth.state !== 'invalid') continue + findings.push({ + group: 'coherence', + status: 'fail', + message: `${providerId} OAuth is partially configured — missing ${oauth.missingFields.join(', ')}`, + fix: oauth.setupCommand, }) } const featureRules: Array<{ flag: string; needs: string[]; label: string }> = [ - { flag: 'BILLING_ENABLED', needs: ['STRIPE_SECRET_KEY'], label: 'billing' }, - { flag: 'E2B_ENABLED', needs: ['E2B_API_KEY'], label: 'E2B code execution' }, + { + flag: 'BILLING_ENABLED', + needs: ['STRIPE_SECRET_KEY'], + label: 'billing', + }, { flag: 'SSO_ENABLED', needs: ['SSO_ISSUER'], label: 'SSO' }, ] for (const rule of featureRules) { @@ -500,7 +523,11 @@ function checkCoherence(ctx: CheckContext): Finding[] { } if (findings.length === 0) { - findings.push({ group: 'coherence', status: 'pass', message: 'no conflicting settings' }) + findings.push({ + group: 'coherence', + status: 'pass', + message: 'no conflicting settings', + }) } return findings } @@ -525,7 +552,11 @@ async function checkDatabase(sim: EnvFile): Promise { fix: 'start Postgres (bun run setup can manage a pgvector container) or fix DATABASE_URL', }) } else { - findings.push({ group: 'live', status: 'pass', message: 'database reachable' }) + findings.push({ + group: 'live', + status: 'pass', + message: 'database reachable', + }) if (!probe.pgvectorAvailable) { findings.push({ group: 'live', @@ -534,7 +565,10 @@ async function checkDatabase(sim: EnvFile): Promise { fix: 'use the pgvector/pgvector:pg17 image or install the extension', }) } - const { applied, journal } = probe.migrations ?? { applied: null, journal: 0 } + const { applied, journal } = probe.migrations ?? { + applied: null, + journal: 0, + } if (applied === null) { findings.push({ group: 'live', @@ -569,8 +603,10 @@ async function checkDatabase(sim: EnvFile): Promise { } async function checkRedis(sim: EnvFile): Promise { + const inspection = inspectCapability(CACHE_CAPABILITY, sim.vars) + if (inspection.error || inspection.providerId !== 'redis') return [] const redisUrl = sim.vars.get('REDIS_URL') - if (!redisUrl) return [] + if (!redisUrl) throw new Error('Redis resolved as ready without REDIS_URL') const ping = await redisPing(redisUrl) return [ ping.ok @@ -586,10 +622,22 @@ async function checkRedis(sim: EnvFile): Promise { async function checkService(label: string, port: number, url: string): Promise { if (!(await portOpen(port))) { - return [{ group: 'live', status: 'skip', message: `${label}: not running on :${port}` }] + return [ + { + group: 'live', + status: 'skip', + message: `${label}: not running on :${port}`, + }, + ] } if (await httpHealth(url)) { - return [{ group: 'live', status: 'pass', message: `${label} healthy on :${port}` }] + return [ + { + group: 'live', + status: 'pass', + message: `${label} healthy on :${port}`, + }, + ] } return [ { diff --git a/scripts/setup/configuration-sources.test.ts b/scripts/setup/configuration-sources.test.ts new file mode 100644 index 00000000000..a30f47d55e6 --- /dev/null +++ b/scripts/setup/configuration-sources.test.ts @@ -0,0 +1,627 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, describe, expect, it } from 'bun:test' +import { + type ConfigurationCommandRunner, + type ConfigurationSourceDiscoveryOptions, + discoverConfigurationSources as discoverConfigurationSourcesFromEnvironment, + parseComposeFileEnvironment, + resolveKubernetesContainerEnvironment, +} from './configuration-sources.ts' + +const temporaryDirectories: string[] = [] + +function temporaryDirectory(): string { + const directory = mkdtempSync(path.join(tmpdir(), 'sim-configuration-sources-')) + temporaryDirectories.push(directory) + return directory +} + +function commandResult(status: number, stdout = '') { + return { status, stdout, stderr: '' } +} + +function discoverConfigurationSources(options: ConfigurationSourceDiscoveryOptions) { + return discoverConfigurationSourcesFromEnvironment({ processEnvironment: {}, ...options }) +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +describe('discoverConfigurationSources', () => { + it('enumerates split development and prepared Compose configuration separately', () => { + const root = temporaryDirectory() + mkdirSync(path.join(root, 'apps/sim'), { recursive: true }) + mkdirSync(path.join(root, 'apps/realtime'), { recursive: true }) + mkdirSync(path.join(root, 'packages/db'), { recursive: true }) + writeFileSync(path.join(root, 'apps/sim/.env'), 'RESEND_API_KEY=dev-email\n') + writeFileSync(path.join(root, 'apps/realtime/.env'), 'REDIS_URL=redis://localhost:6379\n') + writeFileSync(path.join(root, 'packages/db/.env'), 'DATABASE_URL=postgresql://dev\n') + writeFileSync(path.join(root, '.env'), 'RESEND_API_KEY=compose-email\n') + writeFileSync( + path.join(root, 'docker-compose.prod.yml'), + `services: + simstudio: + image: ghcr.io/simstudioai/simstudio:latest + env_file: + - .env + environment: + - DATABASE_URL=postgresql://postgres:postgres@db:5432/simstudio + - NEXT_PUBLIC_APP_URL=\${NEXT_PUBLIC_APP_URL:-http://localhost:3000} + - BETTER_AUTH_URL=\${NEXT_PUBLIC_APP_URL:-http://localhost:3000} + - REDIS_URL=\${REDIS_URL:-redis://redis:6379} +` + ) + const runner: ConfigurationCommandRunner = () => commandResult(1) + + const sources = discoverConfigurationSources({ root, runner }) + + expect(sources).toHaveLength(2) + expect(sources[0]).toMatchObject({ kind: 'dev', managedByCurrentCheckout: true }) + expect(sources[0].values?.get('RESEND_API_KEY')).toBe('dev-email') + expect(sources[0].values?.has('DATABASE_URL')).toBe(false) + expect(sources[1]).toMatchObject({ kind: 'compose', managedByCurrentCheckout: false }) + expect(sources[1].values?.get('RESEND_API_KEY')).toBe('compose-email') + expect(sources[1].values?.get('REDIS_URL')).toBe('redis://redis:6379') + expect(sources[1].values?.get('DATABASE_URL')).toBe( + 'postgresql://postgres:postgres@db:5432/simstudio' + ) + }) + + it('uses the last active value from a prepared Compose .env file', () => { + const root = temporaryDirectory() + writeFileSync(path.join(root, '.env'), 'RESEND_API_KEY=old\nRESEND_API_KEY=current\n') + writeFileSync( + path.join(root, 'docker-compose.prod.yml'), + 'services:\n simstudio:\n image: ghcr.io/simstudioai/simstudio:latest\n env_file: .env\n' + ) + + const sources = discoverConfigurationSources({ root, runner: () => commandResult(1) }) + + expect(sources).toHaveLength(1) + expect(sources[0].values?.get('RESEND_API_KEY')).toBe('current') + }) + + it('uses the effective environment of a stopped Compose app container', () => { + const parent = temporaryDirectory() + const root = path.join(parent, 'checkout') + const deployment = path.join(parent, 'deployment') + mkdirSync(root) + mkdirSync(deployment) + const composeFile = path.join(deployment, 'compose.yml') + writeFileSync( + composeFile, + 'services:\n simstudio:\n image: ghcr.io/simstudioai/simstudio:latest\n' + ) + const runner: ConfigurationCommandRunner = (command, args) => { + if (command === 'docker' && args[0] === 'info') return commandResult(0) + if (command === 'docker' && args[0] === 'compose' && args[1] === 'ls') { + return commandResult( + 0, + JSON.stringify([{ Name: 'external-sim', Status: 'exited', ConfigFiles: composeFile }]) + ) + } + if (command === 'docker' && args[0] === 'ps') return commandResult(0, 'container-id\n') + if (command === 'docker' && args[0] === 'inspect') { + return commandResult( + 0, + JSON.stringify([ + { + Created: '2026-01-01T00:00:00Z', + State: { Running: false }, + Config: { Env: ['RESEND_API_KEY=secret', 'REDIS_URL=redis://redis:6379'] }, + }, + ]) + ) + } + return commandResult(1) + } + + const sources = discoverConfigurationSources({ root, runner }) + + expect(sources).toHaveLength(1) + expect(sources[0]).toMatchObject({ + kind: 'compose', + label: 'Compose project "external-sim"', + managedByCurrentCheckout: false, + }) + expect(sources[0].values?.get('REDIS_URL')).toBe('redis://redis:6379') + expect(sources[0].values?.get('RESEND_API_KEY')).toBe('secret') + }) + + it('replaces the prepared root source with one unambiguous live project', () => { + const root = temporaryDirectory() + writeFileSync(path.join(root, '.env'), 'RESEND_API_KEY=prepared\n') + const composeFile = path.join(root, 'docker-compose.prod.yml') + writeFileSync( + composeFile, + 'services:\n simstudio:\n image: ghcr.io/simstudioai/simstudio:latest\n' + ) + const runner: ConfigurationCommandRunner = (command, args) => { + if (command === 'docker' && args[0] === 'info') return commandResult(0) + if (command === 'docker' && args[0] === 'compose' && args[1] === 'ls') { + return commandResult( + 0, + JSON.stringify([{ Name: 'current-sim', Status: 'running', ConfigFiles: composeFile }]) + ) + } + if (command === 'docker' && args[0] === 'ps') return commandResult(0, 'container-id\n') + if (command === 'docker' && args[0] === 'inspect') { + return commandResult( + 0, + JSON.stringify([ + { + Created: '2026-01-01T00:00:00Z', + State: { Running: true }, + Config: { Env: ['RESEND_API_KEY=effective'] }, + }, + ]) + ) + } + return commandResult(1) + } + + const sources = discoverConfigurationSources({ root, runner }) + + expect(sources).toHaveLength(1) + expect(sources[0]).toMatchObject({ + label: 'Compose project "current-sim"', + managedByCurrentCheckout: true, + }) + expect(sources[0].values?.get('RESEND_API_KEY')).toBe('effective') + }) + + it('does not claim a live current-checkout project is setup-managed without root .env', () => { + const root = temporaryDirectory() + const composeFile = path.join(root, 'docker-compose.prod.yml') + writeFileSync( + composeFile, + 'services:\n simstudio:\n image: ghcr.io/simstudioai/simstudio:latest\n' + ) + const runner: ConfigurationCommandRunner = (command, args) => { + if (command === 'docker' && args[0] === 'info') return commandResult(0) + if (command === 'docker' && args[0] === 'compose' && args[1] === 'ls') { + return commandResult( + 0, + JSON.stringify([{ Name: 'current-sim', Status: 'running', ConfigFiles: composeFile }]) + ) + } + if (command === 'docker' && args[0] === 'ps') return commandResult(0, 'container-id\n') + if (command === 'docker' && args[0] === 'inspect') { + return commandResult( + 0, + JSON.stringify([{ State: { Running: true }, Config: { Env: ['NODE_ENV=production'] } }]) + ) + } + return commandResult(1) + } + + const sources = discoverConfigurationSources({ root, runner }) + + expect(sources).toHaveLength(1) + expect(sources[0].managedByCurrentCheckout).toBe(false) + }) + + it('loads development env files with Next precedence', () => { + const root = temporaryDirectory() + const appDirectory = path.join(root, 'apps/sim') + mkdirSync(appDirectory, { recursive: true }) + writeFileSync(path.join(appDirectory, '.env'), 'STATUS_PRECEDENCE=base\n') + writeFileSync(path.join(appDirectory, '.env.development'), 'STATUS_PRECEDENCE=development\n') + writeFileSync(path.join(appDirectory, '.env.local'), 'STATUS_PRECEDENCE=local\n') + writeFileSync( + path.join(appDirectory, '.env.development.local'), + 'STATUS_PRECEDENCE=development-local\n' + ) + + const sources = discoverConfigurationSources({ root, runner: () => commandResult(1) }) + + expect(sources).toHaveLength(1) + expect(sources[0].values?.get('STATUS_PRECEDENCE')).toBe('development-local') + expect(sources[0].location).toContain('.env.development.local') + expect(sources[0].location).toContain('process environment') + expect(sources[0].managedByCurrentCheckout).toBe(false) + }) + + it('does not offer setup writes when a process-only capability value wins', () => { + const root = temporaryDirectory() + const appDirectory = path.join(root, 'apps/sim') + mkdirSync(appDirectory, { recursive: true }) + writeFileSync(path.join(appDirectory, '.env'), 'SLACK_CLIENT_SECRET=file-secret\n') + const sources = discoverConfigurationSources({ + root, + runner: () => commandResult(1), + processEnvironment: { SLACK_CLIENT_ID: 'process-client-id' }, + }) + + expect(sources[0].values?.get('SLACK_CLIENT_ID')).toBe('process-client-id') + expect(sources[0].managedByCurrentCheckout).toBe(false) + }) + + it('reports missing files in the split development configuration', () => { + const root = temporaryDirectory() + const appDirectory = path.join(root, 'apps/sim') + mkdirSync(appDirectory, { recursive: true }) + writeFileSync( + path.join(appDirectory, '.env'), + [ + 'DATABASE_URL=postgresql://localhost/sim', + `BETTER_AUTH_SECRET=${'a'.repeat(32)}`, + 'BETTER_AUTH_URL=http://localhost:3000', + 'NEXT_PUBLIC_APP_URL=http://localhost:3000', + `ENCRYPTION_KEY=${'b'.repeat(64)}`, + `INTERNAL_API_SECRET=${'c'.repeat(32)}`, + ].join('\n') + ) + + const sources = discoverConfigurationSources({ root, runner: () => commandResult(1) }) + + expect(sources).toHaveLength(1) + expect(sources[0].configurationIssues).toEqual( + expect.arrayContaining(['apps/realtime/.env is missing', 'packages/db/.env is missing']) + ) + }) + + it('reports shared-value drift across split development env files', () => { + const root = temporaryDirectory() + mkdirSync(path.join(root, 'apps/sim'), { recursive: true }) + mkdirSync(path.join(root, 'apps/realtime'), { recursive: true }) + mkdirSync(path.join(root, 'packages/db'), { recursive: true }) + writeFileSync( + path.join(root, 'apps/sim/.env'), + [ + 'DATABASE_URL=postgresql://localhost/sim', + `BETTER_AUTH_SECRET=${'a'.repeat(32)}`, + 'BETTER_AUTH_URL=http://localhost:3000', + 'NEXT_PUBLIC_APP_URL=http://localhost:3000', + `ENCRYPTION_KEY=${'b'.repeat(64)}`, + `INTERNAL_API_SECRET=${'c'.repeat(32)}`, + ].join('\n') + ) + writeFileSync( + path.join(root, 'apps/realtime/.env'), + [ + 'DATABASE_URL=postgresql://localhost/sim', + `BETTER_AUTH_SECRET=${'d'.repeat(32)}`, + 'BETTER_AUTH_URL=http://localhost:3000', + 'NEXT_PUBLIC_APP_URL=http://localhost:3000', + `INTERNAL_API_SECRET=${'c'.repeat(32)}`, + ].join('\n') + ) + writeFileSync(path.join(root, 'packages/db/.env'), 'DATABASE_URL=postgresql://localhost/other') + + const sources = discoverConfigurationSources({ root, runner: () => commandResult(1) }) + + expect(sources[0].configurationIssues).toEqual( + expect.arrayContaining([ + 'BETTER_AUTH_SECRET differs between apps/sim/.env and apps/realtime/.env', + 'DATABASE_URL differs between apps/sim/.env and packages/db/.env', + ]) + ) + }) + + it('identifies a Helm release by current context, namespace, and release', () => { + const root = temporaryDirectory() + const runner: ConfigurationCommandRunner = (command, args) => { + if (command === 'helm') { + if (args.includes('-a') || !args.includes('--deployed') || !args.includes('--failed')) { + return commandResult(1) + } + if (args.at(-2) !== '--kube-context' || args.at(-1) !== 'production-cluster') { + return commandResult(1) + } + return commandResult( + 0, + JSON.stringify([{ name: 'sim-prod', namespace: 'production', chart: 'sim-1.2.3' }]) + ) + } + if (command === 'kubectl' && args[0] === 'config') { + return commandResult(0, 'production-cluster\n') + } + if (!args.includes('--context') || !args.includes('production-cluster')) { + return commandResult(1) + } + if (command === 'kubectl' && args[1] === 'deployments') { + return commandResult( + 0, + JSON.stringify({ + items: [ + { + spec: { + template: { + spec: { + containers: [ + { + name: 'app', + env: [{ name: 'NEXT_PUBLIC_APP_URL', value: 'https://sim.example.com' }], + }, + ], + }, + }, + }, + }, + ], + }) + ) + } + return commandResult(1) + } + + const sources = discoverConfigurationSources({ root, runner }) + + expect(sources).toHaveLength(1) + expect(sources[0]).toMatchObject({ + kind: 'helm', + label: 'Helm release "sim-prod"', + location: 'context production-cluster · namespace production · release sim-prod', + managedByCurrentCheckout: false, + }) + expect(sources[0].values?.get('NEXT_PUBLIC_APP_URL')).toBe('https://sim.example.com') + }) + + it('keeps every Compose config file after one file identifies Sim', () => { + const parent = temporaryDirectory() + const root = path.join(parent, 'checkout') + const deployment = path.join(parent, 'deployment') + mkdirSync(root) + mkdirSync(deployment) + const baseFile = path.join(deployment, 'compose.yml') + const overrideFile = path.join(deployment, 'compose.override.yml') + writeFileSync( + baseFile, + 'services:\n simstudio:\n image: ghcr.io/simstudioai/simstudio:latest\n' + ) + writeFileSync( + overrideFile, + 'services:\n simstudio:\n environment:\n - REDIS_URL=redis://override\n' + ) + let renderedWithOverride = false + const runner: ConfigurationCommandRunner = (command, args) => { + if (command === 'docker' && args[0] === 'info') return commandResult(0) + if (command === 'docker' && args[0] === 'compose' && args[1] === 'ls') { + return commandResult( + 0, + JSON.stringify([ + { + Name: 'external-sim', + Status: 'running', + ConfigFiles: `${baseFile},${overrideFile}`, + }, + ]) + ) + } + if (command === 'docker' && args[0] === 'ps') return commandResult(0) + if (command === 'docker' && args[0] === 'compose' && args.includes('config')) { + renderedWithOverride = args.includes(overrideFile) + return commandResult( + 0, + JSON.stringify({ + services: { simstudio: { environment: { REDIS_URL: 'redis://override' } } }, + }) + ) + } + return commandResult(1) + } + + const sources = discoverConfigurationSources({ root, runner }) + + expect(renderedWithOverride).toBe(true) + expect(sources[0].location).toContain(overrideFile) + expect(sources[0].values?.get('REDIS_URL')).toBe('redis://override') + }) + + it('fails fast when Docker Compose returns malformed discovery output', () => { + const root = temporaryDirectory() + const runner: ConfigurationCommandRunner = (command, args) => { + if (command === 'docker' && args[0] === 'info') return commandResult(0) + if (command === 'docker' && args[0] === 'compose' && args[1] === 'ls') { + return commandResult(0, 'not-json') + } + return commandResult(1) + } + + expect(() => discoverConfigurationSources({ root, runner })).toThrow( + 'Docker Compose returned an invalid project list' + ) + }) + + it('fails fast when a Docker Compose project entry changes shape', () => { + const root = temporaryDirectory() + const runner: ConfigurationCommandRunner = (command, args) => { + if (command === 'docker' && args[0] === 'info') return commandResult(0) + if (command === 'docker' && args[0] === 'compose' && args[1] === 'ls') { + return commandResult(0, JSON.stringify([{ Name: 'sim-without-config-files' }])) + } + return commandResult(1) + } + + expect(() => discoverConfigurationSources({ root, runner })).toThrow( + 'Docker Compose returned an invalid project list' + ) + }) + + it('does not treat unrelated Docker tooling as a Sim configuration source', () => { + const root = temporaryDirectory() + const runner: ConfigurationCommandRunner = (command, args) => { + if (command === 'docker' && args[0] === '--version') return commandResult(0) + return commandResult(1) + } + + const sources = discoverConfigurationSources({ root, runner }) + + expect(sources).toHaveLength(0) + }) + + it('does not treat an unrelated Kubernetes context as a Sim configuration source', () => { + const root = temporaryDirectory() + const runner: ConfigurationCommandRunner = (command, args) => { + if (command === 'kubectl' && args[0] === 'config') { + return commandResult(0, 'production-cluster\n') + } + return commandResult(1) + } + + const sources = discoverConfigurationSources({ root, runner }) + + expect(sources).toHaveLength(0) + }) + + it('does not substitute desired Compose values when a known container cannot be inspected', () => { + const root = temporaryDirectory() + const composeFile = path.join(root, 'docker-compose.prod.yml') + writeFileSync( + composeFile, + 'services:\n simstudio:\n image: ghcr.io/simstudioai/simstudio:latest\n' + ) + const runner: ConfigurationCommandRunner = (command, args) => { + if (command === 'docker' && args[0] === 'info') return commandResult(0) + if (command === 'docker' && args[0] === 'compose' && args[1] === 'ls') { + return commandResult( + 0, + JSON.stringify([{ Name: 'known-sim', Status: 'running', ConfigFiles: composeFile }]) + ) + } + if (command === 'docker' && args[0] === 'ps') return commandResult(0, 'container-id\n') + if (command === 'docker' && args[0] === 'inspect') return commandResult(1) + if (command === 'docker' && args[0] === 'compose') { + return commandResult( + 0, + JSON.stringify({ + services: { simstudio: { environment: { RESEND_API_KEY: 'desired' } } }, + }) + ) + } + return commandResult(1) + } + + const sources = discoverConfigurationSources({ root, runner }) + + expect(sources).toHaveLength(1) + expect(sources[0].values).toBeNull() + expect(sources[0].warning).toContain('could not be inspected') + }) + + it('reports a known Compose project as unknown when its containers cannot be enumerated', () => { + const root = temporaryDirectory() + const composeFile = path.join(root, 'docker-compose.prod.yml') + writeFileSync( + composeFile, + 'services:\n simstudio:\n image: ghcr.io/simstudioai/simstudio:latest\n' + ) + const runner: ConfigurationCommandRunner = (command, args) => { + if (command === 'docker' && args[0] === 'info') return commandResult(0) + if (command === 'docker' && args[0] === 'compose' && args[1] === 'ls') { + return commandResult( + 0, + JSON.stringify([{ Name: 'known-sim', Status: 'running', ConfigFiles: composeFile }]) + ) + } + if (command === 'docker' && args[0] === 'ps') return commandResult(1) + return commandResult(1) + } + + const sources = discoverConfigurationSources({ root, runner }) + + expect(sources).toHaveLength(1) + expect(sources[0].values).toBeNull() + expect(sources[0].warning).toContain('could not be enumerated') + }) + + it('fails fast when a Helm release entry changes shape', () => { + const root = temporaryDirectory() + const runner: ConfigurationCommandRunner = (command, args) => { + if (command === 'kubectl' && args[0] === 'config') { + return commandResult(0, 'production-cluster\n') + } + if (command === 'helm' && args[0] === 'list') { + return commandResult(0, JSON.stringify([{ name: 'sim-prod', namespace: 'production' }])) + } + return commandResult(1) + } + + expect(() => discoverConfigurationSources({ root, runner })).toThrow( + 'Helm returned an invalid release list' + ) + }) +}) + +describe('parseComposeFileEnvironment', () => { + it('does not inject root .env when the service has no env_file', () => { + const values = parseComposeFileEnvironment( + 'services:\n simstudio:\n image: ghcr.io/simstudioai/simstudio:latest\n environment:\n - REDIS_URL=redis://redis:6379\n', + new Map([['RESEND_API_KEY', 'must-not-be-injected']]) + ) + + expect(values?.get('REDIS_URL')).toBe('redis://redis:6379') + expect(values?.has('RESEND_API_KEY')).toBe(false) + }) + + it('returns unknown for unsupported inline environment syntax', () => { + const values = parseComposeFileEnvironment( + 'services:\n simstudio:\n image: ghcr.io/simstudioai/simstudio:latest\n env_file: .env\n environment: { RESEND_API_KEY: override }\n', + new Map([['RESEND_API_KEY', 'root-value']]) + ) + + expect(values).toBeNull() + }) +}) + +describe('resolveKubernetesContainerEnvironment', () => { + it('applies envFrom order and then explicit env/valueFrom precedence', () => { + const resources = new Map([ + [ + 'secret/app-secret', + { + data: { + SHARED: Buffer.from('secret').toString('base64'), + SECRET_ONLY: Buffer.from('secret-only').toString('base64'), + EXPLICIT_SOURCE: Buffer.from('from-secret-key').toString('base64'), + }, + }, + ], + ['configmap/app-config', { data: { SHARED: 'configmap', CONFIG_ONLY: 'config-only' } }], + ]) + const resolution = resolveKubernetesContainerEnvironment( + { + envFrom: [{ secretRef: { name: 'app-secret' } }, { configMapRef: { name: 'app-config' } }], + env: [ + { name: 'SHARED', value: 'explicit' }, + { + name: 'FROM_SECRET', + valueFrom: { secretKeyRef: { name: 'app-secret', key: 'EXPLICIT_SOURCE' } }, + }, + ], + }, + (kind, name) => { + const resource = resources.get(`${kind}/${name}`) + return resource ? { state: 'found', resource } : { state: 'missing' } + } + ) + + expect(resolution.warning).toBeUndefined() + expect(resolution.values).toEqual( + new Map([ + ['SHARED', 'explicit'], + ['SECRET_ONLY', 'secret-only'], + ['EXPLICIT_SOURCE', 'from-secret-key'], + ['CONFIG_ONLY', 'config-only'], + ['FROM_SECRET', 'from-secret-key'], + ]) + ) + }) + + it('returns unknown instead of claiming missing configuration when a Secret is inaccessible', () => { + const resolution = resolveKubernetesContainerEnvironment( + { envFrom: [{ secretRef: { name: 'restricted-secret' } }] }, + () => ({ state: 'inaccessible' }) + ) + + expect(resolution.values).toBeNull() + expect(resolution.warning).toContain('RBAC') + expect(resolution.warning).not.toContain('secret-value') + }) +}) diff --git a/scripts/setup/configuration-sources.ts b/scripts/setup/configuration-sources.ts new file mode 100644 index 00000000000..296ab2f4217 --- /dev/null +++ b/scripts/setup/configuration-sources.ts @@ -0,0 +1,1072 @@ +import { spawnSync } from 'node:child_process' +import { existsSync, readFileSync, realpathSync } from 'node:fs' +import path from 'node:path' +import { loadEnvConfig } from '@next/env' +import { + CORE_CONFIGURATION_KEYS, + DEPLOYMENT_CONFIGURATION_KEYS, +} from '../../apps/sim/lib/core/config/env-capabilities.ts' +import { isPlaceholder, parseEnv, ROOT, SHARED_KEYS } from './env-files.ts' + +export type ConfigurationSourceKind = 'dev' | 'compose' | 'helm' + +export interface ConfigurationSource { + kind: ConfigurationSourceKind + label: string + location: string + values: Map | null + warning?: string + configurationIssues?: readonly string[] + managedByCurrentCheckout: boolean +} + +export interface ConfigurationCommandResult { + status: number | null + stdout: string + stderr: string +} + +export type ConfigurationCommandRunner = ( + command: string, + args: readonly string[], + cwd: string +) => ConfigurationCommandResult + +export interface ConfigurationSourceDiscoveryOptions { + root?: string + runner?: ConfigurationCommandRunner + processEnvironment?: Readonly +} + +export type KubernetesResourceKind = 'secret' | 'configmap' + +export type KubernetesResourceLookupResult = + | { state: 'found'; resource: unknown } + | { state: 'missing' } + | { state: 'inaccessible' } + +export type KubernetesResourceLookup = ( + kind: KubernetesResourceKind, + name: string +) => KubernetesResourceLookupResult + +export interface EnvironmentResolution { + values: Map | null + warning?: string +} + +interface ComposeProject { + name: string + configFiles: string[] +} + +interface DiscoveredComposeProject { + project: ComposeProject + configFiles: string[] + source: ConfigurationSource +} + +interface ComposeDiscovery { + projects: DiscoveredComposeProject[] +} + +interface HelmRelease { + name: string + namespace: string + chart: string +} + +const SIM_COMPOSE_MARKERS = ['ghcr.io/simstudioai/simstudio', 'docker/app.Dockerfile'] as const + +const COMPOSE_FILES = ['docker-compose.prod.yml', 'docker-compose.local.yml'] as const +const DEVELOPMENT_ENV_FILES = [ + '.env.development.local', + '.env.local', + '.env.development', + '.env', +] as const +const DISCOVERY_COMMAND_TIMEOUT_MS = 10_000 +const DEPLOYMENT_CONFIGURATION_KEY_SET = new Set(DEPLOYMENT_CONFIGURATION_KEYS) +const SPLIT_REQUIRED_KEYS = { + sim: CORE_CONFIGURATION_KEYS, + realtime: [ + 'DATABASE_URL', + 'BETTER_AUTH_URL', + 'BETTER_AUTH_SECRET', + 'INTERNAL_API_SECRET', + 'NEXT_PUBLIC_APP_URL', + ], + db: ['DATABASE_URL'], +} as const + +function defaultRunner( + command: string, + args: readonly string[], + cwd: string +): ConfigurationCommandResult { + const result = spawnSync(command, [...args], { + cwd, + encoding: 'utf8', + timeout: DISCOVERY_COMMAND_TIMEOUT_MS, + }) + return { + status: result.status, + stdout: typeof result.stdout === 'string' ? result.stdout : '', + stderr: typeof result.stderr === 'string' ? result.stderr : '', + } +} + +function runSafely( + runner: ConfigurationCommandRunner, + command: string, + args: readonly string[], + cwd: string +): ConfigurationCommandResult { + try { + return runner(command, args, cwd) + } catch { + return { status: null, stdout: '', stderr: '' } + } +} + +function asRecord(value: unknown): Record | null { + return typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as Record) + : null +} + +function parseJson(value: string): unknown | null { + try { + return JSON.parse(value) as unknown + } catch { + return null + } +} + +function canonicalPath(value: string): string { + const resolved = path.resolve(value) + try { + return realpathSync(resolved) + } catch { + return resolved + } +} + +function readEnv(file: string): Map { + return parseEnv(readFileSync(file, 'utf8')) +} + +function filterConfigurationValues(values: ReadonlyMap): Map { + return new Map([...values].filter(([key]) => DEPLOYMENT_CONFIGURATION_KEY_SET.has(key))) +} + +function inspectSplitConfiguration(root: string): string[] { + const targets = [ + { id: 'sim', label: 'apps/sim/.env', file: path.join(root, 'apps/sim/.env') }, + { + id: 'realtime', + label: 'apps/realtime/.env', + file: path.join(root, 'apps/realtime/.env'), + }, + { id: 'db', label: 'packages/db/.env', file: path.join(root, 'packages/db/.env') }, + ] as const + const issues: string[] = [] + const values = new Map<(typeof targets)[number]['id'], Map>() + + for (const target of targets) { + if (!existsSync(target.file)) { + issues.push(`${target.label} is missing`) + continue + } + const targetValues = readEnv(target.file) + values.set(target.id, targetValues) + for (const key of SPLIT_REQUIRED_KEYS[target.id]) { + const value = targetValues.get(key) + if (!value || isPlaceholder(value)) { + issues.push(`${target.label}: ${key} is missing, empty, or a placeholder`) + } + } + } + + const sim = values.get('sim') + const realtime = values.get('realtime') + if (sim && realtime) { + for (const key of SHARED_KEYS) { + const simValue = sim.get(key) + const realtimeValue = realtime.get(key) + if (simValue && realtimeValue && simValue !== realtimeValue) { + issues.push(`${key} differs between apps/sim/.env and apps/realtime/.env`) + } + } + } + + const db = values.get('db') + const simDatabaseUrl = sim?.get('DATABASE_URL') + const dbDatabaseUrl = db?.get('DATABASE_URL') + if (simDatabaseUrl && dbDatabaseUrl && simDatabaseUrl !== dbDatabaseUrl) { + issues.push('DATABASE_URL differs between apps/sim/.env and packages/db/.env') + } + return issues +} + +function restoreProcessEnvironment(original: NodeJS.ProcessEnv): void { + for (const key of Object.keys(process.env)) { + if (!(key in original)) delete process.env[key] + } + Object.assign(process.env, original) +} + +function loadDevelopmentEnvironment( + appDirectory: string, + processEnvironment: Readonly +): { + values: Map + loadedFiles: string[] + hasProcessOverrides: boolean +} { + const originalEnvironment = { ...process.env } + const configuredEnvironment = { ...processEnvironment } + let loadFailed = false + try { + restoreProcessEnvironment(configuredEnvironment) + process.env.NODE_ENV = 'development' + const result = loadEnvConfig( + appDirectory, + true, + { + info: () => undefined, + error: () => { + loadFailed = true + }, + }, + true + ) + if (loadFailed) { + throw new Error('One or more development environment files could not be loaded') + } + const loadedKeys = new Set(result.loadedEnvFiles.flatMap((file) => Object.keys(file.env))) + const values = new Map() + for (const key of loadedKeys) { + const loadedValue = result.combinedEnv[key] + if (loadedValue !== undefined) values.set(key, loadedValue) + } + for (const key of DEPLOYMENT_CONFIGURATION_KEYS) { + const processValue = configuredEnvironment[key] + if (processValue !== undefined) values.set(key, processValue) + } + const hasProcessOverrides = DEPLOYMENT_CONFIGURATION_KEYS.some( + (key) => configuredEnvironment[key] !== undefined + ) + return { + values, + loadedFiles: result.loadedEnvFiles.map((file) => path.join(appDirectory, file.path)), + hasProcessOverrides, + } + } finally { + restoreProcessEnvironment(originalEnvironment) + } +} + +function hasSimComposeMarker(file: string): boolean { + try { + const contents = readFileSync(file, 'utf8') + return SIM_COMPOSE_MARKERS.some((marker) => contents.includes(marker)) + } catch { + return false + } +} + +/** Parses Docker's `KEY=value` environment representation without logging values. */ +export function parseEnvironmentEntries(entries: readonly unknown[]): Map | null { + const values = new Map() + for (const entry of entries) { + if (typeof entry !== 'string') return null + const separator = entry.indexOf('=') + if (separator <= 0) return null + values.set(entry.slice(0, separator), entry.slice(separator + 1)) + } + return values +} + +/** Extracts the newest running, or newest stopped, container's effective environment. */ +export function parseDockerInspectEnvironment(output: string): Map | null { + const parsed = parseJson(output) + if (!Array.isArray(parsed)) return null + + const candidates: Array<{ + running: boolean + created: string + values: Map + }> = [] + for (const item of parsed) { + const record = asRecord(item) + const config = asRecord(record?.Config) + if (!config || !Array.isArray(config.Env)) continue + const values = parseEnvironmentEntries(config.Env) + if (!values) continue + const state = asRecord(record?.State) + candidates.push({ + running: state?.Running === true, + created: typeof record?.Created === 'string' ? record.Created : '', + values, + }) + } + + candidates.sort((left, right) => { + if (left.running !== right.running) return left.running ? -1 : 1 + return right.created.localeCompare(left.created) + }) + return candidates[0]?.values ?? null +} + +function parseComposeProjects(output: string): ComposeProject[] | null { + const parsed = parseJson(output) + if (!Array.isArray(parsed)) return null + const projects: ComposeProject[] = [] + for (const item of parsed) { + const record = asRecord(item) + const name = record?.Name + const configFiles = record?.ConfigFiles + if (typeof name !== 'string' || typeof configFiles !== 'string') return null + const files = configFiles + .split(',') + .map((file) => file.trim()) + .filter(Boolean) + if (files.length === 0) return null + projects.push({ name, configFiles: files }) + } + return projects +} + +/** Reads `docker compose config --format json` and returns the resolved app environment. */ +export function parseComposeConfigEnvironment(output: string): Map | null { + const parsed = asRecord(parseJson(output)) + const services = asRecord(parsed?.services) + const app = asRecord(services?.simstudio) + if (!app) return null + if (Array.isArray(app.environment)) return parseEnvironmentEntries(app.environment) + + const environment = asRecord(app.environment) + if (!environment) return null + const values = new Map() + for (const [key, value] of Object.entries(environment)) { + if (value === null) continue + if (typeof value !== 'string') return null + values.set(key, value) + } + return values +} + +function stripYamlScalar(value: string): string { + const trimmed = value.trim() + if (trimmed.length >= 2) { + const first = trimmed[0] + const last = trimmed[trimmed.length - 1] + if ((first === '"' && last === '"') || (first === "'" && last === "'")) { + return trimmed.slice(1, -1) + } + } + return trimmed +} + +function normalizeComposeEnvFilePath(value: string): string { + const withoutPathKey = value.replace(/^path:\s*/, '') + return stripYamlScalar(withoutPathKey).replace(/^\.\//, '') +} + +function serviceRootEnvFileUsage( + lines: readonly string[], + serviceStart: number, + serviceEnd: number +): boolean | null { + const envFileStart = lines.findIndex( + (line, index) => index > serviceStart && index < serviceEnd && /^ {4}env_file:\s*/.test(line) + ) + if (envFileStart === -1) return false + + const inline = /^ {4}env_file:\s*(.+)$/.exec(lines[envFileStart]) + if (inline) return normalizeComposeEnvFilePath(inline[1]) === '.env' ? true : null + + const referencedFiles: string[] = [] + for (let index = envFileStart + 1; index < serviceEnd; index += 1) { + const line = lines[index] + if (/^ {4}\S/.test(line)) break + const listEntry = /^ {6}-\s+(.+)$/.exec(line) + const nestedPath = /^ {8}path:\s*(.+)$/.exec(line) + if (listEntry) referencedFiles.push(normalizeComposeEnvFilePath(listEntry[1])) + else if (nestedPath) referencedFiles.push(normalizeComposeEnvFilePath(nestedPath[1])) + } + if (referencedFiles.length !== 1) return null + return referencedFiles[0] === '.env' ? true : null +} + +function expandComposeValue(raw: string, variables: ReadonlyMap): string | null { + let valid = true + const expanded = raw.replace( + /\$\{([A-Za-z_][A-Za-z0-9_]*)(?:(:-|-|:\+|\+|:\?|\?)([^}]*))?\}/g, + (_match, key: string, operator: string | undefined, operand: string | undefined) => { + const value = variables.get(key) + const set = value !== undefined + const nonEmpty = set && value !== '' + if (!operator) return value ?? '' + if (operator === ':-') return nonEmpty ? value : (operand ?? '') + if (operator === '-') return set ? value : (operand ?? '') + if (operator === ':+') return nonEmpty ? (operand ?? '') : '' + if (operator === '+') return set ? (operand ?? '') : '' + if (operator === ':?' && !nonEmpty) { + valid = false + return '' + } + if (operator === '?' && !set) { + valid = false + return '' + } + return value ?? '' + } + ) + if (!valid || expanded.includes('${')) return null + return expanded.replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (_match, key: string) => { + return variables.get(key) ?? '' + }) +} + +/** + * Resolves the current repository's simple Compose app environment when Docker + * is unavailable. The checked-in files use list-form `KEY=value` entries. + */ +export function parseComposeFileEnvironment( + contents: string, + envFileValues: ReadonlyMap +): Map | null { + const lines = contents.split('\n') + const serviceStart = lines.findIndex((line) => /^ {2}simstudio:\s*$/.test(line)) + if (serviceStart === -1) return null + + let serviceEnd = lines.length + for (let index = serviceStart + 1; index < lines.length; index += 1) { + if (/^ {2}[A-Za-z0-9_.-]+:\s*$/.test(lines[index])) { + serviceEnd = index + break + } + } + + const rootEnvFileUsage = serviceRootEnvFileUsage(lines, serviceStart, serviceEnd) + if (rootEnvFileUsage === null) return null + const values = rootEnvFileUsage ? new Map(envFileValues) : new Map() + + let environmentStart = -1 + for (let index = serviceStart + 1; index < serviceEnd; index += 1) { + const environment = /^ {4}environment:\s*(.*)$/.exec(lines[index]) + if (environment && environment[1] !== '') return null + if (environment) { + environmentStart = index + break + } + } + if (environmentStart === -1) return values + + for (let index = environmentStart + 1; index < serviceEnd; index += 1) { + const line = lines[index] + if (/^ {4}\S/.test(line) && !/^ {6}/.test(line)) break + const listEntry = /^ {6}-\s+(.+)$/.exec(line) + const mapEntry = /^ {6}([A-Za-z_][A-Za-z0-9_]*):\s*(.*)$/.exec(line) + let key: string + let raw: string + if (listEntry) { + const entry = stripYamlScalar(listEntry[1]) + const separator = entry.indexOf('=') + if (separator === -1) { + if (envFileValues.has(entry)) values.set(entry, envFileValues.get(entry) ?? '') + continue + } + key = entry.slice(0, separator) + raw = entry.slice(separator + 1) + } else if (mapEntry) { + key = mapEntry[1] + raw = stripYamlScalar(mapEntry[2]) + } else { + continue + } + const expanded = expandComposeValue(raw, envFileValues) + if (expanded === null) return null + values.set(key, expanded) + } + return values +} + +function preparedComposeSource(root: string, managed: boolean): ConfigurationSource | null { + const envPath = path.join(root, '.env') + if (!existsSync(envPath)) return null + const envValues = readEnv(envPath) + const composeFiles = COMPOSE_FILES.map((file) => path.join(root, file)).filter( + (file) => existsSync(file) && hasSimComposeMarker(file) + ) + if (composeFiles.length === 0) { + return { + kind: 'compose', + label: 'Current checkout (Compose prepared)', + location: envPath, + values: null, + warning: + 'No checked-in Sim Compose file could be read; effective container values are unknown.', + managedByCurrentCheckout: managed, + } + } + if (composeFiles.length > 1) { + return { + kind: 'compose', + label: 'Current checkout (Compose prepared)', + location: `${envPath} · candidates ${composeFiles.join(', ')}`, + values: null, + warning: + 'More than one Sim Compose variant is present and no live project identifies which one is selected.', + managedByCurrentCheckout: managed, + } + } + + const composeFile = composeFiles[0] + const resolvedValues = parseComposeFileEnvironment(readFileSync(composeFile, 'utf8'), envValues) + const values = resolvedValues ? filterConfigurationValues(resolvedValues) : null + return { + kind: 'compose', + label: 'Current checkout (Compose prepared)', + location: `${envPath} · ${composeFile}`, + values, + ...(values + ? {} + : { + warning: + 'The Sim app environment could not be resolved from the Compose file; run Docker Compose before relying on this status.', + }), + managedByCurrentCheckout: managed, + } +} + +function localSources( + root: string, + processEnvironment: Readonly +): { + sources: ConfigurationSource[] + setupSplitExists: boolean + prepared: ConfigurationSource | null +} { + const appDirectory = path.join(root, 'apps/sim') + const simEnv = path.join(appDirectory, '.env') + const setupSplitFiles = [ + simEnv, + path.join(root, 'apps/realtime/.env'), + path.join(root, 'packages/db/.env'), + ] + const setupSplitExists = setupSplitFiles.some(existsSync) + const developmentExists = + setupSplitExists || + DEVELOPMENT_ENV_FILES.some((file) => existsSync(path.join(appDirectory, file))) + const sources: ConfigurationSource[] = [] + if (developmentExists) { + const development = loadDevelopmentEnvironment(appDirectory, processEnvironment) + const hasHigherPrecedenceFile = development.loadedFiles.some( + (file) => canonicalPath(file) !== canonicalPath(simEnv) + ) + sources.push({ + kind: 'dev', + label: 'Current checkout (development)', + location: + development.loadedFiles.length > 0 + ? `process environment + ${development.loadedFiles.join(', ')}` + : `${appDirectory} (process environment only)`, + values: development.values, + ...(development.loadedFiles.length > 0 + ? {} + : { warning: 'No application env file was loaded; only the process environment applies.' }), + configurationIssues: inspectSplitConfiguration(root), + managedByCurrentCheckout: + setupSplitExists && !hasHigherPrecedenceFile && !development.hasProcessOverrides, + }) + } + return { + sources, + setupSplitExists, + prepared: preparedComposeSource(root, !setupSplitExists), + } +} + +function resolveComposeProjectEnvironment( + runner: ConfigurationCommandRunner, + root: string, + project: ComposeProject, + configFiles: readonly string[] +): EnvironmentResolution { + const containers = runSafely( + runner, + 'docker', + [ + 'ps', + '-a', + '--filter', + `label=com.docker.compose.project=${project.name}`, + '--filter', + 'label=com.docker.compose.service=simstudio', + '--format', + '{{.ID}}', + ], + root + ) + if (containers.status !== 0) { + return { + values: null, + warning: 'The Compose app containers could not be enumerated; effective values are unknown.', + } + } + const ids = containers.stdout + .split('\n') + .map((id) => id.trim()) + .filter(Boolean) + if (ids.length > 0) { + const inspect = runSafely(runner, 'docker', ['inspect', ...ids], root) + if (inspect.status !== 0) { + return { + values: null, + warning: 'The Compose app container could not be inspected; effective values are unknown.', + } + } + const values = parseDockerInspectEnvironment(inspect.stdout) + if (values) return { values: filterConfigurationValues(values) } + return { + values: null, + warning: + 'Docker returned an invalid Compose app container definition; effective values are unknown.', + } + } + + const composeArgs = ['compose', '-p', project.name] + for (const file of configFiles) composeArgs.push('-f', file) + composeArgs.push('config', '--format', 'json') + const config = runSafely(runner, 'docker', composeArgs, path.dirname(configFiles[0])) + if (config.status === 0) { + const values = parseComposeConfigEnvironment(config.stdout) + if (values) return { values: filterConfigurationValues(values) } + } + + return { + values: null, + warning: + 'The simstudio container and resolved Compose configuration could not be read; effective values are unknown.', + } +} + +function discoverComposeProjects( + runner: ConfigurationCommandRunner, + root: string +): ComposeDiscovery { + const info = runSafely(runner, 'docker', ['info'], root) + if (info.status !== 0) return { projects: [] } + const listed = runSafely(runner, 'docker', ['compose', 'ls', '-a', '--format', 'json'], root) + if (listed.status !== 0) return { projects: [] } + const projects = parseComposeProjects(listed.stdout) + if (!projects) throw new Error('Docker Compose returned an invalid project list') + + const discovered: DiscoveredComposeProject[] = [] + const seen = new Set() + for (const project of projects) { + const configFiles = project.configFiles.map((file) => path.resolve(root, file)) + if (!configFiles.some(hasSimComposeMarker)) continue + const identity = `${project.name}\0${configFiles.map(canonicalPath).sort().join('\0')}` + if (seen.has(identity)) continue + seen.add(identity) + const resolution = resolveComposeProjectEnvironment(runner, root, project, configFiles) + discovered.push({ + project, + configFiles, + source: { + kind: 'compose', + label: `Compose project "${project.name}"`, + location: configFiles.join(', '), + values: resolution.values, + ...(resolution.warning ? { warning: resolution.warning } : {}), + managedByCurrentCheckout: false, + }, + }) + } + return { projects: discovered } +} + +function decodeBase64(value: string): string | null { + if ( + value !== '' && + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value) + ) { + return null + } + return Buffer.from(value, 'base64').toString('utf8') +} + +function resourceEnvironment( + kind: KubernetesResourceKind, + resource: unknown +): EnvironmentResolution { + const record = asRecord(resource) + if (!record) return { values: null, warning: 'Kubernetes returned an invalid resource.' } + const values = new Map() + const data = record.data === undefined ? {} : asRecord(record.data) + if (!data) return { values: null, warning: 'Kubernetes returned invalid resource data.' } + for (const [key, value] of Object.entries(data)) { + if (typeof value !== 'string') { + return { values: null, warning: 'Kubernetes returned a non-string environment value.' } + } + if (kind === 'secret') { + const decoded = decodeBase64(value) + if (decoded === null) { + return { values: null, warning: 'Kubernetes returned invalid Secret data.' } + } + values.set(key, decoded) + } else { + values.set(key, value) + } + } + + const secondaryKey = kind === 'secret' ? 'stringData' : 'binaryData' + const secondary = record[secondaryKey] === undefined ? {} : asRecord(record[secondaryKey]) + if (!secondary) return { values: null, warning: 'Kubernetes returned invalid resource data.' } + for (const [key, value] of Object.entries(secondary)) { + if (typeof value !== 'string') { + return { values: null, warning: 'Kubernetes returned a non-string environment value.' } + } + if (kind === 'configmap') { + const decoded = decodeBase64(value) + if (decoded === null) { + return { values: null, warning: 'Kubernetes returned invalid ConfigMap binary data.' } + } + values.set(key, decoded) + } else { + values.set(key, value) + } + } + return { values } +} + +function inaccessibleResourceWarning(kind: KubernetesResourceKind, name: string): string { + const displayKind = kind === 'secret' ? 'Secret' : 'ConfigMap' + return `Cannot read ${displayKind} "${name}"; configuration status is unknown. Verify kubectl RBAC allows get access in this namespace.` +} + +function missingResourceWarning(kind: KubernetesResourceKind, name: string): string { + const displayKind = kind === 'secret' ? 'Secret' : 'ConfigMap' + return `${displayKind} "${name}" is required by the app Deployment but does not exist; configuration status is unknown.` +} + +function loadKubernetesResource( + kind: KubernetesResourceKind, + name: string, + optional: boolean, + lookup: KubernetesResourceLookup +): EnvironmentResolution & { missing?: boolean } { + const result = lookup(kind, name) + if (result.state === 'inaccessible') { + return { values: null, warning: inaccessibleResourceWarning(kind, name) } + } + if (result.state === 'missing') { + return optional + ? { values: new Map(), missing: true } + : { values: null, warning: missingResourceWarning(kind, name) } + } + const resolution = resourceEnvironment(kind, result.resource) + return resolution.values + ? resolution + : { + values: null, + warning: `${resolution.warning ?? 'Kubernetes resource could not be decoded'} Configuration status is unknown.`, + } +} + +/** Resolves Kubernetes `envFrom` then explicit `env`, matching Kubernetes precedence. */ +export function resolveKubernetesContainerEnvironment( + container: unknown, + lookup: KubernetesResourceLookup +): EnvironmentResolution { + const record = asRecord(container) + if (!record) return { values: null, warning: 'The app container definition is invalid.' } + const values = new Map() + const envFrom = record.envFrom === undefined ? [] : record.envFrom + if (!Array.isArray(envFrom)) { + return { values: null, warning: 'The app container envFrom definition is invalid.' } + } + + for (const source of envFrom) { + const sourceRecord = asRecord(source) + if (!sourceRecord) + return { values: null, warning: 'The app container envFrom entry is invalid.' } + const secretRef = asRecord(sourceRecord.secretRef) + const configMapRef = asRecord(sourceRecord.configMapRef) + const kind: KubernetesResourceKind | null = secretRef + ? 'secret' + : configMapRef + ? 'configmap' + : null + const reference = secretRef ?? configMapRef + if (!kind || !reference || typeof reference.name !== 'string') { + return { values: null, warning: 'The app container envFrom reference is invalid.' } + } + const loaded = loadKubernetesResource(kind, reference.name, reference.optional === true, lookup) + if (!loaded.values) return loaded + const prefix = sourceRecord.prefix === undefined ? '' : sourceRecord.prefix + if (typeof prefix !== 'string') { + return { values: null, warning: 'The app container envFrom prefix is invalid.' } + } + for (const [key, value] of loaded.values) values.set(`${prefix}${key}`, value) + } + + const explicitEnv = record.env === undefined ? [] : record.env + if (!Array.isArray(explicitEnv)) { + return { values: null, warning: 'The app container env definition is invalid.' } + } + for (const entry of explicitEnv) { + const env = asRecord(entry) + if (!env || typeof env.name !== 'string' || env.name === '') { + return { values: null, warning: 'The app container has an invalid explicit env entry.' } + } + if (env.value !== undefined) { + if (typeof env.value !== 'string' || env.valueFrom !== undefined) { + return { values: null, warning: `Explicit env "${env.name}" is invalid.` } + } + values.set(env.name, env.value) + continue + } + if (env.valueFrom === undefined) { + values.set(env.name, '') + continue + } + + const valueFrom = asRecord(env.valueFrom) + const secretKeyRef = asRecord(valueFrom?.secretKeyRef) + const configMapKeyRef = asRecord(valueFrom?.configMapKeyRef) + const kind: KubernetesResourceKind | null = secretKeyRef + ? 'secret' + : configMapKeyRef + ? 'configmap' + : null + const reference = secretKeyRef ?? configMapKeyRef + if ( + !kind || + !reference || + typeof reference.name !== 'string' || + typeof reference.key !== 'string' + ) { + return { + values: null, + warning: `Explicit env "${env.name}" uses an unsupported or invalid valueFrom source.`, + } + } + const loaded = loadKubernetesResource(kind, reference.name, reference.optional === true, lookup) + if (!loaded.values) return loaded + if (loaded.missing) continue + const value = loaded.values.get(reference.key) + if (value === undefined) { + if (reference.optional === true) continue + return { + values: null, + warning: `${kind === 'secret' ? 'Secret' : 'ConfigMap'} "${reference.name}" does not contain key "${reference.key}"; configuration status is unknown.`, + } + } + values.set(env.name, value) + } + return { values } +} + +function parseHelmReleases(output: string): HelmRelease[] | null { + const parsed = parseJson(output) + if (!Array.isArray(parsed)) return null + const releases: HelmRelease[] = [] + for (const item of parsed) { + const record = asRecord(item) + if ( + typeof record?.name !== 'string' || + typeof record.namespace !== 'string' || + typeof record.chart !== 'string' + ) { + return null + } + if (record.chart.startsWith('sim-')) { + releases.push({ name: record.name, namespace: record.namespace, chart: record.chart }) + } + } + return releases +} + +function helmLocation(context: string, release: HelmRelease): string { + return `context ${context} · namespace ${release.namespace} · release ${release.name}` +} + +function helmUnknown(release: HelmRelease, context: string, warning: string): ConfigurationSource { + return { + kind: 'helm', + label: `Helm release "${release.name}"`, + location: helmLocation(context, release), + values: null, + warning, + managedByCurrentCheckout: false, + } +} + +function discoverHelmReleases( + runner: ConfigurationCommandRunner, + root: string +): ConfigurationSource[] { + const contextResult = runSafely(runner, 'kubectl', ['config', 'current-context'], root) + if (contextResult.status !== 0) return [] + const context = contextResult.stdout.trim() + if (!context) return [] + const listed = runSafely( + runner, + 'helm', + ['list', '-A', '--deployed', '--failed', '-o', 'json', '--kube-context', context], + root + ) + if (listed.status !== 0) return [] + const releases = parseHelmReleases(listed.stdout) + if (!releases) throw new Error('Helm returned an invalid release list') + const seen = new Set() + const sources: ConfigurationSource[] = [] + + for (const release of releases) { + const identity = `${release.namespace}\0${release.name}` + if (seen.has(identity)) continue + seen.add(identity) + const selector = `app.kubernetes.io/instance=${release.name},app.kubernetes.io/component=app` + const deploymentResult = runSafely( + runner, + 'kubectl', + [ + 'get', + 'deployments', + '-n', + release.namespace, + '-l', + selector, + '-o', + 'json', + '--context', + context, + ], + root + ) + if (deploymentResult.status !== 0) { + sources.push( + helmUnknown( + release, + context, + `Cannot read the app Deployment in namespace "${release.namespace}"; verify kubectl access to the current context.` + ) + ) + continue + } + const deploymentList = asRecord(parseJson(deploymentResult.stdout)) + const items = deploymentList?.items + const itemCount = Array.isArray(items) ? items.length : null + if (!Array.isArray(items) || items.length !== 1) { + sources.push( + helmUnknown( + release, + context, + itemCount === 0 + ? 'No app Deployment matched the Sim instance/component labels.' + : 'More than one app Deployment matched the Sim instance/component labels.' + ) + ) + continue + } + const deployment = asRecord(items[0]) + const spec = asRecord(deployment?.spec) + const template = asRecord(spec?.template) + const podSpec = asRecord(template?.spec) + const containers = podSpec?.containers + if (!Array.isArray(containers)) { + sources.push( + helmUnknown(release, context, 'The app Deployment has no readable containers list.') + ) + continue + } + const appContainer = containers.find((container) => asRecord(container)?.name === 'app') + if (!appContainer) { + sources.push( + helmUnknown(release, context, 'The app Deployment has no container named "app".') + ) + continue + } + + const resourceCache = new Map() + const lookup: KubernetesResourceLookup = (kind, name) => { + const key = `${kind}\0${name}` + const cached = resourceCache.get(key) + if (cached) return cached + const result = runSafely( + runner, + 'kubectl', + [ + 'get', + kind, + name, + '-n', + release.namespace, + '--ignore-not-found', + '-o', + 'json', + '--context', + context, + ], + root + ) + let resolved: KubernetesResourceLookupResult + if (result.status !== 0) resolved = { state: 'inaccessible' } + else if (result.stdout.trim() === '') resolved = { state: 'missing' } + else { + const resource = parseJson(result.stdout) + resolved = resource === null ? { state: 'inaccessible' } : { state: 'found', resource } + } + resourceCache.set(key, resolved) + return resolved + } + const resolution = resolveKubernetesContainerEnvironment(appContainer, lookup) + sources.push({ + kind: 'helm', + label: `Helm release "${release.name}"`, + location: helmLocation(context, release), + values: resolution.values ? filterConfigurationValues(resolution.values) : null, + ...(resolution.warning ? { warning: resolution.warning } : {}), + managedByCurrentCheckout: false, + }) + } + return sources +} + +/** Discovers every configuration snapshot without writing or displaying environment values. */ +export function discoverConfigurationSources( + options: ConfigurationSourceDiscoveryOptions = {} +): ConfigurationSource[] { + const root = path.resolve(options.root ?? ROOT) + const runner = options.runner ?? defaultRunner + const local = localSources(root, options.processEnvironment ?? process.env) + const compose = discoverComposeProjects(runner, root) + const rootPath = canonicalPath(root) + const currentRootProjects = compose.projects.filter((project) => { + return project.configFiles.some((file) => canonicalPath(path.dirname(file)) === rootPath) + }) + const knownCurrentFiles = new Set( + COMPOSE_FILES.map((file) => canonicalPath(path.join(root, file))) + ) + const rootEnvExists = existsSync(path.join(root, '.env')) + const canManageCurrentCompose = + !local.setupSplitExists && rootEnvExists && currentRootProjects.length === 1 + + for (const project of currentRootProjects) { + const exactCurrentFileSet = + project.configFiles.length === 1 && + knownCurrentFiles.has(canonicalPath(project.configFiles[0])) + project.source.managedByCurrentCheckout = canManageCurrentCompose && exactCurrentFileSet + } + + const sources = [...local.sources] + if (currentRootProjects.length === 0 && local.prepared) sources.push(local.prepared) + sources.push(...compose.projects.map((project) => project.source)) + sources.push(...discoverHelmReleases(runner, root)) + return sources +} diff --git a/scripts/setup/env-files.test.ts b/scripts/setup/env-files.test.ts new file mode 100644 index 00000000000..c4359deeb6c --- /dev/null +++ b/scripts/setup/env-files.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'bun:test' +import { + isPlaceholder, + isUsableSecret, + parseEnv, + reconcileEnvContent, + upsertEnv, +} from './env-files.ts' + +describe('placeholder detection', () => { + it('recognizes underscore and hyphen template prefixes', () => { + expect(isPlaceholder('your_secret_key')).toBe(true) + expect(isPlaceholder('your-secure-production-auth-secret-here')).toBe(true) + expect(isUsableSecret('BETTER_AUTH_SECRET', 'your-secure-production-auth-secret-here')).toBe( + false + ) + expect(isPlaceholder('yourActualSecret')).toBe(false) + }) +}) + +describe('upsertEnv', () => { + it('writes the value that parseEnv will use', () => { + const updated = upsertEnv('RESEND_API_KEY=old\nOTHER=value\n', 'RESEND_API_KEY', 'current') + + expect(parseEnv(updated).get('RESEND_API_KEY')).toBe('current') + }) + + it('fails fast when duplicate active entries make the effective write ambiguous', () => { + expect(() => + upsertEnv( + 'RESEND_API_KEY=old\nexport RESEND_API_KEY=current\n', + 'RESEND_API_KEY', + 'replacement' + ) + ).toThrow('Duplicate active RESEND_API_KEY entries') + }) +}) + +describe('reconcileEnvContent', () => { + it('applies provider removals and replacements to one snapshot', () => { + const reconciled = reconcileEnvContent( + 'SMTP_HOST=old-host\nSMTP_PORT=587\nRESEND_API_KEY=old-key\n', + ['SMTP_HOST', 'SMTP_PORT'], + { RESEND_API_KEY: 'new-key' } + ) + + expect(parseEnv(reconciled)).toEqual(new Map([['RESEND_API_KEY', 'new-key']])) + }) + + it('fails before returning content when a replacement key is duplicated', () => { + const content = 'SMTP_HOST=old-host\nRESEND_API_KEY=old-key\nRESEND_API_KEY=newer-key\n' + + expect(() => + reconcileEnvContent(content, ['SMTP_HOST'], { RESEND_API_KEY: 'replacement' }) + ).toThrow('Duplicate active RESEND_API_KEY entries') + expect(parseEnv(content).get('SMTP_HOST')).toBe('old-host') + }) +}) diff --git a/scripts/setup/env-files.ts b/scripts/setup/env-files.ts index cf504f6a54b..726c98b63ae 100644 --- a/scripts/setup/env-files.ts +++ b/scripts/setup/env-files.ts @@ -75,7 +75,7 @@ export function parseEnv(content: string): Map { const vars = new Map() for (const line of content.split('\n')) { const match = LINE_RE.exec(line) - if (match && !vars.has(match[1])) vars.set(match[1], parseValue(match[2])) + if (match) vars.set(match[1], parseValue(match[2])) } return vars } @@ -95,7 +95,11 @@ export function upsertEnv(content: string, key: string, value: string): string { const lines = content.split('\n') const activeRe = new RegExp(`^\\s*(?:export\\s+)?${key}\\s*=`) const commentedRe = new RegExp(`^#\\s*${key}\\s*=`) - const activeIdx = lines.findIndex((l) => activeRe.test(l)) + const activeIndexes = lines.flatMap((line, index) => (activeRe.test(line) ? [index] : [])) + if (activeIndexes.length > 1) { + throw new Error(`Duplicate active ${key} entries found in environment file`) + } + const activeIdx = activeIndexes[0] ?? -1 const idx = activeIdx !== -1 ? activeIdx : lines.findIndex((l) => commentedRe.test(l)) const newLine = `${key}=${value}` if (idx === -1) { @@ -108,8 +112,33 @@ export function upsertEnv(content: string, key: string, value: string): string { return lines.join('\n') } -/** Writes values into an env file, seeding a missing file from its .env.example. */ -export function writeEnvValues(target: EnvTarget, values: Record): void { +/** Applies removals and replacements to one in-memory snapshot before it is written. */ +export function reconcileEnvContent( + content: string, + remove: readonly string[], + values: Record +): string { + const replacementKeys = new Set(Object.keys(values)) + const removalKeys = new Set(remove.filter((key) => !replacementKeys.has(key))) + let reconciled = content + .split('\n') + .filter((line) => { + const match = LINE_RE.exec(line) + return !match || !removalKeys.has(match[1]) + }) + .join('\n') + for (const [key, value] of Object.entries(values)) { + reconciled = upsertEnv(reconciled, key, value) + } + return reconciled +} + +/** Computes removals and replacements before writing the env file once. */ +export function reconcileEnvValues( + target: EnvTarget, + remove: readonly string[], + values: Record +): void { const filePath = ENV_PATHS[target] let content: string if (existsSync(filePath)) { @@ -118,10 +147,12 @@ export function writeEnvValues(target: EnvTarget, values: Record const example = EXAMPLE_PATHS[target] content = example && existsSync(example) ? readFileSync(example, 'utf8') : '' } - for (const [key, value] of Object.entries(values)) { - content = upsertEnv(content, key, value) - } - writeFileSync(filePath, content) + writeFileSync(filePath, reconcileEnvContent(content, remove, values)) +} + +/** Writes values into an env file, seeding a missing file from its .env.example. */ +export function writeEnvValues(target: EnvTarget, values: Record): void { + reconcileEnvValues(target, [], values) } export function archiveEnvFile(target: EnvTarget): string | null { @@ -162,7 +193,7 @@ export function secretRequirement(key: string): string { } export function isPlaceholder(value: string): boolean { - return PLACEHOLDER_VALUES.has(value) || value.startsWith('your_') + return PLACEHOLDER_VALUES.has(value) || value.startsWith('your_') || value.startsWith('your-') } /** diff --git a/scripts/setup/feature-setup.test.ts b/scripts/setup/feature-setup.test.ts new file mode 100644 index 00000000000..8026acfb1d7 --- /dev/null +++ b/scripts/setup/feature-setup.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, it } from 'bun:test' +import { + SANDBOX_CAPABILITY, + validateCapabilityFieldInput, +} from '../../apps/sim/lib/core/config/env-capabilities.ts' +import { SANDBOX_SETUP } from './capability-config.ts' +import { buildCapabilitySetupTransition } from './capability-setup.ts' +import type { ConfigurationSource } from './configuration-sources.ts' +import { reconcileLlmSetup, resolveFeatureSetupDestination } from './feature-setup.ts' + +function source( + kind: ConfigurationSource['kind'], + managedByCurrentCheckout: boolean, + values: Map | null = new Map() +): ConfigurationSource { + return { + kind, + label: `${kind} source`, + location: `${kind} location`, + values, + managedByCurrentCheckout, + } +} + +describe('resolveFeatureSetupDestination', () => { + it('uses the effective values of the one setup-managed source', () => { + const effective = new Map([['RESEND_API_KEY', 'effective-key']]) + const destination = resolveFeatureSetupDestination([ + source('compose', false), + source('dev', true, effective), + ]) + + expect(destination.target).toBe('sim') + expect(destination.containerized).toBe(false) + expect(destination.vars).toBe(effective) + }) + + it('maps a managed Compose source to the root env file', () => { + const destination = resolveFeatureSetupDestination([source('compose', true)]) + + expect(destination.target).toBe('root') + expect(destination.containerized).toBe(true) + }) + + it('refuses effective sources this checkout cannot safely update', () => { + expect(() => resolveFeatureSetupDestination([source('dev', false)])).toThrow( + /No effective configuration is safely writable/ + ) + expect(() => resolveFeatureSetupDestination([source('helm', false)])).toThrow( + /No effective configuration is safely writable/ + ) + }) + + it('refuses an unreadable managed source instead of claiming success', () => { + expect(() => resolveFeatureSetupDestination([source('compose', true, null)])).toThrow( + /effective environment could not be resolved/ + ) + }) +}) + +describe('sandbox capability setup', () => { + it('requires an explicit non-floating snapshot tag', () => { + const validate = (value: string) => + validateCapabilityFieldInput(SANDBOX_CAPABILITY, 'DAYTONA_SHELL_SNAPSHOT_ID', value) + expect(validate('mothership-shell:v1')).toBeUndefined() + expect(validate('mothership-shell')).toContain('name:tag') + expect(validate('mothership-shell:latest')).toContain('name:tag') + }) + + it('writes Daytona API and shell snapshot configuration and disables E2B', () => { + const result = buildCapabilitySetupTransition( + SANDBOX_SETUP, + 'daytona', + { + DAYTONA_API_KEY: 'daytona-key', + DAYTONA_SHELL_SNAPSHOT_ID: 'mothership-shell:v1', + }, + {} + ) + + expect(result.remove).toContain('E2B_API_KEY') + expect(result.values).toMatchObject({ + DAYTONA_API_KEY: 'daytona-key', + DAYTONA_SHELL_SNAPSHOT_ID: 'mothership-shell:v1', + E2B_ENABLED: 'false', + NEXT_PUBLIC_E2B_ENABLED: 'false', + NEXT_PUBLIC_SANDBOX_ENABLED: 'true', + }) + }) + + it('removes stale Daytona configuration for E2B and disabled modes', () => { + expect( + buildCapabilitySetupTransition(SANDBOX_SETUP, 'e2b', { E2B_API_KEY: 'e2b-key' }, {}).remove + ).toEqual(expect.arrayContaining(['DAYTONA_API_KEY', 'DAYTONA_SHELL_SNAPSHOT_ID'])) + expect(buildCapabilitySetupTransition(SANDBOX_SETUP, 'disabled', {}, {}).remove).toEqual( + expect.arrayContaining(['DAYTONA_API_KEY', 'DAYTONA_SHELL_SNAPSHOT_ID']) + ) + }) +}) + +describe('reconcileLlmSetup', () => { + it('removes trailing rotation keys omitted after empty-to-finish', () => { + expect(reconcileLlmSetup('openai', { OPENAI_API_KEY_1: 'replacement' })).toEqual({ + values: { OPENAI_API_KEY_1: 'replacement' }, + remove: ['OPENAI_API_KEY_2', 'OPENAI_API_KEY_3', 'OPENAI_API_KEY'], + }) + }) + + it('removes a legacy fallback when rotation keys replace it', () => { + expect(reconcileLlmSetup('fireworks', { FIREWORKS_API_KEY_1: 'replacement' })).toEqual({ + values: { FIREWORKS_API_KEY_1: 'replacement' }, + remove: ['FIREWORKS_API_KEY_2', 'FIREWORKS_API_KEY_3', 'FIREWORKS_API_KEY'], + }) + }) +}) diff --git a/scripts/setup/feature-setup.ts b/scripts/setup/feature-setup.ts new file mode 100644 index 00000000000..b973e2272b0 --- /dev/null +++ b/scripts/setup/feature-setup.ts @@ -0,0 +1,205 @@ +import { + LLM_KEY_POOLS, + OAUTH_CLIENT_CAPABILITIES, + resolveOAuthClientCapabilityId, +} from '../../apps/sim/lib/core/config/env-capabilities.ts' +import { + getCapabilitySetup, + getOAuthClientSetupFields, + SETUP_FEATURES, + type SetupFeatureId, +} from './capability-config.ts' +import { promptCapabilitySetup } from './capability-setup.ts' +import { type ConfigurationSource, discoverConfigurationSources } from './configuration-sources.ts' +import { type EnvTarget, reconcileEnvValues } from './env-files.ts' +import * as p from './prompter.ts' +import { theme } from './theme.ts' + +function isSetupFeatureId(value: string): value is SetupFeatureId { + return SETUP_FEATURES.some((feature) => feature.id === value) +} + +async function setupIntegration( + requestedId: string | undefined, + vars: Map +): Promise> { + if (!requestedId) { + throw new Error('Missing integration id. Example: bun run setup integration slack') + } + const providerId = resolveOAuthClientCapabilityId(requestedId) + if (!providerId) { + throw new Error( + `Unknown OAuth integration "${requestedId}". Expected one of: ${Object.keys(OAUTH_CLIENT_CAPABILITIES).join(', ')}` + ) + } + const fields = getOAuthClientSetupFields(providerId) + + const values: Record = {} + for (const field of fields) { + const existing = vars.get(field.key) + if (field.input === 'secret') { + const value = await p.password({ + message: existing ? `${field.key} (Currently used); leave empty to keep it` : field.key, + validate: (candidate) => (candidate || existing ? undefined : 'required'), + }) + const resolved = value || existing + if (!resolved) throw new Error(`${field.key} was not provided`) + values[field.key] = resolved + } else { + values[field.key] = await p.text({ + message: `${field.key}${existing ? ' (Currently used)' : ''}`, + initialValue: existing, + validate: (candidate) => (candidate ? undefined : 'required'), + }) + } + } + p.log.info(`Configured the ${providerId} OAuth client.`) + return values +} + +type LlmKeyPoolId = keyof typeof LLM_KEY_POOLS + +export interface LlmSetupResult { + remove: readonly string[] + values: Record +} + +/** Reconciles every rotation and legacy fallback key owned by the selected pool. */ +export function reconcileLlmSetup( + providerId: LlmKeyPoolId, + values: Record +): LlmSetupResult { + const pool = LLM_KEY_POOLS[providerId] + const fields = [...pool.keys, ...('fallbackKey' in pool ? [pool.fallbackKey] : [])] + return { + values, + remove: fields.filter((key) => !Object.hasOwn(values, key)), + } +} + +async function setupLlm(vars: Map): Promise { + const currentProvider = Object.entries(LLM_KEY_POOLS).find(([, pool]) => + [...pool.keys, ...('fallbackKey' in pool ? [pool.fallbackKey] : [])].some((key) => + vars.has(key) + ) + )?.[0] as LlmKeyPoolId | undefined + const provider = await p.select({ + message: 'LLM key pool?', + options: Object.keys(LLM_KEY_POOLS).map((id) => ({ + value: id as LlmKeyPoolId, + label: id, + hint: id === currentProvider ? 'Currently used' : undefined, + })), + initialValue: currentProvider, + }) + const pool = LLM_KEY_POOLS[provider] + const keys = pool.keys + const values: Record = {} + for (const [index, key] of keys.entries()) { + const legacyKey = index === 0 && 'fallbackKey' in pool ? pool.fallbackKey : undefined + const existingKey = vars.has(key) ? key : legacyKey + const existing = existingKey ? vars.get(existingKey) : undefined + const value = await p.password({ + message: existing + ? `${key} (${existingKey} is currently used); leave empty to keep it` + : `${key}${index === 0 ? '' : ' (empty to finish)'}`, + validate: + index === 0 ? (candidate) => (candidate || existing ? undefined : 'required') : undefined, + }) + const resolved = value || existing + if (!resolved) break + values[key] = resolved + } + return reconcileLlmSetup(provider, values) +} + +export function setupFeatureUsage(): string { + return SETUP_FEATURES.map((feature) => + feature.id === 'integration' ? 'integration ' : feature.id + ).join(' | ') +} + +export interface FeatureSetupDestination { + source: ConfigurationSource + target: Extract + vars: Map + containerized: boolean +} + +/** Resolves the one effective configuration this checkout can safely update. */ +export function resolveFeatureSetupDestination( + sources: readonly ConfigurationSource[] +): FeatureSetupDestination { + if (sources.length === 0) { + throw new Error('No Sim configuration was detected. Run bun run setup first.') + } + + const managed = sources.filter((source) => source.managedByCurrentCheckout) + if (managed.length === 0) { + throw new Error( + 'No effective configuration is safely writable by this checkout. Process overrides, higher-precedence development env files, external Compose projects, and Helm releases must be updated at their source. Run bun run setup status for the detected sources.' + ) + } + if (managed.length > 1) { + throw new Error( + `More than one effective configuration is writable by this checkout (${managed.map((source) => source.label).join(', ')}). Run bun run setup status and remove the ambiguity before configuring a feature.` + ) + } + + const source = managed[0] + if (!source.values) { + throw new Error( + `${source.label} is managed by this checkout, but its effective environment could not be resolved. Run bun run setup status and fix the reported source error first.` + ) + } + if (source.kind === 'helm') { + throw new Error( + 'Helm configuration cannot be updated by bun run setup. Update the release Secret or values and upgrade the release.' + ) + } + + return { + source, + target: source.kind === 'compose' ? 'root' : 'sim', + vars: source.values, + containerized: source.kind === 'compose', + } +} + +export async function runFeatureSetup(feature: string, args: readonly string[]): Promise { + if (!isSetupFeatureId(feature)) { + throw new Error(`Unknown setup feature "${feature}". Expected: ${setupFeatureUsage()}`) + } + const destination = resolveFeatureSetupDestination(discoverConfigurationSources()) + const { target, vars } = destination + let values: Record + let remove: readonly string[] + const capabilitySetup = getCapabilitySetup(feature) + + if (capabilitySetup) { + const result = await promptCapabilitySetup(capabilitySetup, vars, { + containerized: destination.containerized, + }) + values = result.values + remove = result.remove + } else if (feature === 'integration') { + values = await setupIntegration(args[0], vars) + remove = [] + } else if (feature === 'llm') { + const result = await setupLlm(vars) + values = result.values + remove = result.remove + } else { + throw new Error(`Setup feature ${feature} has no handler`) + } + + reconcileEnvValues(target, remove, values) + const label = SETUP_FEATURES.find((item) => item.id === feature)?.label + p.outro( + theme.accent( + destination.containerized + ? `${label} written to .env. Recreate the app container for it to take effect.` + : `${label} configured.` + ) + ) +} diff --git a/scripts/setup/index.ts b/scripts/setup/index.ts index 13165216da2..cc5660e3da1 100755 --- a/scripts/setup/index.ts +++ b/scripts/setup/index.ts @@ -2,14 +2,20 @@ import { getErrorMessage } from '@sim/utils/errors' import { runDoctor } from './doctor.ts' import { SetupError } from './errors.ts' +import { runFeatureSetup, setupFeatureUsage } from './feature-setup.ts' import { isLifecycleCommand, runLifecycle } from './lifecycle.ts' +import { runSetupStatus } from './setup-status.ts' import { exitWith, restoreTerminal } from './terminal.ts' import { theme } from './theme.ts' import { runWizard, type WizardMode } from './wizard.ts' const USAGE = `Usage: bun run setup run the setup wizard + bun run setup status show configured capabilities and integrations + bun run setup configure ${setupFeatureUsage()} bun run sim setup [--quick] [--mode compose|dev|k8s] + bun run sim setup status show configured capabilities and integrations + bun run sim setup configure one feature bun run sim doctor [--fix] [--json] check your setup bun run sim start | stop | restart bring your install up / down / cycle bun run sim status what's installed and healthy @@ -56,6 +62,16 @@ async function main(): Promise { if (command === 'setup') { const setupArgs = args.slice(1) + const feature = setupArgs[0]?.startsWith('-') ? undefined : setupArgs[0] + if (feature === 'status') { + process.exitCode = await runSetupStatus() + return + } + if (feature) { + const featureIndex = setupArgs.indexOf(feature) + await runFeatureSetup(feature, setupArgs.slice(featureIndex + 1)) + return + } const modeIdx = setupArgs.indexOf('--mode') await runWizard({ quick: setupArgs.includes('--quick'), diff --git a/scripts/setup/launcher.test.ts b/scripts/setup/launcher.test.ts new file mode 100644 index 00000000000..204ba8e5d8b --- /dev/null +++ b/scripts/setup/launcher.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'bun:test' +import { + isMissingDependencyError, + missingDependenciesMessage, + retrySetupCommand, +} from './launcher.ts' + +describe('setup launcher', () => { + it('recognizes missing packages without masking application import errors', () => { + expect( + isMissingDependencyError({ + code: 'ERR_MODULE_NOT_FOUND', + message: "Cannot find package '@clack/prompts' from '/repo/scripts/setup/prompter.ts'", + }) + ).toBe(true) + expect( + isMissingDependencyError({ + code: 'ERR_MODULE_NOT_FOUND', + message: "Cannot find module './missing-application-file.ts'", + }) + ).toBe(false) + expect(isMissingDependencyError(new Error('Invalid setup configuration'))).toBe(false) + }) + + it('prints the install command and the public retry command', () => { + expect(retrySetupCommand(['setup', 'status'])).toBe('bun run setup status') + expect(retrySetupCommand(['doctor'])).toBe('bun run sim doctor') + expect(missingDependenciesMessage('bun run setup status')).toContain('Run: bun install') + expect(missingDependenciesMessage('bun run setup status')).toContain( + 'Then retry: bun run setup status' + ) + }) +}) diff --git a/scripts/setup/launcher.ts b/scripts/setup/launcher.ts new file mode 100755 index 00000000000..29e057809ff --- /dev/null +++ b/scripts/setup/launcher.ts @@ -0,0 +1,52 @@ +#!/usr/bin/env bun + +interface ModuleResolutionError { + code?: unknown + message?: unknown +} + +function asModuleResolutionError(error: unknown): ModuleResolutionError | null { + return typeof error === 'object' && error !== null ? error : null +} + +/** Identifies dependency-resolution failures without masking setup/configuration errors. */ +export function isMissingDependencyError(error: unknown): boolean { + const candidate = asModuleResolutionError(error) + if (candidate?.code !== 'ERR_MODULE_NOT_FOUND' && candidate?.code !== 'MODULE_NOT_FOUND') { + return false + } + return ( + typeof candidate.message === 'string' && + /Cannot find (?:package|module) ['"][^./][^'"]*['"]/.test(candidate.message) + ) +} + +/** Reconstructs the public command instead of exposing the internal launcher path. */ +export function retrySetupCommand(args: readonly string[]): string { + if (args[0] === 'setup') return ['bun run setup', ...args.slice(1)].join(' ') + return ['bun run sim', ...args].join(' ') +} + +export function missingDependenciesMessage(retryCommand: string): string { + return [ + '', + '✗ Setup dependencies are missing or out of date.', + '', + ' Run: bun install', + ` Then retry: ${retryCommand}`, + ].join('\n') +} + +export async function launchSetupCli( + args: readonly string[] = process.argv.slice(2) +): Promise { + try { + await import('./index.ts') + } catch (error) { + if (!isMissingDependencyError(error)) throw error + console.error(missingDependenciesMessage(retrySetupCommand(args))) + process.exitCode = 1 + } +} + +if (import.meta.main) await launchSetupCli() diff --git a/scripts/setup/modes/compose.ts b/scripts/setup/modes/compose.ts index 9e60b60bd48..aa1be4ed3bd 100644 --- a/scripts/setup/modes/compose.ts +++ b/scripts/setup/modes/compose.ts @@ -1,7 +1,9 @@ import { spawnSync } from 'node:child_process' +import { EMAIL_SETUP, STORAGE_SETUP } from '../capability-config.ts' +import { promptCapabilitySetup, stageCapabilitySetupTransition } from '../capability-setup.ts' import type { Detection } from '../detect.ts' import { ensureDocker } from '../docker.ts' -import { ROOT, readEnvFile, writeEnvValues } from '../env-files.ts' +import { ROOT, readEnvFile, reconcileEnvValues } from '../env-files.ts' import { SetupError } from '../errors.ts' import { ensurePortsFree } from '../ports.ts' import { httpHealth, waitFor } from '../probes.ts' @@ -11,11 +13,9 @@ import { collectSecrets, mothershipOverride, promptCopilotKey, - promptEmail, promptLlmKeys, promptSecurity, promptSignInProviders, - promptStorage, promptUnlocks, } from '../steps.ts' import { glyph, theme } from '../theme.ts' @@ -108,6 +108,7 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom const root = readEnvFile('root') const values = collectSecrets(root) + const remove = new Set() // Before the key is minted: a half-set override mints against one environment // and validates against the other, and warning afterwards is too late — the // bad key is already stored, and the next run offers to keep it. @@ -117,11 +118,18 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom Object.assign(values, chatFlagValues(copilotKey)) Object.assign(values, await promptLlmKeys(detection, !quick)) if (!quick) { - const storage = await promptStorage(root.vars, true) - if (storage) Object.assign(values, storage) + const stagedVars = new Map(root.vars) + for (const [key, value] of Object.entries(values)) stagedVars.set(key, value) + const storage = await promptCapabilitySetup(STORAGE_SETUP, stagedVars, { + containerized: true, + }) + stageCapabilitySetupTransition(stagedVars, values, remove, storage) const appUrl = root.vars.get('NEXT_PUBLIC_APP_URL') ?? APP_URL - Object.assign(values, await promptSignInProviders(root.vars, appUrl)) - Object.assign(values, await promptEmail(root.vars)) + Object.assign(values, await promptSignInProviders(stagedVars, appUrl)) + const email = await promptCapabilitySetup(EMAIL_SETUP, stagedVars, { + containerized: true, + }) + stageCapabilitySetupTransition(stagedVars, values, remove, email) const security = await promptSecurity(root.vars) Object.assign(values, security.sim, security.mirrorToRealtime) Object.assign(values, await promptUnlocks(root.vars)) @@ -133,7 +141,8 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom ) } if (!root.vars.get('NEXT_TELEMETRY_DISABLED')) values.NEXT_TELEMETRY_DISABLED = '1' - writeEnvValues('root', values) + for (const key of Object.keys(values)) remove.delete(key) + reconcileEnvValues('root', [...remove], values) p.log.step('Wrote .env (compose reads it for variable substitution)') await ensureComposePortsFree(composeFile) diff --git a/scripts/setup/modes/dev.ts b/scripts/setup/modes/dev.ts index 0effc129bcc..388adc98149 100644 --- a/scripts/setup/modes/dev.ts +++ b/scripts/setup/modes/dev.ts @@ -1,9 +1,11 @@ import { spawnSync } from 'node:child_process' import path from 'node:path' import { truncate } from '@sim/utils/string' +import { EMAIL_SETUP, JOBS_SETUP, STORAGE_SETUP } from '../capability-config.ts' +import { promptCapabilitySetup, stageCapabilitySetupTransition } from '../capability-setup.ts' import { resolveDatabase } from '../db.ts' import type { Detection } from '../detect.ts' -import { ROOT, readEnvFile, writeEnvValues } from '../env-files.ts' +import { ROOT, readEnvFile, reconcileEnvValues, writeEnvValues } from '../env-files.ts' import { SetupError } from '../errors.ts' import { pgProbe } from '../probes.ts' import * as p from '../prompter.ts' @@ -13,11 +15,9 @@ import { collectSecrets, mothershipOverride, promptCopilotKey, - promptEmail, promptLlmKeys, promptSecurity, promptSignInProviders, - promptStorage, promptUnlocks, } from '../steps.ts' import { glyph, theme } from '../theme.ts' @@ -70,27 +70,6 @@ async function promptRedis(detection: Detection, existing?: string): Promise | null> { - const wants = await p.confirm({ - message: 'Enable Trigger.dev for background jobs? (off = jobs run via the DB queue)', - initialValue: false, - }) - if (!wants) return null - const secretKey = await p.password({ - message: 'TRIGGER_SECRET_KEY', - validate: (v) => (v ? undefined : 'required'), - }) - const projectId = await p.text({ - message: 'TRIGGER_PROJECT_ID', - validate: (v) => (v ? undefined : 'required'), - }) - return { - TRIGGER_DEV_ENABLED: 'true', - TRIGGER_SECRET_KEY: secretKey, - TRIGGER_PROJECT_ID: projectId, - } -} - export async function runDevMode( detection: Detection, quick: boolean @@ -118,6 +97,7 @@ export async function runDevMode( const simAfter = readEnvFile('sim') const values: Record = {} + const remove = new Set() // Before the key is minted: a half-set override mints against one environment // and validates against the other, and warning afterwards is too late — the // bad key is already stored, and the next run offers to keep it. @@ -141,12 +121,15 @@ export async function runDevMode( } if (!quick) { - const trigger = await promptTrigger() - if (trigger) Object.assign(values, trigger) - const storage = await promptStorage(simAfter.vars, false) - if (storage) Object.assign(values, storage) - Object.assign(values, await promptSignInProviders(simAfter.vars, APP_URL)) - Object.assign(values, await promptEmail(simAfter.vars)) + const stagedVars = new Map(simAfter.vars) + for (const [key, value] of Object.entries(values)) stagedVars.set(key, value) + for (const setup of [JOBS_SETUP, STORAGE_SETUP, EMAIL_SETUP] as const) { + const transition = await promptCapabilitySetup(setup, stagedVars, { + containerized: false, + }) + stageCapabilitySetupTransition(stagedVars, values, remove, transition) + } + Object.assign(values, await promptSignInProviders(stagedVars, APP_URL)) const security = await promptSecurity(simAfter.vars) Object.assign(values, security.sim) if (Object.keys(security.mirrorToRealtime).length > 0) { @@ -154,7 +137,10 @@ export async function runDevMode( } Object.assign(values, await promptUnlocks(simAfter.vars)) } - if (Object.keys(values).length > 0) writeEnvValues('sim', values) + for (const key of Object.keys(values)) remove.delete(key) + if (remove.size > 0 || Object.keys(values).length > 0) { + reconcileEnvValues('sim', [...remove], values) + } let script = 'dev:full' if (detection.specs.hostMemGb < 16) { diff --git a/scripts/setup/setup-status.test.ts b/scripts/setup/setup-status.test.ts new file mode 100644 index 00000000000..d69339eb3f5 --- /dev/null +++ b/scripts/setup/setup-status.test.ts @@ -0,0 +1,140 @@ +import { describe, expect, it } from 'bun:test' +import type { ConfigurationSource } from './configuration-sources' +import { buildSetupStatusReport, renderSetupStatusReport } from './setup-status' + +function source(values: Record): ConfigurationSource { + return { + kind: 'dev', + label: 'Local dev', + location: 'apps/sim/.env', + values: new Map(Object.entries(values)), + managedByCurrentCheckout: true, + } +} + +const CORE_VALUES = { + DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/sim', + BETTER_AUTH_SECRET: 'a'.repeat(32), + BETTER_AUTH_URL: 'http://localhost:3000', + NEXT_PUBLIC_APP_URL: 'http://localhost:3000', + ENCRYPTION_KEY: 'b'.repeat(64), + INTERNAL_API_SECRET: 'c'.repeat(32), +} + +describe('setup status', () => { + it('renders providers and missing integrations without exposing configured values', () => { + const report = buildSetupStatusReport( + source({ + ...CORE_VALUES, + RESEND_API_KEY: 'resend-super-secret', + SLACK_CLIENT_ID: 'slack-client-secret-value', + SLACK_CLIENT_SECRET: 'slack-client-secret', + }) + ) + const output = renderSetupStatusReport(report) + + expect(report.failed).toBe(false) + expect(output).toContain('Email delivery: Resend') + expect(output).toContain('OAuth ready') + expect(output).toContain('Slack') + expect(output).toContain('Unavailable') + expect(output).not.toContain('resend-super-secret') + expect(output).not.toContain('slack-client-secret-value') + expect(output).not.toContain('slack-client-secret') + expect(report.source).not.toHaveProperty('values') + }) + + it('fails on partial configuration while continuing to render other capabilities', () => { + const report = buildSetupStatusReport( + source({ + ...CORE_VALUES, + SMTP_HOST: 'localhost', + MICROSOFT_CLIENT_ID: 'partial-client', + }) + ) + const output = renderSetupStatusReport(report) + + expect(report.failed).toBe(true) + expect(output).toContain('Email delivery: Not configured') + expect(output).toContain('SMTP_PORT') + expect(output).toContain('Misconfigured') + expect(output).toContain('MICROSOFT_CLIENT_SECRET') + expect(output).not.toContain('partial-client') + }) + + it('warns about an incomplete email fallback without failing a configured provider', () => { + const report = buildSetupStatusReport( + source({ + ...CORE_VALUES, + RESEND_API_KEY: 'resend-super-secret', + SMTP_HOST: 'localhost', + }) + ) + const output = renderSetupStatusReport(report) + + expect(report.failed).toBe(false) + expect(report.capabilityStatus?.features.email.state).toBe('configured') + expect(output).toContain('Email delivery: Resend') + expect(output).toContain('SMTP_PORT') + expect(output).toContain('configure: bun run setup email') + expect(output).not.toContain('resend-super-secret') + }) + + it('marks an unreadable effective source unknown instead of missing', () => { + const unknown: ConfigurationSource = { + kind: 'helm', + label: 'Helm release sim', + location: 'context prod · namespace sim', + values: null, + warning: 'cannot read app Secret', + managedByCurrentCheckout: false, + } + const report = buildSetupStatusReport(unknown) + const output = renderSetupStatusReport(report) + + expect(report.failed).toBe(true) + expect(output).toContain('Effective environment is unavailable') + expect(output).not.toContain('Not configured') + }) + + it('fails on a partial OAuth client even when it is not in the visible catalog', () => { + const report = buildSetupStatusReport( + source({ + ...CORE_VALUES, + SPOTIFY_CLIENT_ID: 'partial-client-value', + }) + ) + const output = renderSetupStatusReport(report) + + expect(report.failed).toBe(true) + expect(output).toContain('Spotify OAuth client: partial') + expect(output).toContain('SPOTIFY_CLIENT_SECRET') + expect(output).not.toContain('partial-client-value') + }) + + it('names fields missing from an explicitly selected storage provider', () => { + const report = buildSetupStatusReport( + source({ + ...CORE_VALUES, + STORAGE_PROVIDER: 's3', + AWS_REGION: 'us-east-1', + }) + ) + const output = renderSetupStatusReport(report) + + expect(report.failed).toBe(true) + expect(output).toContain('S3_BUCKET_NAME') + }) + + it('fails on missing split-development files without retaining source values', () => { + const development = source(CORE_VALUES) + development.configurationIssues = ['apps/realtime/.env is missing'] + + const report = buildSetupStatusReport(development) + const output = renderSetupStatusReport(report) + + expect(report.failed).toBe(true) + expect(output).toContain('apps/realtime/.env is missing') + expect(report.source).not.toHaveProperty('values') + }) +}) diff --git a/scripts/setup/setup-status.ts b/scripts/setup/setup-status.ts new file mode 100644 index 00000000000..2b948ca8117 --- /dev/null +++ b/scripts/setup/setup-status.ts @@ -0,0 +1,358 @@ +import { + type EnvCapabilityValues, + hasEnvCapabilityValue, +} from '../../apps/sim/lib/core/config/env-capabilities.ts' +import { + type IntegrationAvailability, + resolveIntegrationAvailability, +} from '../../apps/sim/lib/integrations/availability.ts' +import { SETUP_FEATURES } from './capability-config.ts' +import { + buildEnvCapabilityStatus, + type EnvCapabilityFeatureStatuses, + type SetupStatusFeatureId, +} from './capability-status.ts' +import { REQUIRED_APP_KEYS } from './checks.ts' +import { type ConfigurationSource, discoverConfigurationSources } from './configuration-sources.ts' +import { isPlaceholder, isUsableSecret } from './env-files.ts' +import { glyph, theme } from './theme.ts' + +type FeatureStatus = EnvCapabilityFeatureStatuses[keyof EnvCapabilityFeatureStatuses] + +interface CoreConfigurationIssue { + key: (typeof REQUIRED_APP_KEYS)[number] + reason: 'missing' | 'placeholder' | 'invalid secret' | 'invalid URL' +} + +export type SetupStatusSource = Omit + +export interface SetupStatusReport { + source: SetupStatusSource + environmentAvailable: boolean + coreIssues: readonly CoreConfigurationIssue[] + capabilityStatus: ReturnType | null + integrationAvailability: readonly IntegrationAvailability[] | null + failed: boolean +} + +const SECRET_KEYS = new Set(['BETTER_AUTH_SECRET', 'ENCRYPTION_KEY', 'INTERNAL_API_SECRET']) +const URL_KEYS = new Set(['DATABASE_URL', 'BETTER_AUTH_URL', 'NEXT_PUBLIC_APP_URL']) +const FEATURE_ORDER: readonly SetupStatusFeatureId[] = SETUP_FEATURES.flatMap((feature) => + feature.id === 'integration' ? [] : [feature.id] +) + +function readString(values: EnvCapabilityValues, key: string): string | undefined { + const value = + values instanceof Map ? values.get(key) : (values as Readonly>)[key] + return value === undefined || value === null ? undefined : String(value) +} + +function inspectCoreConfiguration(values: EnvCapabilityValues): CoreConfigurationIssue[] { + const issues: CoreConfigurationIssue[] = [] + for (const key of REQUIRED_APP_KEYS) { + const value = readString(values, key) + if (!hasEnvCapabilityValue(values, key)) { + issues.push({ key, reason: 'missing' }) + } else if (value && isPlaceholder(value)) { + issues.push({ key, reason: 'placeholder' }) + } else if (value && SECRET_KEYS.has(key) && !isUsableSecret(key, value)) { + issues.push({ key, reason: 'invalid secret' }) + } else if (value && URL_KEYS.has(key)) { + try { + new URL(value) + } catch { + issues.push({ key, reason: 'invalid URL' }) + } + } + } + return issues +} + +function titleCase(value: string): string { + if (value === 'openai') return 'OpenAI' + return value + .split(/[-_]/) + .map((part) => `${part.charAt(0).toUpperCase()}${part.slice(1)}`) + .join(' ') +} + +function featureDetail(feature: FeatureStatus): string { + switch (feature.id) { + case 'email': + return feature.providerIds.length > 0 + ? feature.providerIds + .map( + (id) => + feature.providers.find((provider) => provider.id === id)?.label ?? titleCase(id) + ) + .join(' → ') + : 'Not configured' + case 'storage': + if (feature.providerId === 'local') return 'Local disk (default)' + return ( + feature.providers.find((provider) => provider.id === feature.providerId)?.label ?? + (feature.providerId ? titleCase(feature.providerId) : 'Not configured') + ) + case 'sandbox': + if (feature.providerId === 'disabled') return 'Disabled (local JavaScript only)' + return feature.providerId ? titleCase(feature.providerId) : 'Not configured' + case 'jobs': + return feature.providerId === 'database' ? 'Database queue (default)' : 'Trigger.dev' + case 'cache': + return feature.providerId === 'database' ? 'Postgres (default)' : 'Redis' + case 'knowledge': + if (feature.providerId === 'local') return 'Local parser (default)' + if (feature.providerId === 'azure-mistral') return 'Azure Mistral OCR' + return feature.providerId === 'mistral' ? 'Mistral OCR' : 'Not configured' + case 'llm': { + const configured = Object.values(feature.pools) + .filter((pool) => pool.state === 'configured') + .map((pool) => `${titleCase(pool.id)} (${pool.effectiveKeyCount})`) + return configured.length > 0 ? configured.join(', ') : 'No global key pools' + } + } +} + +function featureGlyph(feature: FeatureStatus): string { + if (feature.issue && (feature.state === 'configured' || feature.state === 'default')) { + return glyph.warn + } + if (feature.issue || feature.state === 'partial' || feature.state === 'invalid') return glyph.fail + if (feature.state === 'missing') return glyph.skip + return glyph.pass +} + +function withoutSetupCommand(message: string): string { + return message.replace(/\s+Run bun run setup[^.]*\.$/, '') +} + +function setupHint(source: SetupStatusSource, command: string): string | null { + if (source.managedByCurrentCheckout) return command + if (source.kind === 'helm') return 'update the app Secret/values and upgrade this Helm release' + if (source.kind === 'compose') return 'update this Compose project and recreate its app container' + return null +} + +function uniqueNames(integrations: readonly IntegrationAvailability[]): string[] { + return [...new Set(integrations.map((integration) => integration.name))].sort((a, b) => + a.localeCompare(b) + ) +} + +interface IntegrationGroup { + setupCommand?: string + missingFields: readonly string[] + names: string[] +} + +function groupIntegrations( + integrations: readonly IntegrationAvailability[] +): readonly IntegrationGroup[] { + const groups = new Map() + for (const integration of integrations) { + const missingFields = [...integration.missingFields].sort() + const key = `${integration.setupCommand ?? 'clientless'}:${missingFields.join(',')}` + const existing = groups.get(key) + if (existing) { + if (!existing.names.includes(integration.name)) existing.names.push(integration.name) + continue + } + groups.set(key, { + setupCommand: integration.setupCommand, + missingFields, + names: [integration.name], + }) + } + return [...groups.values()].map((group) => ({ + ...group, + names: group.names.sort((left, right) => left.localeCompare(right)), + })) +} + +function renderIntegrationGroup( + source: SetupStatusSource, + marker: string, + group: IntegrationGroup +): string[] { + const missing = + group.missingFields.length > 0 ? ` — missing ${group.missingFields.join(', ')}` : '' + const lines = [` ${marker} ${group.names.join(', ')}${missing}`] + if (group.setupCommand) { + const hint = setupHint(source, group.setupCommand) + if (hint) lines.push(` ${theme.muted(`configure: ${hint}`)}`) + } + return lines +} + +/** Builds a non-secret report for one effective deployment configuration. */ +export function buildSetupStatusReport(source: ConfigurationSource): SetupStatusReport { + const safeSource: SetupStatusSource = { + kind: source.kind, + label: source.label, + location: source.location, + managedByCurrentCheckout: source.managedByCurrentCheckout, + ...(source.warning ? { warning: source.warning } : {}), + ...(source.configurationIssues ? { configurationIssues: source.configurationIssues } : {}), + } + if (!source.values) { + return { + source: safeSource, + environmentAvailable: false, + coreIssues: [], + capabilityStatus: null, + integrationAvailability: null, + failed: true, + } + } + + const coreIssues = inspectCoreConfiguration(source.values) + const capabilityStatus = buildEnvCapabilityStatus(source.values) + const integrationAvailability = resolveIntegrationAvailability(source.values) + const brokenCapability = Object.values(capabilityStatus.features).some( + (feature) => + feature.state === 'partial' || + feature.state === 'invalid' || + (feature.state === 'missing' && Boolean(feature.issue)) + ) + const brokenIntegration = integrationAvailability.some( + (integration) => integration.state === 'misconfigured' + ) + const brokenOAuthClient = + capabilityStatus.oauthClients.partialCount > 0 || capabilityStatus.oauthClients.invalidCount > 0 + + return { + source: safeSource, + environmentAvailable: true, + coreIssues, + capabilityStatus, + integrationAvailability, + failed: + coreIssues.length > 0 || + Boolean(source.configurationIssues?.length) || + brokenCapability || + brokenIntegration || + brokenOAuthClient, + } +} + +/** Renders a report without including any configured environment values. */ +export function renderSetupStatusReport(report: SetupStatusReport): string { + const { source } = report + const lines = [ + theme.heading(source.label), + ` ${report.environmentAvailable ? glyph.pass : glyph.fail} ${source.location}`, + ] + if (source.warning) lines.push(` ${glyph.warn} ${source.warning}`) + lines.push('') + + if (!report.environmentAvailable || !report.capabilityStatus || !report.integrationAvailability) { + lines.push(theme.heading('Configuration')) + lines.push(` ${glyph.fail} Effective environment is unavailable.`) + return lines.join('\n') + } + + lines.push(theme.heading('Core configuration')) + if (report.coreIssues.length === 0 && !source.configurationIssues?.length) { + lines.push(` ${glyph.pass} All ${REQUIRED_APP_KEYS.length} required app values are present`) + } else { + for (const coreIssue of report.coreIssues) { + lines.push(` ${glyph.fail} ${coreIssue.key}: ${coreIssue.reason}`) + } + for (const configurationIssue of source.configurationIssues ?? []) { + lines.push(` ${glyph.fail} ${configurationIssue}`) + } + } + lines.push('') + + lines.push(theme.heading('Capabilities')) + for (const id of FEATURE_ORDER) { + const feature = report.capabilityStatus.features[id] + lines.push(` ${featureGlyph(feature)} ${feature.label}: ${featureDetail(feature)}`) + if (feature.issue) { + lines.push(` ${theme.muted(withoutSetupCommand(feature.issue.message))}`) + } + if (feature.state === 'missing' || feature.issue) { + const hint = setupHint(source, feature.setupCommand) + if (hint) lines.push(` ${theme.muted(`configure: ${hint}`)}`) + } + } + lines.push('') + + const deploymentIntegrations = report.integrationAvailability.filter( + (integration) => integration.setupCommand || integration.serviceAccountAvailable + ) + const ready = deploymentIntegrations.filter((integration) => integration.state === 'ready') + const limited = deploymentIntegrations.filter((integration) => integration.state === 'limited') + const unavailable = deploymentIntegrations.filter( + (integration) => integration.state === 'unavailable' + ) + const misconfigured = deploymentIntegrations.filter( + (integration) => integration.state === 'misconfigured' + ) + + lines.push(theme.heading('OAuth integrations')) + const readyNames = uniqueNames(ready) + lines.push( + readyNames.length > 0 + ? ` ${glyph.pass} OAuth ready (${readyNames.length}): ${readyNames.join(', ')}` + : ` ${glyph.skip} OAuth ready: none` + ) + const limitedNames = uniqueNames(limited) + if (limitedNames.length > 0) { + lines.push( + ` ${glyph.warn} OAuth unavailable; workspace credential option exists (${limitedNames.length}): ${limitedNames.join(', ')}` + ) + for (const group of groupIntegrations(limited)) { + lines.push(...renderIntegrationGroup(source, glyph.warn, group)) + } + } + if (unavailable.length > 0) { + lines.push(` ${glyph.skip} Unavailable (${uniqueNames(unavailable).length})`) + for (const group of groupIntegrations(unavailable)) { + lines.push(...renderIntegrationGroup(source, glyph.skip, group)) + } + } + if (misconfigured.length > 0) { + lines.push(` ${glyph.fail} Misconfigured (${uniqueNames(misconfigured).length})`) + for (const group of groupIntegrations(misconfigured)) { + lines.push(...renderIntegrationGroup(source, glyph.fail, group)) + } + } + const representedOAuthClients = new Set( + deploymentIntegrations.flatMap((integration) => { + if (!integration.setupCommand) return [] + return [integration.setupCommand.replace('bun run setup integration ', '')] + }) + ) + const additionalOAuthClients = Object.values(report.capabilityStatus.oauthClients.clients).filter( + (client) => !representedOAuthClients.has(client.id) && client.state !== 'absent' + ) + for (const client of additionalOAuthClients) { + const marker = client.state === 'ready' ? glyph.pass : glyph.fail + const missing = + client.missingFields.length > 0 ? ` — missing ${client.missingFields.join(', ')}` : '' + lines.push(` ${marker} ${titleCase(client.id)} OAuth client: ${client.state}${missing}`) + if (client.state !== 'ready') { + const hint = setupHint(source, client.setupCommand) + if (hint) lines.push(` ${theme.muted(`configure: ${hint}`)}`) + } + } + lines.push( + ` ${theme.muted('API-key integrations are configured per workspace and are not listed.')}` + ) + return lines.join('\n') +} + +export async function runSetupStatus(): Promise { + const sources = await discoverConfigurationSources() + console.log(`\n${theme.heading('◆ Sim setup status')}\n`) + if (sources.length === 0) { + console.log(` ${glyph.fail} No local-dev, Docker Compose, or Helm configuration detected.`) + console.log(` ${theme.muted('run: bun run setup')}`) + return 1 + } + + const reports = sources.map(buildSetupStatusReport) + console.log(reports.map(renderSetupStatusReport).join('\n\n')) + return reports.some((report) => report.failed) ? 1 : 0 +} diff --git a/scripts/setup/steps.test.ts b/scripts/setup/steps.test.ts new file mode 100644 index 00000000000..2eb2d10183d --- /dev/null +++ b/scripts/setup/steps.test.ts @@ -0,0 +1,232 @@ +import { describe, expect, it } from 'bun:test' +import { + EMAIL_CAPABILITY, + inspectCapability, + requireCapability, + STORAGE_CAPABILITY, + validateCapabilityFieldInput, +} from '../../apps/sim/lib/core/config/env-capabilities.ts' +import { EMAIL_SETUP, STORAGE_SETUP } from './capability-config.ts' +import { + buildCapabilitySetupTransition, + resolveCurrentCapabilitySetupOptionId, +} from './capability-setup.ts' + +function applyResult( + initial: Record, + result: { values: Record; remove: readonly string[] } +): Record { + const reconciled = { ...initial } + for (const key of result.remove) Reflect.deleteProperty(reconciled, key) + return Object.assign(reconciled, result.values) +} + +describe('setup provider reconciliation', () => { + it('defaults email setup to the first runtime-ready fallback', () => { + const vars = new Map([ + ['SMTP_HOST', 'smtp.example.com'], + [ + 'GMAIL_CREDENTIALS_JSON', + JSON.stringify({ client_email: 'service@example.com', private_key: 'secret' }), + ], + ['GMAIL_SENDER', 'sender@example.com'], + ]) + + expect(resolveCurrentCapabilitySetupOptionId(EMAIL_SETUP, vars)).toBe('gmail') + expect( + resolveCurrentCapabilitySetupOptionId( + EMAIL_SETUP, + new Map([['SMTP_HOST', 'smtp.example.com']]) + ) + ).toBe('smtp') + }) + + it('uses canonical storage selection for setup defaults', () => { + expect( + resolveCurrentCapabilitySetupOptionId(STORAGE_SETUP, new Map([['AWS_REGION', 'us-east-1']])) + ).toBe('local') + expect( + resolveCurrentCapabilitySetupOptionId( + STORAGE_SETUP, + new Map([ + ['STORAGE_PROVIDER', ' S3 '], + ['AWS_REGION', 'us-east-1'], + ['S3_BUCKET_NAME', 'files'], + ['S3_ENDPOINT', 'https://storage.example.com'], + ]) + ) + ).toBe('s3') + }) + + it('preserves other ready email providers when another fallback is configured', () => { + const result = buildCapabilitySetupTransition( + EMAIL_SETUP, + 'resend', + { RESEND_API_KEY: 'new-resend-key' }, + {} + ) + const reconciled = applyResult( + { + SMTP_HOST: 'smtp.example.com', + SMTP_PORT: '587', + SMTP_USER: 'old-user', + SMTP_PASS: 'old-pass', + }, + result + ) + + expect(result.remove).not.toEqual(expect.arrayContaining(['SMTP_HOST', 'SMTP_PORT'])) + expect(inspectCapability(EMAIL_CAPABILITY, reconciled).providerIds).toEqual(['resend', 'smtp']) + }) + + it('clears stale SMTP auth for an unauthenticated relay', () => { + const result = buildCapabilitySetupTransition( + EMAIL_SETUP, + 'smtp', + { SMTP_HOST: 'localhost', SMTP_PORT: '1025' }, + {} + ) + const reconciled = applyResult({ SMTP_USER: 'old-user', SMTP_PASS: 'old-pass' }, result) + + expect(reconciled).not.toHaveProperty('SMTP_USER') + expect(reconciled).not.toHaveProperty('SMTP_PASS') + expect(inspectCapability(EMAIL_CAPABILITY, reconciled).providerIds).toEqual(['smtp']) + }) + + it('clears stale static S3 credentials when IAM is selected', () => { + const result = buildCapabilitySetupTransition( + STORAGE_SETUP, + 's3', + { + AWS_REGION: 'us-east-1', + S3_BUCKET_NAME: 'files', + }, + {} + ) + const reconciled = applyResult( + { + AWS_ACCESS_KEY_ID: 'old-access-key', + AWS_SECRET_ACCESS_KEY: 'old-secret-key', + S3_ENDPOINT: 'https://old-endpoint.example.com', + }, + result + ) + + expect(reconciled).not.toHaveProperty('AWS_ACCESS_KEY_ID') + expect(reconciled).not.toHaveProperty('AWS_SECRET_ACCESS_KEY') + expect(reconciled).not.toHaveProperty('S3_ENDPOINT') + expect(requireCapability(STORAGE_CAPABILITY, reconciled).providerId).toBe('s3') + }) + + it('preserves specialized S3 bucket overrides that setup does not prompt for', () => { + const result = buildCapabilitySetupTransition( + STORAGE_SETUP, + 's3', + { + AWS_REGION: 'us-east-1', + S3_BUCKET_NAME: 'files', + }, + { + S3_KB_BUCKET_NAME: 'knowledge', + S3_CHAT_BUCKET_NAME: 'chat', + } + ) + const reconciled = applyResult( + { + S3_KB_BUCKET_NAME: 'knowledge', + S3_CHAT_BUCKET_NAME: 'chat', + }, + result + ) + + expect(result.remove).not.toEqual( + expect.arrayContaining(['S3_KB_BUCKET_NAME', 'S3_CHAT_BUCKET_NAME']) + ) + expect(reconciled.S3_KB_BUCKET_NAME).toBe('knowledge') + expect(reconciled.S3_CHAT_BUCKET_NAME).toBe('chat') + expect(requireCapability(STORAGE_CAPABILITY, reconciled).providerId).toBe('s3') + }) + + it('clears specialized S3 bucket overrides when switching to local storage', () => { + const result = buildCapabilitySetupTransition( + STORAGE_SETUP, + 'local', + {}, + { + AWS_REGION: 'us-east-1', + S3_BUCKET_NAME: 'files', + S3_KB_BUCKET_NAME: 'knowledge', + S3_CHAT_BUCKET_NAME: 'chat', + } + ) + const reconciled = applyResult( + { + AWS_REGION: 'us-east-1', + S3_BUCKET_NAME: 'files', + S3_KB_BUCKET_NAME: 'knowledge', + S3_CHAT_BUCKET_NAME: 'chat', + }, + result + ) + + expect(result.remove).toEqual( + expect.arrayContaining(['S3_KB_BUCKET_NAME', 'S3_CHAT_BUCKET_NAME']) + ) + expect(reconciled).not.toHaveProperty('S3_KB_BUCKET_NAME') + expect(reconciled).not.toHaveProperty('S3_CHAT_BUCKET_NAME') + expect(requireCapability(STORAGE_CAPABILITY, reconciled).providerId).toBe('local') + }) + + it('clears stale inline GCS credentials when ADC is selected', () => { + const result = buildCapabilitySetupTransition( + STORAGE_SETUP, + 'gcs', + { GCS_BUCKET_NAME: 'files' }, + {} + ) + const reconciled = applyResult( + { + GCS_PROJECT_ID: 'old-project', + GCS_CREDENTIALS_JSON: JSON.stringify({ + client_email: 'old@example.com', + private_key: 'old-key', + }), + }, + result + ) + + expect(reconciled).not.toHaveProperty('GCS_PROJECT_ID') + expect(reconciled).not.toHaveProperty('GCS_CREDENTIALS_JSON') + expect(requireCapability(STORAGE_CAPABILITY, reconciled).providerId).toBe('gcs') + }) +}) + +describe('setup input validation', () => { + it('validates SMTP ports with the runtime capability rule', () => { + const validate = (value: string) => + validateCapabilityFieldInput(EMAIL_CAPABILITY, 'SMTP_PORT', value) + expect(validate('587')).toBeUndefined() + expect(validate('0')).toContain('between 1 and 65535') + expect(validate('587.5')).toContain('between 1 and 65535') + }) + + it('accepts only HTTP(S) S3 endpoints', () => { + const validate = (value: string) => + validateCapabilityFieldInput(STORAGE_CAPABILITY, 'S3_ENDPOINT', value) + expect(validate('https://account.r2.cloudflarestorage.com')).toBeUndefined() + expect(validate('http://minio:9000')).toBeUndefined() + expect(validate('ftp://storage.example.com')).toContain('http:// or https://') + expect(validate('not-a-url')).toContain('http:// or https://') + }) + + it('requires complete inline service-account JSON', () => { + const validate = (value: string) => + validateCapabilityFieldInput(EMAIL_CAPABILITY, 'GMAIL_CREDENTIALS_JSON', value) + expect( + validate(JSON.stringify({ client_email: 'service@example.com', private_key: 'secret' })) + ).toBeUndefined() + expect(validate('{"client_email":"service@example.com"}')).toContain( + 'client_email and private_key' + ) + }) +}) diff --git a/scripts/setup/steps.ts b/scripts/setup/steps.ts index 1d3edf6afb9..caf3a49c226 100644 --- a/scripts/setup/steps.ts +++ b/scripts/setup/steps.ts @@ -11,7 +11,7 @@ import { } from './env-files.ts' import * as p from './prompter.ts' import { link, theme } from './theme.ts' -import { FLAG_TWINS, hasMailProvider, LOGIN_PROVIDERS, SELF_HOST_UNLOCKS } from './twins.ts' +import { FLAG_TWINS, LOGIN_PROVIDERS, SELF_HOST_UNLOCKS } from './twins.ts' /** Where the Chat key is minted when SIM_CLI_AUTH_ORIGIN is unset. */ const DEFAULT_CLI_AUTH_ORIGIN = 'https://www.sim.ai' @@ -149,116 +149,6 @@ export async function promptLlmKeys( return values } -type StorageBackend = 'local' | 's3' | 's3compat' | 'azure' | 'gcs' - -function detectStorageBackend(vars: Map): StorageBackend { - if (vars.get('AZURE_CONNECTION_STRING') || vars.get('AZURE_ACCOUNT_NAME')) return 'azure' - if (vars.get('S3_ENDPOINT')) return 's3compat' - if (vars.get('S3_BUCKET_NAME') || vars.get('AWS_REGION')) return 's3' - if (vars.get('GCS_BUCKET_NAME')) return 'gcs' - return 'local' -} - -async function required(message: string, initialValue?: string): Promise { - return p.text({ message, initialValue, validate: (v) => (v ? undefined : 'required') }) -} - -/** - * Custom-flow storage step. Local disk is the default; a cloud backend is - * strongly recommended for containerized deployments (uploads are ephemeral - * there). Returns the env vars for the chosen backend, or null to keep local. - */ -export async function promptStorage( - vars: Map, - containerized: boolean -): Promise | null> { - const current = detectStorageBackend(vars) - const backend = await p.select({ - message: 'File storage?', - options: [ - { - value: 'local', - label: 'Local disk', - hint: containerized - ? 'files live in the container — LOST on restart; fine only for evaluation' - : 'fine for local dev (external-fetch flows like Instagram publish need cloud storage)', - }, - { value: 's3', label: 'AWS S3', hint: 'region + bucket; keys optional with IAM/IRSA' }, - { - value: 's3compat', - label: 'S3-compatible (R2, MinIO, B2)', - hint: 'custom endpoint — fully self-hostable with MinIO', - }, - { value: 'azure', label: 'Azure Blob', hint: 'connection string or account name + key' }, - { - value: 'gcs', - label: 'Google Cloud Storage', - hint: 'bucket; credentials via ADC by default', - }, - ], - initialValue: current, - }) - if (backend === 'local') return null - - const values: Record = {} - if (backend === 's3' || backend === 's3compat') { - if (backend === 's3compat') { - values.S3_ENDPOINT = await required( - 'S3_ENDPOINT (e.g. https://.r2.cloudflarestorage.com)', - vars.get('S3_ENDPOINT') - ) - const pathStyle = await p.confirm({ - message: 'Force path-style addressing? (required for MinIO/Ceph, not for R2)', - initialValue: false, - }) - if (pathStyle) values.S3_FORCE_PATH_STYLE = 'true' - } - values.AWS_REGION = await required( - 'AWS_REGION', - vars.get('AWS_REGION') ?? (backend === 's3compat' ? 'auto' : undefined) - ) - values.S3_BUCKET_NAME = await required('S3_BUCKET_NAME', vars.get('S3_BUCKET_NAME')) - const accessKey = await p.password({ - message: 'AWS_ACCESS_KEY_ID (empty = IAM/instance credential chain)', - }) - if (accessKey) { - values.AWS_ACCESS_KEY_ID = accessKey - values.AWS_SECRET_ACCESS_KEY = await p.password({ - message: 'AWS_SECRET_ACCESS_KEY', - validate: (v) => (v ? undefined : 'required when an access key id is set'), - }) - } - } else if (backend === 'azure') { - const connectionString = await p.password({ - message: 'AZURE_CONNECTION_STRING (empty = use account name + key)', - }) - if (connectionString) { - values.AZURE_CONNECTION_STRING = connectionString - } else { - values.AZURE_ACCOUNT_NAME = await required( - 'AZURE_ACCOUNT_NAME', - vars.get('AZURE_ACCOUNT_NAME') - ) - values.AZURE_ACCOUNT_KEY = await p.password({ - message: 'AZURE_ACCOUNT_KEY', - validate: (v) => (v ? undefined : 'required'), - }) - } - values.AZURE_STORAGE_CONTAINER_NAME = await required( - 'AZURE_STORAGE_CONTAINER_NAME', - vars.get('AZURE_STORAGE_CONTAINER_NAME') ?? 'sim-files' - ) - } else { - values.GCS_BUCKET_NAME = await required('GCS_BUCKET_NAME', vars.get('GCS_BUCKET_NAME')) - p.log.info( - theme.muted( - 'Credentials use Application Default Credentials unless GCS_CREDENTIALS_JSON is set.' - ) - ) - } - return values -} - const PROVIDER_CONSOLES: Record = { google: 'https://console.cloud.google.com/apis/credentials', github: 'https://github.com/settings/developers', @@ -276,7 +166,7 @@ export async function promptSignInProviders( options: LOGIN_PROVIDERS.map((prov) => ({ value: prov.id, label: prov.label, - hint: configured.includes(prov.id) ? 'already configured' : undefined, + hint: configured.includes(prov.id) ? 'Currently used' : undefined, })), initialValues: configured, }) @@ -288,59 +178,20 @@ export async function promptSignInProviders( `${provider.label}: create an OAuth app at ${link(PROVIDER_CONSOLES[id], PROVIDER_CONSOLES[id])}\n Redirect URI: ${theme.command(`${appUrl}/api/auth/callback/${id}`)}` ) values[provider.idKey] = await p.text({ - message: provider.idKey, + message: `${provider.idKey}${vars.has(provider.idKey) ? ' (Currently used)' : ''}`, initialValue: vars.get(provider.idKey), validate: (v) => (v ? undefined : 'required'), }) - values[provider.secretKey] = await p.password({ - message: provider.secretKey, - validate: (v) => (v ? undefined : 'required'), + const existingSecret = vars.get(provider.secretKey) + const secret = await p.password({ + message: existingSecret + ? `${provider.secretKey} (Currently used); leave empty to keep it` + : provider.secretKey, + validate: (value) => (value || existingSecret ? undefined : 'required'), }) - } - return values -} - -/** Email step: console logging is the default; MailHog is the one-tap local option. */ -export async function promptEmail(vars: Map): Promise> { - const choice = await p.select({ - message: 'Email sending?', - options: [ - { - value: 'console', - label: 'None', - hint: 'emails are logged to the console — fine for local', - }, - { value: 'mailhog', label: 'MailHog (local)', hint: 'wires SMTP to localhost:1025' }, - { value: 'resend', label: 'Resend', hint: 'paste an API key' }, - { value: 'smtp', label: 'SMTP', hint: 'any SMTP relay' }, - ], - initialValue: hasMailProvider(vars) ? (vars.get('SMTP_HOST') ? 'smtp' : 'resend') : 'console', - }) - if (choice === 'console') return {} - if (choice === 'mailhog') return { SMTP_HOST: 'localhost', SMTP_PORT: '1025' } - if (choice === 'resend') { - return { - RESEND_API_KEY: await p.password({ - message: 'RESEND_API_KEY', - validate: (v) => (v ? undefined : 'required'), - }), - } - } - const values: Record = { - SMTP_HOST: await p.text({ - message: 'SMTP_HOST', - initialValue: vars.get('SMTP_HOST'), - validate: (v) => (v ? undefined : 'required'), - }), - SMTP_PORT: await p.text({ message: 'SMTP_PORT', initialValue: vars.get('SMTP_PORT') ?? '587' }), - } - const user = await p.text({ - message: 'SMTP_USER (empty for unauthenticated relays)', - defaultValue: '', - }) - if (user) { - values.SMTP_USER = user - values.SMTP_PASS = await p.password({ message: 'SMTP_PASS' }) + const resolvedSecret = secret || existingSecret + if (!resolvedSecret) throw new Error(`${provider.secretKey} was not provided`) + values[provider.secretKey] = resolvedSecret } return values } diff --git a/scripts/setup/twins.ts b/scripts/setup/twins.ts index 66ca5b2d39f..1f3374ca762 100644 --- a/scripts/setup/twins.ts +++ b/scripts/setup/twins.ts @@ -1,3 +1,8 @@ +import { + EMAIL_CAPABILITY, + inspectCapability, +} from '../../apps/sim/lib/core/config/env-capabilities.ts' + /** * Server/client feature-flag pairs that must be set together — server code * reads the bare var, the browser bundle reads the NEXT_PUBLIC_ twin @@ -51,16 +56,13 @@ export const SELF_HOST_UNLOCKS: ReadonlyArray<{ server: string; label: string; h }, ] -const MAIL_PROVIDER_KEYS = [ - 'RESEND_API_KEY', - 'AWS_SES_REGION', - 'SMTP_HOST', - 'AZURE_ACS_CONNECTION_STRING', - 'GMAIL_CREDENTIALS_JSON', -] as const - -export function hasMailProvider(vars: Map): boolean { - return MAIL_PROVIDER_KEYS.some((key) => vars.get(key)) +export function getConfiguredMailProvider(vars: Map): string { + const inspection = inspectCapability(EMAIL_CAPABILITY, vars) + return ( + inspection.providerIds[0] ?? + inspection.providers.find((provider) => provider.active)?.id ?? + 'console' + ) } export const LOGIN_PROVIDERS = [