diff --git a/.changeset/account-router-listing-cost.md b/.changeset/account-router-listing-cost.md new file mode 100644 index 0000000000..2af7995b1e --- /dev/null +++ b/.changeset/account-router-listing-cost.md @@ -0,0 +1,20 @@ +--- +"apps": patch +"@executor-js/sdk": patch +--- + +`accountRouter` merges a listing's tools in time that follows the distinct schema definitions +they hold, rather than their number times the definitions each tool reaches. Routers give every +tool that reaches a definition the same object, and a densely connected API reaches most of its +definitions from every tool: Stripe's 610 tools name about 330,000 `$defs` entries but only 1,537 +distinct definitions. Merging wrote each tool's whole schemas to compare its accounts', wrote each +of its definitions again to combine them, and checked each merged tool's schemas on their own, so +a cold Stripe listing spent most of a Cloud app Worker's CPU time there: about 17 seconds with +one account, and with three it reached the Worker's CPU limit. A listing now reads each definition object once, and +the catalog read checks every tool's schemas in its existing single pass. Listings are unchanged. +Apps get this when they are rebuilt on this `apps` release. + +When loading an app's tools stops at a resource limit, the recovery still allows one retry, then +says to stop retrying unchanged and investigate: rebuild the app on the host's `apps` version if it +was built on an older one, otherwise reduce what the app does while listing. The limit is matched +from the failure's text, so the recovery does not claim a repeat proves its cause. diff --git a/.changeset/agent-billing-and-connection-reads.md b/.changeset/agent-billing-and-connection-reads.md new file mode 100644 index 0000000000..9403083262 --- /dev/null +++ b/.changeset/agent-billing-and-connection-reads.md @@ -0,0 +1,9 @@ +--- +"@executor-js/hosted-server": patch +"@executor-js/hosted-cloud": patch +--- + +Agents can read and manage billing on Cloud: owners' and admins' agents see the +plans and member limit and get checkout and billing portal links to open. Agents +can also list the member's scoped connections. Creating, editing and revoking a +connection still needs the dashboard. diff --git a/.changeset/execute-file-content.md b/.changeset/execute-file-content.md new file mode 100644 index 0000000000..9a28b68ee1 --- /dev/null +++ b/.changeset/execute-file-content.md @@ -0,0 +1,17 @@ +--- +"@executor-js/mcp": patch +"@executor-js/app-management": patch +"@executor-js/app-templates": patch +--- + +Files that app tools return now reach the MCP client as embedded resources instead of base64 text. +When the result of `execute`, `call` or `resume` holds an MCP resource block with binary contents, +`{ type: "resource", resource: { uri, mimeType, blob } }`, it is sent once as its own content block +after the JSON text, as images and audio are, and the JSON keeps the block without its bytes plus +`contentIndex`. A client that saves resources as files, such as Claude Code, can then read a PDF an +app returns, such as an email attachment. Text resources stay in the JSON. + +`executor call`, `execute` and `resume` now write every image, audio and file block a result holds +to a file, in a new temporary directory or the one `--files` names, and print each file's `path` +beside its `contentIndex`. Before, the printed JSON kept only the placeholder, so the bytes were +lost. diff --git a/.changeset/motel-resend-once.md b/.changeset/motel-resend-once.md deleted file mode 100644 index 9248cb9ae9..0000000000 --- a/.changeset/motel-resend-once.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -"@executor-js/telemetry": patch -"@executor-js/hosted-self-host": patch ---- - -The bundled Motel collector no longer rewrites an export the server sends -again. An export that waits past its 5-second timeout is resent, and the -collector used to delete and rewrite every span of the resend. On a full store -under traffic one resent export took 16 seconds, so the collector fell further -behind with each one. A resent span identical to the one already stored is now -skipped. diff --git a/.changeset/motel-store-low-water.md b/.changeset/motel-store-low-water.md deleted file mode 100644 index 4f573a3521..0000000000 --- a/.changeset/motel-store-low-water.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -"@executor-js/telemetry": patch -"@executor-js/hosted-self-host": patch ---- - -The bundled Motel collector no longer churns once its store is full. It used to -evict back to exactly its 1 GiB bound, so the next export crossed it again, and -every export, about one a second on an idle server, paid for an eviction pass -that scanned the whole store. A full store now evicts down to 90% of its bound -and then stores exports without evicting until it fills again. On a full 1 GiB -store with an idle server's exports, the collector's CPU fell from about 9% of a -core to under 1%, its reads from about 30 MiB/s to about 1 MiB/s, and each -export's time from about 120 ms to about 20 ms. diff --git a/.changeset/rust-telemetry-collector.md b/.changeset/rust-telemetry-collector.md new file mode 100644 index 0000000000..ba7d8c25b4 --- /dev/null +++ b/.changeset/rust-telemetry-collector.md @@ -0,0 +1,16 @@ +--- +"@executor-js/telemetry": patch +"@executor-js/hosted-self-host": patch +--- + +The bundled telemetry collector is now `executor-collector`, written in Rust, +in place of Motel. Local, desktop and self-host builds start it where they +started Motel, with the same bounds: seven days and 1 GiB of telemetry, and +exports of at most 16 MiB, four at a time. Measured, it holds 7 to 18 MiB at rest and 27 to 38 MiB under the +product's load, and keeps up at its bound, where Motel fell behind. Locally it serves the trace +viewer at the address in `diagnostics/collector.json`; it stores telemetry in +`diagnostics/collector`. Self-host keeps `EXECUTOR_MOTEL_DATA_DIR`, whose name +is historical, and `executor-host telemetry`. Telemetry Motel stored, up to +about 1 GiB, is neither read nor removed: delete `diagnostics/motel` in the +data directory (by default `~/.executor/v2/cli/diagnostics/motel`), or on a +self-hosted volume run `docker exec rm -rf /app/motel-data/motel`. diff --git a/.changeset/untraced-event-poll.md b/.changeset/untraced-event-poll.md index 10ada67374..0469b3155e 100644 --- a/.changeset/untraced-event-poll.md +++ b/.changeset/untraced-event-poll.md @@ -5,4 +5,4 @@ Self-host and local check for due event deliveries every two seconds. That check no longer records traces when it finds nothing to deliver; each delivery attempt and each subscription it expires is still traced. An idle self-host -instance no longer fills the bundled Motel store with polling spans. +instance no longer fills the bundled collector's store with polling spans. diff --git a/.changeset/untraced-idle-polls.md b/.changeset/untraced-idle-polls.md index 8a03940249..f2611c3188 100644 --- a/.changeset/untraced-idle-polls.md +++ b/.changeset/untraced-idle-polls.md @@ -7,5 +7,5 @@ The scheduler's check for due runs, every second on self-host and local, and self-host's check for queued provisioning jobs, every second, no longer record traces when they find nothing to do. Each scheduled operation a check finds is still traced, in its own `schedule.dispatch` trace, and so is each provisioning -job. An idle self-host instance no longer fills the bundled Motel store with +job. An idle self-host instance no longer fills the bundled collector's store with polling spans. diff --git a/.github/workflows/fork-ci.yml b/.github/workflows/fork-ci.yml index dd17f20bed..37d2a12b67 100644 --- a/.github/workflows/fork-ci.yml +++ b/.github/workflows/fork-ci.yml @@ -43,9 +43,10 @@ jobs: echo "::endgroup::" done - # Self-host scenarios for scoped-connection widening, OAuth registration, app - # User-Agent, and load-failure containment, plus local MCP search. Both jobs use - # Bun 1.4.4: the collector build and upstream's new-file patches need the newer Bun. + # Self-host scenarios for scoped-connection widening, OAuth registration and held + # scopes, app User-Agent, private-registry build egress, load-failure containment + # and the apps SDK release gate, plus local MCP search. Both jobs use Bun 1.4.4: + # the collector build and upstream's new-file patches need the newer Bun. # No extra secrets: the runner mints its own API key. CI disables the prepare cache. security-e2e: runs-on: ubuntu-latest @@ -63,10 +64,21 @@ jobs: - run: bun install --frozen-lockfile - name: Install Playwright Chromium run: bunx playwright install --with-deps chromium + # Upstream CI downloads a separately built collector after prepare; this fork + # has no such artifact job. Build and qualify the exact pinned crate here. + - name: Build and verify the Linux x64 telemetry collector + run: node packages/telemetry/scripts/build-collector.ts --target linux-x64 --install-toolchain build check smoke - run: bun run e2e:prepare + # Filters match scheduled scenario titles from e2e/test-plan.ts, not suite names; the + # "App dependency registry" suite's only scenario is selected by the private-fetch prefix. - name: Security scenarios run: | - bun run e2e:self-host -- --test-name 'application_type|fetch without a User-Agent names Executor|cannot create an all-apps connection|cannot widen a connection or mint|fails to load fails only its own reads' + bun run e2e:self-host -- --test-name 'application_type|fetch without a User-Agent names Executor|cannot create an all-apps connection|cannot widen a connection or mint|fails to load fails only its own reads|with private fetch off, a build|App dependency registry|keeps renewing grants that hold a scope|own Google client still signs in gmail.modify' + # The apps SDK release gate is its own suite, not a self-host scenario. + - name: SDK release gate scenarios + run: | + bun run apps:build + bun run e2e:apps-published -- --testNamePattern 'self-host gate' - name: Tool error contract scenarios run: bun run e2e:self-host --test-name 'Every error tools/call and tools/run declare|Every error a write reports after its app code received the call' - name: Operator feedback scenarios diff --git a/AGENTS.md b/AGENTS.md index e3b7408fa7..86975d9c65 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -127,9 +127,12 @@ Its `cargo test` suite in `tests/` is limited to seven files, which share export limit emptying the store and a raised one keeping it, every transition of the read gate, a gate opening when checkpoints fail, readers held at the gate so - exports are not refused, reads during eviction, and resident memory returned - after a burst. - These would be E2E scenarios, but nothing in `e2e/` builds the binary yet. + exports are not refused, reads during eviction, resident memory returned + after a burst, and the viewer's files: served only from what was read at start, + never a path a request names, only GET and HEAD, and every viewer directory it + cannot serve ending the start before the store is touched. + These run the binary alone, with stores, clocks and resource limits a scenario + through the product cannot set up. CI runs it in the `collector-native` job when the crate, its build script (`packages/telemetry/scripts/build-collector.ts`) or the `collector` and `rustup` actions in @@ -156,6 +159,9 @@ tests run the deployed suite. When a change is cross-cutting, pick one or two scenarios per changed path and name them in the handoff. Investigate a failure CI reports instead of re-running suites to look for one. +A starting run prunes the workspace's `.local/e2e` (`e2e/sdk/run-retention.ts`): it removes runs older than 3 days or past the cap, and shrinks older passed runs, and failed ones a day after they ended, to their logs, reports and evidence (`shrunk.json` says what went). A run that ended in the last 2 hours stays whole unless the 3 GiB cap needs the space. +To keep every run whole while investigating, `touch .local/e2e/keep-all-runs`; remove it when done. + For authenticated testing and bug reproduction, use [test accounts](notes/test-accounts.md). The fixture command provisions synthetic users, organization roles and short-lived sessions for self-host, @@ -239,7 +245,11 @@ a `workflow_call` workflow, so another repository can call the same jobs. ### Choosing a PR's E2E scenarios A pull request runs only the E2E scenarios it selects. Pushes to `main` run the -full suite. The static checks (`check` and the jobs it gates, `apps-version`, +full suite. The selection reaches the jobs as title patterns; no value it puts in +an environment variable or argument may approach Linux's 128 KiB limit, and a case in +`e2e/tests/ci-selection.spec.ts` fails at a quarter of it, naming the value. Make room by +splitting (more shards) or by passing it through a file: scenario titles are not a budget, +so never shorten them to fit. The static checks (`check` and the jobs it gates, `apps-version`, `self-host-native`) always run, except in a skipped stack layer. Put exactly one fenced `e2e` block in the PR description, listing spec files from `e2e/tests/`: @@ -250,12 +260,24 @@ invitation-roles.spec.ts ``` ```` +Before the first push, commit and check the description as CI's `select` job will: +`bun run e2e:check-description --file description.md` (or `-` for stdin, or `--pr ` +once the PR is open, from a checkout at its head commit). It gives CI's verdict and prints what +would run; a `skip` can only be checked once the layer above is open. An edit to the description +after the push needs no re-run unless it changes the selection. + Spec files the PR adds or changes are always included, as are the scenarios that guard a dependency patch when the PR's tree pins that dependency or its patch differently from main (`patchGuards` in `e2e/ci-selection.ts`): Bun skips a stale patch without an error. So are the scenarios that assert a contract one file states in full when the tree's copy differs from main's (`fileGuards`): a new host protocol in `app-protocols.ts` runs `app-package.spec.ts`, which reads back the supported list. +The Executor app's skill texts and the MCP instructions generated from them are read back by +scenarios too (`e2e/file-guards.ts`), so changing them runs those scenarios. +A change to the code MCP search and sessions run through (`e2e/mcp-search-guard.ts`) also runs +the Cloud MCP search and session scenarios. That is a floor, not a substitute for naming the spec +files a change can affect: those Cloud scenarios live in `cloud-database-placement.spec.ts` and +`mcp-catalog-scale.spec.ts`, whose names do not say search. A changed spec file's Cloud scenarios that otherwise first run in Cloud tests on main, after merge, run in the `cloud-product` job when managed local Cloud can run them. The ones that need a deployed stage (`runtime: "attached"`) get a warning with the `e2e:deployed` command that runs @@ -297,7 +319,12 @@ checks the combined change: its static checks run against the whole tree, scenarios for every layer's changes. `skip` fails the `select` job unless another open PR targets the layer's branch, so a lone PR or the top layer cannot skip. The `select` job waits up to 3 minutes for that PR to open, so open the stack's layers together -(`gh stack submit` does). +(`gh stack submit` does). It then checks that that PR's latest run on its head was built on +this layer's head and compared Worker sizes with the base the stack ends on +([`e2e/ci-skip-check.ts`](e2e/ci-skip-check.ts)). For now it only logs what it would +decide (`skip-check` in the select log and summary); once enforced, it fails and says which command +runs the layer above again. Never write `skip` on a long-lived branch's PR into `main`: it must be +compared with `main` itself. After changing a lower layer, rebase the layers above it so the top runs again. Merge the stack only when the top layer passes, bottom first, without pausing between layers: each merge deploys production. @@ -319,8 +346,11 @@ Cloudflare compiles and evaluates the modules a Worker's entry imports staticall when an isolate starts, about 40 ms for each megabyte on every cold request, and refuses an upload whose startup takes more than a second of CPU. The `site` job compares each Cloud Worker's upload, and the bytes its entry imports statically, -with `main`, and fails a pull request that adds more than 50 KB to either, about -2 ms, unless the description allows it: +with the base of the bottom of the pull request's stack, and fails a pull request +that adds more than 50 KB to either, about 2 ms, unless the description allows it. +A stack that ends on `main` is compared with `main`; one that ends on another +branch is compared with that branch, which is charged its total against `main` +when it merges there: ````md ```worker-growth @@ -338,6 +368,17 @@ backstops with room above each Worker's size; the deploy enforces them too. Change one only in a PR that does nothing else and says what the added cold-start time buys. Never raise one to fit a change. +A push to `main` is held to the same rule against the commit before it: growth +over 50 KB fails `site` on `main` unless the descriptions of the pull requests the +push merges allow it (summed by Worker). That catches growth no pull request was +compared with `main` for, such as a layer retargeted onto `main` after its run, a +`skip` layer above a long-lived branch, or a direct push. A red `site` on `main` +does not stop the deploy, which has already started; it is a red `CI` on `main` +that someone must answer: add the allowance to the merged pull request's +description and rerun the failed job, which reads the descriptions afresh, or +shrink the Worker in a new pull request. Sizes that cannot be had, descriptions +that cannot be read, and commits no pull request merged are notices, never failures. + A ruleset on `main` requires a pull request whose `checks / check` and `checks / e2e` passed. When GitHub refuses a merge as blocked ("the base branch policy prohibits the merge"), a required check is pending, failed or missing: wait @@ -449,9 +490,9 @@ scenarios on Linux instead of moving them to a Mac. site and dashboard, with its own origins, into the one `apps/hosted/cloud/.generated/site` that both serve. On a full run `cloud-product` passes what one local Cloud holds, and each of its local Clouds has a runner of its own; the longest, about five and a half minutes, waits out - the MCP session restore scenario's 270-second idle. Each runner builds the apps package and - Motel once before its runs; runs only serve that Motel bundle, because every target in a run - shares it. + the MCP session restore scenario's 270-second idle. Each runner builds the apps package and the + telemetry collector and its viewer once before its runs; runs only start that build, because + every target in a run shares it. - `e2e` passes when `select` succeeded, every E2E job it chose passed and every other one was skipped; it runs even after a cancellation, which fails it. The matrix runners' check names change with the selection, so this is the one E2E check to require. diff --git a/README.md b/README.md index 808207d90e..02e29221d4 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,9 @@ running Executor from source, the repository layout and how the pieces fit together. Executor 1 lives on the [`main`](https://github.com/UsefulSoftwareCo/executor/tree/main) branch. +To report a security problem, follow [SECURITY.md](SECURITY.md) rather than +opening a public issue. + ## License [MIT](LICENSE) diff --git a/RELEASING.md b/RELEASING.md index 9ee25390ed..71fae25b71 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -48,7 +48,9 @@ that publish build the collector without restoring from the cache, because an en cache scope is not proof of what built it; the cache rules judge restores by workflow file, so the paced check and `build` runs build it too (`.github/actions/collector`). A narrowed build runs a collector job only for the targets its packaging jobs run on, and none when it runs only the image. -Nothing packages the collector yet; Motel is still the bundled collector. +Every packaging job packages the collector it fetched, executable before anything is signed, and +builds the viewer the CLI and desktop serve from it; the image builds its own collector in its +Dockerfile's `collector` stage. A Bun runtime release built from a pull request's branch, as one that changes the lockdown patch needs, is the one main's publishing runs use from then on. If that @@ -246,6 +248,20 @@ existing release or npm version stops publication for inspection. Never republish an accepted immutable npm version merely because its registry entry is still propagating. +No publish step moves a channel backwards. The job's first step after setup +reads every channel (npm's tag, the image's channel tag, the published formula, +the desktop feed) and refuses a release older than any of them, and each step +that moves a channel checks its own again just before it moves it +(`scripts/releases/channel-guard.ts`). After a failure the job summary says what +the release made public and what it did not, read from the public surfaces, and +whether re-running the failed jobs would finish it +(`scripts/releases/publication-report.ts`). + +A re-run uses the workflow file of the run's own commit, so runs started before +those checks were added carry none. Never re-run the publish job of an older +release once a newer one exists: beta.51's run (43640118286) and beta.52's, +re-run after a newer beta, would point the image's `beta` tag back at it. + The homepage resolves desktop downloads in the browser from GitHub's public release list, choosing the newest published release with this major version's tag prefix. A merged version bump keeps linking to the previous release until @@ -260,7 +276,12 @@ A successful upload alone does not establish public availability. ## Stable cutover Get explicit approval for the v2 stable release. Change the version to `2.0.0` -and dispatch the same workflow with **latest**. The version, tags, filenames +and dispatch the same workflow with **latest**. npm already serves `executor@2.0.0`: +it was published on 2026-03-16 from a stale historical tag and is deprecated. +npm never accepts a version twice, so a release numbered `2.0.0` cannot be +published, and the publish job's preflight refuses it before anything moves. The +first stable release needs a number above the betas that npm does not hold; the +choice is the owner's. The version, tags, filenames and links change together. The fixed desktop identity `com.usefulsoftware.executor.v2`, product name **Executor 2**, profile **Executor v2**, and CLI data directory `~/.executor/v2/cli` remain unchanged. Stable publication writes the desktop diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..35b57c3f5a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,20 @@ +# Security + +If you find a security problem in Executor, please tell us privately. Either +channel is fine: + +- Email [security@executor.sh](mailto:security@executor.sh). +- Open a private report through + [GitHub private vulnerability reporting](https://github.com/UsefulSoftwareCo/executor/security/advisories/new). + +Please do not open a public issue or pull request for a vulnerability. + +Include what you can of: + +- The affected version or commit. +- Where it runs: hosted Executor at executor.sh, a self-hosted install, or + local Executor. +- Steps to reproduce. +- The impact: what an attacker can do, and what they need first. + +We do not run a bug bounty. diff --git a/apps/cli/CHANGELOG.md b/apps/cli/CHANGELOG.md index 1354c1524b..7783f070a5 100644 --- a/apps/cli/CHANGELOG.md +++ b/apps/cli/CHANGELOG.md @@ -1,5 +1,41 @@ # executor +## 2.0.0-beta.59 + +Beta of the Executor local runtime, CLI, desktop, and self-host Docker image. + +## 2.0.0-beta.58 + +Beta of the Executor local runtime, CLI, desktop, and self-host Docker image. + +## 2.0.0-beta.57 + +Beta of the Executor local runtime, CLI, desktop, and self-host Docker image. + +## 2.0.0-beta.56 + +Beta of the Executor local runtime, CLI, desktop, and self-host Docker image. + +## 2.0.0-beta.55 + +Beta of the Executor local runtime, CLI, desktop, and self-host Docker image. + +## 2.0.0-beta.54 + +Beta of the Executor local runtime, CLI, desktop, and self-host Docker image. + +## 2.0.0-beta.53 + +Beta of the Executor local runtime, CLI, desktop, and self-host Docker image. + +## 2.0.0-beta.52 + +Beta of the Executor local runtime, CLI, desktop, and self-host Docker image. + +## 2.0.0-beta.51 + +Beta of the Executor local runtime, CLI, desktop, and self-host Docker image. + ## 2.0.0-beta.50 Beta of the Executor local runtime, CLI, desktop, and self-host Docker image. diff --git a/apps/cli/package.json b/apps/cli/package.json index b136ff3546..f5ad0bd42e 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "executor", - "version": "2.0.0-beta.50", + "version": "2.0.0-beta.59", "private": true, "description": "Run Executor locally and connect your agents to your apps.", "homepage": "https://executor.sh", diff --git a/apps/docs/content/author-an-app.mdx b/apps/docs/content/author-an-app.mdx index d7deae8e15..2fca17e145 100644 --- a/apps/docs/content/author-an-app.mdx +++ b/apps/docs/content/author-an-app.mdx @@ -540,7 +540,21 @@ Deploying reads the chosen commit and leaves the Git branch unchanged. Ordinary Git pushes save source without deploying it. Code, every file outside `ui/public/`, is UTF-8 text: at most 4,096 files and -16 MiB of file contents. Files under `ui/public/` are served as they are and may +16 MiB of file contents. Each of the app's own JavaScript or TypeScript files is +at most 2,000,000 bytes (2 MB). JSON files are outside this limit, so keep a +large document such as an OpenAPI description in JSON. The host checks this +limit while it builds, not the CLI before uploading: a build that reaches a +larger file of the app's own fails with `VersionBuildFailed`, naming the file, +its size and the limit, and no app or version is created. + +Files inside the npm packages an app imports are not limited: a large file +there is still compiled. On self-host and the local server, a build compiles on +the thread that runs apps, so while it compiles a large file, of the app's own +or a package's, every call to an app waits, and so does any other request +served from that thread. A package module of 3,000,000 bytes of dense code held +app calls for about 1.2 seconds in our tests; a larger one holds them for +longer. +Files under `ui/public/` are served as they are and may be binary, such as images, fonts, audio and WebAssembly: at most 10,000 files, each at most 32 MB and 1 GB in total. Builds never load them; each deploy copies them to the host's storage and serves them from there. The CLI checks a directory diff --git a/apps/docs/content/mcp.md b/apps/docs/content/mcp.md index 0a23672d5a..e1222a1a20 100644 --- a/apps/docs/content/mcp.md +++ b/apps/docs/content/mcp.md @@ -99,16 +99,23 @@ may still have made changes. `call` lists it the same way. An outcome of `success` only means no tool error was reported; check the result before treating its action as done. -Images and audio reach the agent as media. Wherever the value `execute`, `call` -or `resume` returns holds `{ type: "image", mimeType, data }` or -`{ type: "audio", mimeType, data }`, with `data` in standard or URL-safe base64, -the response carries that block as its own MCP content after the JSON text, once -even if the value repeats it. The JSON keeps `{ type, mimeType, contentIndex }` -in its place, the block's index in the response's `content`. MCP image and audio -results, such as screenshots, already have that shape. A program can show bytes -a tool returns another way, such as an email attachment's base64, by returning -them in it. Inside a program the block is whole, so its data can be passed on to -another tool. +Images, audio and files reach the agent as MCP content, not base64 text. +Wherever the value `execute`, `call` or `resume` returns holds +`{ type: "image", mimeType, data }`, `{ type: "audio", mimeType, data }` or an +embedded resource with binary contents, +`{ type: "resource", resource: { uri, mimeType, blob } }`, with its bytes in +standard or URL-safe base64, the response carries that block as its own MCP +content after the JSON text, once even if the value repeats it. The JSON keeps +the block without its bytes, plus `contentIndex`, the block's index in the +response's `content`. MCP image and audio results, such as screenshots, already +have that shape. Each client decides how to present a resource; Claude Code, for +example, saves it as a file the model can open, so a PDF attachment can be read. +A text resource stays in the JSON. A program can show bytes a tool returns +another way, such as `{ base64 }`, by returning them in one of these shapes. +Inside a program the block is whole, so its data can be passed on to another +tool. The `executor call`, `execute` and `resume` commands write each block to a +file, in a new temporary directory or the one `--files` names, and print its +`path` beside its `contentIndex`. The interpreter has no `fetch`, no `process`, no filesystem, no timers and no imports. Its only way out is a tool call, which enters the trusted runtime where your @@ -169,6 +176,10 @@ revoking the connection revokes every grant issued through it. Personal access tokens and the local server key are not limited by a connection; they keep full access at the plain URL and are refused at a connection's URL. +An agent can list your scoped connections with the Executor app, but creating, +editing and revoking them happens on the dashboard, so an agent never widens its +own access. + On hosted and self-host, **Connected agents** on the same page lists each client you approved, with the apps it can reach, and revokes its grant. diff --git a/apps/docs/content/run/self-host.mdx b/apps/docs/content/run/self-host.mdx index 66da33f737..4df67839e1 100644 --- a/apps/docs/content/run/self-host.mdx +++ b/apps/docs/content/run/self-host.mdx @@ -62,7 +62,10 @@ the server and do not use the network. Thus the bundled Executor app works when that origin resolves to a private address, such as a Tailscale name. To let app code reach other private addresses, set `EXECUTOR_APPS_ALLOW_PRIVATE_FETCH=true`. The default is `false`. Every app in the instance shares that network position, -so grant it deliberately. +so grant it deliberately. App builds follow the same setting for every download +off the registry's origin: a dependency declared by tarball URL, and a tarball or +redirect the registry's metadata sends elsewhere. Downloads from the registry's +own origin, including a private `EXECUTOR_NPM_REGISTRY`, work either way. Each app runs in its own Worker for each set of accounts it is called with, and each loaded Worker holds its own memory. The server keeps at most @@ -106,7 +109,7 @@ from the internet, so turn this on only when that URL is reachable publicly over HTTPS. Startup fails for any other address on your own origin. Clients your users already registered or entered stay in use when you turn this on. -The bundled Motel collector records traces by default. You can export to your own collector instead. See +The bundled telemetry collector records traces by default. You can export to your own collector instead. See [Tracing](/run/tracing). ## Run it @@ -284,14 +287,17 @@ mounted at `/app/data`. This includes the database, app source and builds, app data and generated keys (`auth-secret.key` and `encryption.key`). The key files are readable only by the server user. Back up the whole volume. -Motel stores telemetry separately at `/app/motel-data`. Replacing the container +The telemetry collector stores telemetry separately at `/app/motel-data` (`EXECUTOR_MOTEL_DATA_DIR`; +the name is historical, from Motel, the collector earlier images ran). Replacing the container discards that telemetry by default. Mount a separate volume there if you need -to retain it. Motel keeps telemetry until the first of three bounds is reached: -seven days, one million spans or 1 GiB, evicting the oldest traces first. A busy -server reaches 1 GiB first, so it may keep only the last hours or minutes of -traces. When exports arrive faster than Motel can store them, it refuses the -excess rather than holding it in memory, and it counts every export it does not -store. The server log reports those losses. +to retain it. The collector keeps telemetry for seven days and at most 1 GiB, evicting the oldest +first. A busy server reaches 1 GiB first, so it may keep only the last hours of traces. When +exports arrive faster than it can store them, it refuses the excess rather than holding it in +memory, and it counts every export it does not store; `executor-host telemetry /api/ingest` +reports the counts. Images that ran Motel left its store, up to about 1 GiB, in a `motel` folder +there; the collector neither reads nor removes it. Without a volume at `/app/motel-data` it went with +the old container; with one, delete it by hand with +`docker exec executor-v2 rm -rf /app/motel-data/motel`. If you supply keys through environment variables, keep those values in your secret manager as well; Executor does not save those overrides to disk. Keep diff --git a/apps/docs/content/run/tracing.md b/apps/docs/content/run/tracing.md index 9106679a71..f04d25754a 100644 --- a/apps/docs/content/run/tracing.md +++ b/apps/docs/content/run/tracing.md @@ -10,36 +10,41 @@ they go. ## The bundled collector -Local, desktop and self-host builds package [Motel](https://github.com/kitlangton/motel), -a small OTLP store and viewer over a SQLite file. It is the default destination, -so traces are being recorded already. +Local, desktop and self-host builds package Executor's telemetry collector, a small OTLP store over +one SQLite file. It is the default destination, so traces are being recorded already. Local and +desktop builds also serve a trace viewer from it. Where it writes depends on where you run: | Deployment | Diagnostics directory | | ---------- | --------------------------------------------------------------------------- | | Local | `/diagnostics`, by default `~/.executor/v2/cli/diagnostics` | -| Self-host Docker | `EXECUTOR_MOTEL_DATA_DIR`, by default `/app/motel-data` | +| Self-host Docker | `EXECUTOR_MOTEL_DATA_DIR`, by default `/app/motel-data` (the name is historical) | The local diagnostics directory holds: -- `collector.json` — the collector's state, process ID, query URL and database - path. -- `motel/` — the stored traces and logs, as Motel's SQLite files. +- `collector.json` — the collector's state, process ID, URL and data directory. + Open the URL in a browser for the trace viewer. +- `collector/` — the stored traces and logs, in `telemetry.sqlite`. +- `motel/` — only after an upgrade from a version that ran Motel, the collector + Executor bundled before: its traces and logs, up to about 1 GiB. Executor neither + reads nor removes it. To reclaim the space, stop Executor and delete it, for the + default local directory with `rm -rf ~/.executor/v2/cli/diagnostics/motel`. - `executor-local.jsonl` or `executor-selfhost.jsonl` — Effect logs, written independently of the collector. The background Executor writes its logs only here. -Motel keeps seven days and targets 1 GiB. Each log file keeps four rotated -archives at about 10 MiB each. +The collector keeps seven days and at most 1 GiB, evicting the oldest first. Each log file keeps +four rotated archives at about 10 MiB each. -Motel runs in workerd. In self-host Docker it runs as its own workerd process -beside the product and keeps its SQLite files in the `motel` folder of -`EXECUTOR_MOTEL_DATA_DIR`. This directory must be separate from `/app/data`. Replacing the container discards telemetry by -default. Mount a separate volume at the Motel directory to retain it. The -container does not include Node or Bun. +In self-host Docker the collector runs as its own process beside the product and keeps its files in +`EXECUTOR_MOTEL_DATA_DIR`. This directory must be separate from `/app/data`. Replacing the +container discards telemetry by default. Mount a separate volume there to retain it. An image that +ran Motel left its store, up to about 1 GiB, in a `motel` folder of that directory, which the +collector neither reads nor removes. Without a volume there it went with the old container; with +one, delete it by hand: `docker exec executor-v2 rm -rf /app/motel-data/motel`. The container does not include Node or Bun, and does not serve the trace viewer. -Motel also stores the log records that Executor writes. A record written +The collector also stores the log records that Executor writes. A record written during a request has the trace ID of that request. Executor does not write a log record for each request, so most requests produce only a trace. Warnings, errors, and app page requests produce log records. An empty `/api/logs/search` @@ -63,8 +68,9 @@ That port is gone. Scripts that read `http://127.0.0.1:4318` must use at `127.0.0.1:4318` now sends to a port nothing listens on: remove it to use the bundled collector, or point it at your own collector. -The collector serves `/api/health`, `/api/traces`, -`/api/traces//spans`, `/api/logs/search` and `/openapi.json`. +The collector serves `/api/health`, `/api/ingest` (what it stored, refused and evicted), +`/api/services`, `/api/traces`, `/api/traces/`, `/api/traces//spans`, +`/api/traces//logs`, `/api/spans/search`, `/api/logs` and `/api/logs/search`. ## Which traces are recorded @@ -121,7 +127,7 @@ stays absent, which keeps the bundled collector as the destination. ## Turning off the bundled collector -Set a traces or logs endpoint. Executor then sends nothing to Motel: no +Set a traces or logs endpoint. Executor then sends nothing to the bundled collector: no traces and no logs. One variable is enough: ```bash @@ -136,18 +142,16 @@ kept anywhere. Set `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` too if your collector should receive logs, including the browser's. With only the logs endpoint set, traces are not recorded anywhere. -With either endpoint set, local and desktop builds do not start Motel at all. -In self-host Docker, the Motel workerd process still starts, because the host +With either endpoint set, local and desktop builds do not start the collector at all. +In self-host Docker, the collector process still starts, because the host always supervises it. It receives no new records and stays idle. Records it -stored before the switch remain until its retention removes them. A busy -server can spend more memory and CPU in Motel than in the product itself, so -an external collector also frees those resources. There is no setting that -skips the Docker Motel process. +stored before the switch remain until its retention removes them. There is no setting that +skips the Docker collector process. To see that the change took effect, check that new traces reach your -collector and that Motel records none. Switching does not delete what Motel -already stored, so a retained Motel volume still lists its earlier traces. A -few minutes after the restart, use Executor, then list the traces Motel +collector and that the bundled one records none. Switching does not delete what it +already stored, so a retained volume still lists its earlier traces. A +few minutes after the restart, use Executor, then list the traces the bundled collector received in the last two minutes. The result stays empty: ```bash @@ -156,7 +160,8 @@ docker exec executor-v2 executor-host telemetry '/api/traces?lookback=2m&limit=2 ## Using Motel on your own machine -Motel runs standalone too. It needs [Bun](https://bun.sh/) and listens on port +[Motel](https://github.com/kitlangton/motel), the collector Executor bundled before, runs +standalone and makes a convenient destination for a server you run yourself. It needs [Bun](https://bun.sh/) and listens on port `27686`. ```bash diff --git a/apps/hosted/cloud/scripts/edge-contract.ts b/apps/hosted/cloud/scripts/edge-contract.ts index ece081a1be..e8a477cbb4 100644 --- a/apps/hosted/cloud/scripts/edge-contract.ts +++ b/apps/hosted/cloud/scripts/edge-contract.ts @@ -66,6 +66,7 @@ const forwarded: ReadonlyArray = [ ["GET", "/og-image.png"], ["GET", "/pattern-graph-paper.svg"], ["GET", "/.well-known/agent-skills/index.json"], + ["GET", "/.well-known/security.txt"], ["POST", `${analyticsProxy}/e/?ip=0`], ["GET", `${analyticsProxy}/static/array.js`], ["POST", errorTunnel], diff --git a/apps/hosted/cloud/scripts/r2-write-fixture.ts b/apps/hosted/cloud/scripts/r2-write-fixture.ts new file mode 100644 index 0000000000..09db35a809 --- /dev/null +++ b/apps/hosted/cloud/scripts/r2-write-fixture.ts @@ -0,0 +1,170 @@ +/** + * Local-only probe of which R2 write failures Cloud's blob store repeats (`persistR2Object`), with a + * stand-in for the R2 binding's put that fails with R2's own error shapes, its code last in the + * message as the binding reports it, before succeeding. Prints, as JSON, each case's attempts and + * outcome, for `build-retain-storage.spec.ts` to judge. + * + * - `rate-limited`: 10058 (one write a second per key), then stored. + * - `internal`: 10001 (internal error), then stored. + * - `unavailable`: 10043 (temporarily unavailable), then stored. + * - `precondition`: 10031 (a failed condition): not repeated. + * - `persistent`: 10058 on every attempt: three attempts, then the failure. + * - `invalid-key`: 10020 (an invalid key): not repeated; Cloud's blob store keeps R2's 400 on its + * failure (`statuses`), and nothing on a rate limit or internal error. + * - `framework`: 17 builds keep their shared framework at once through the SDK's retain stage + * and this write (`writeR2Object`), against a stand-in put that refuses a key's writes faster + * than one a second with 10058: the write retry and the retain stage's look-again together. + * Counts are of writes to the framework's key only. + * - `attempts`: the attempts a write took, as its `storage.blob.put` span records them: one for a + * write stored at once, two after one rate-limit refusal, three for a persistent refusal. + */ +import { BlobStore, BuildId, type BlobStorage } from "@executor-js/sdk/core"; +import { retainWorkerBuild } from "@executor-js/sdk/workerd"; +import { Effect, Schema, Tracer } from "effect"; +import { persistR2Object, writeR2Object } from "../src/implementation/r2-write.ts"; + +const r2Error = (code: number) => new Error(`put: We encountered an internal error (${code})`); + +const cases = { + "rate-limited": [10058], + internal: [10001], + unavailable: [10043], + precondition: [10031], + persistent: [10058, 10058, 10058, 10058], + "invalid-key": [10020], +} as const; + +const run = (failures: ReadonlyArray) => + Effect.gen(function* () { + let attempts = 0; + const started = Date.now(); + const write = Effect.suspend(() => { + attempts += 1; + const code = failures[attempts - 1]; + return code === undefined ? Effect.succeed("stored") : Effect.fail(r2Error(code)); + }); + const result = yield* persistR2Object(write).pipe(Effect.result); + return { + attempts, + outcome: result._tag === "Success" ? result.success : "failed", + elapsedMs: Date.now() - started, + }; + }); + +/** 17 retains at once, each through `writeR2Object` over a put that refuses fast same-key writes. */ +/** Each write in a span of its own, under a tracer that keeps the span to read its attempts. */ +const attempts = Effect.forEach( + [ + ["stored-at-once", []], + ["rate-limited-once", [10058]], + ["persistent", [10058, 10058, 10058, 10058]], + ] as const, + ([name, failures]) => { + let calls = 0; + const spans: Array = []; + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + spans.push(span); + return span; + }, + }); + return writeR2Object( + Effect.suspend(() => { + const code = failures[calls++]; + return code === undefined ? Effect.succeed("stored") : Effect.fail(r2Error(code)); + }), + ).pipe( + Effect.result, + Effect.withSpan("storage.blob.put"), + Effect.withTracer(tracer), + Effect.map(() => [name, spans[0]?.attributes.get("storage.blob.attempts") ?? null] as const), + ); + }, +).pipe(Effect.map((entries) => Object.fromEntries(entries))); + +/** The status Cloud's blob store keeps on a write's failure: R2's 400 for an invalid key only. */ +const statuses = Effect.forEach([10020, 10058, 10001], (code) => + writeR2Object(Effect.fail(r2Error(code))).pipe( + Effect.flip, + Effect.map((error) => [`r2.${code}`, error.status ?? null] as const), + ), +).pipe(Effect.map((entries) => Object.fromEntries(entries))); + +const framework = Effect.gen(function* () { + const objects = new Map(); + const lastWrite = new Map(); + const seen = { attempts: 0, refused: 0, stored: 0 }; + const isFramework = (key: string) => key.startsWith("frameworks/"); + const r2Put = (key: string, body: Uint8Array) => + Effect.gen(function* () { + if (isFramework(key)) seen.attempts += 1; + const last = lastWrite.get(key); + if (last !== undefined && Date.now() - last < 1_000) { + if (isFramework(key)) seen.refused += 1; + return yield* Effect.fail(r2Error(10058)); + } + lastWrite.set(key, Date.now() + 300); + yield* Effect.sleep("300 millis"); + objects.set(key, body); + lastWrite.set(key, Date.now()); + if (isFramework(key)) seen.stored += 1; + return { key }; + }); + const store = { + exists: (key) => Effect.sync(() => objects.has(key)), + put: (key, body) => writeR2Object(r2Put(key, body)), + } as Pick as BlobStorage; + const started = Date.now(); + const builds = yield* Effect.forEach( + Array.from({ length: 17 }, (_, index) => index), + () => + Effect.gen(function* () { + const build = yield* Schema.decodeUnknownEffect(BuildId)(`bld_${crypto.randomUUID()}`); + return yield* retainWorkerBuild( + build, + { + mainModule: "index.js", + modules: { "index.js": "export default {};" }, + database: false, + protocol: 1, + }, + { version: "0.0.1-fixture", modules: { "apps.js": "export const apps = 1;" } }, + undefined, + ).pipe(Effect.provideService(BlobStore, store), Effect.result); + }), + { concurrency: "unbounded" }, + ); + const frameworkWrites = [...objects.keys()].filter(isFramework); + return { + retained: builds.filter((result) => result._tag === "Success").length, + failed: builds + .filter((result) => result._tag === "Failure") + .map((result) => (result._tag === "Failure" ? String(result.failure._tag) : "")), + frameworkObjects: frameworkWrites.length, + ...seen, + elapsedMs: Date.now() - started, + }; +}); + +Effect.runPromise( + Effect.forEach(Object.entries(cases), ([name, failures]) => + run(failures).pipe(Effect.map((result) => [name, result] as const)), + ).pipe( + Effect.map((entries) => Object.fromEntries(entries)), + Effect.flatMap((writes) => + Effect.all({ statuses, framework, attempts }).pipe( + Effect.map((more) => ({ ...writes, ...more })), + ), + ), + ), +).then( + (result) => { + console.log(JSON.stringify(result)); + process.exit(0); + }, + (error) => { + console.error(String(error).slice(0, 2000)); + process.exit(1); + }, +); diff --git a/apps/hosted/cloud/scripts/test-stage-branch.ts b/apps/hosted/cloud/scripts/test-stage-branch.ts new file mode 100644 index 0000000000..78d04a1cb6 --- /dev/null +++ b/apps/hosted/cloud/scripts/test-stage-branch.ts @@ -0,0 +1,53 @@ +/** + * Remove one deployed E2E stage's PlanetScale branch by its exact name, as `test-stage destroy` + * does after Alchemy's destroy (test-stage-branch.ts). Prints the outcome, its message and whether + * the stage's lease would be kept; exits 1 unless the branch is gone. + * + * node scripts/test-stage-branch.ts --slug e2e-abc123 + * + * `TEST_STAGE_BRANCH_POLL` (a duration, 5 seconds by default) sets how often it reads the branch. + * + * `test-stage-branch.spec.ts` runs it against a local stand-in for PlanetScale's API; the API + * override accepts only a loopback origin (test-stage-orphans.ts). + */ +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import { Console, Effect, Option } from "effect"; +import { FetchHttpClient } from "effect/http"; +import { + branchGone, + branchOutcomeMessage, + removeStageBranch, +} from "../src/implementation/test-stage-branch.ts"; + +const option = (name: string) => { + const at = process.argv.indexOf(name); + return at === -1 ? undefined : process.argv[at + 1]; +}; + +NodeRuntime.runMain( + Effect.gen(function* () { + const slug = option("--slug") ?? ""; + const outcome = yield* removeStageBranch(slug); + if (Option.isNone(outcome)) return yield* Console.log(JSON.stringify({ skipped: true })); + const gone = branchGone(outcome.value); + yield* Console.log( + JSON.stringify({ + outcome: outcome.value, + message: branchOutcomeMessage(`test-${slug}`, outcome.value), + leaseKept: !gone, + }), + ); + if (!gone) process.exitCode = 1; + }).pipe( + Effect.provide(FetchHttpClient.layer), + Effect.catch((error) => + Console.error(error.message).pipe( + Effect.andThen( + Effect.sync(() => { + process.exitCode = 1; + }), + ), + ), + ), + ), +); diff --git a/apps/hosted/cloud/scripts/worker-sizes.ts b/apps/hosted/cloud/scripts/worker-sizes.ts index a0c96af37a..8ac6688f29 100644 --- a/apps/hosted/cloud/scripts/worker-sizes.ts +++ b/apps/hosted/cloud/scripts/worker-sizes.ts @@ -9,9 +9,11 @@ * `--preview` the pull request's preview stage too. * * bun run hosted:cloud:worker-sizes [--preview ] [--base ] - * [--description ] [--uploads ] + * [--baseline ] [--description | --merged ] [--uploads ] * - * With `--base`, each Worker is compared with main's uploads, and the packages and files that + * With `--base`, each Worker is compared with the baseline's uploads (`--baseline` names it in the + * report: `main`, or the branch a stack ends on at a commit; see scripts/ci/worker-baseline.ts), + * and the packages and files that * grew are named. A budget is a backstop with room under it, so the comparison is what keeps a * Worker from growing unnoticed: with `--description`, the pull request's description, a Worker * that adds more than `growthLimit` to its upload, or to the modules its entry imports statically @@ -22,8 +24,14 @@ * ``` * * Each line names a Worker, the most bytes it may add, and why that is worth the cold-start time. - * Without `--description`, as on a push to main, growth is only reported. A Worker main does not - * have yet is held to its budget alone. + * A Worker main does not have yet is held to its budget alone. + * + * On a push to main, `--merged` holds the descriptions of the pull requests the push merges + * (scripts/ci/worker-baseline.ts `merged`): growth over the push's `before` is held to the same + * rule, each Worker allowed the sum of what those descriptions allow it. That catches growth no + * pull request was compared with main for: a retargeted upper layer, a skip layer above a branch, + * a direct push. On main an allowance line it cannot read is a notice, not a failure. Without + * either, growth is only reported. * * With `--uploads`, nothing is built: the report and the checks read uploads a build already * wrote, such as the `worker-uploads` artifact of a CI run. @@ -73,6 +81,29 @@ interface Allowance { * The allowances in a description's `worker-growth` blocks, one line for each Worker, and the * lines that do not read as one. */ +/** What the failure message prints where an allowance's reason goes. */ +const placeholder = ""; + +/** The allowances of each merged pull request's description, summed by Worker. */ +const mergedAllowances = ( + merged: ReadonlyArray<{ readonly number: number; readonly body: string }>, +) => { + const allowances = new Map(); + const unread: Array = []; + for (const pull of merged) { + const own = allowancesIn(pull.body); + unread.push(...own.unread.map((line) => `#${pull.number}: ${line}`)); + for (const [worker, allowance] of own.allowances) { + const sum = allowances.get(worker); + allowances.set(worker, { + bytes: (sum?.bytes ?? 0) + allowance.bytes, + reason: `${sum === undefined ? "" : `${sum.reason}; `}#${pull.number}: ${allowance.reason}`, + }); + } + } + return { allowances, unread }; +}; + const allowancesIn = (description: string) => { const allowances = new Map(); const unread: Array = []; @@ -82,7 +113,14 @@ const allowancesIn = (description: string) => { for (const line of block.split("\n").map((line) => line.trim())) { if (line === "") continue; const [, worker, size, reason] = /^([a-z-]+) \+?([0-9][0-9,]*): *(\S.*)$/u.exec(line) ?? []; - if (worker === undefined || size === undefined || reason === undefined) unread.push(line); + // The placeholder the failure message prints is not a reason. + if ( + worker === undefined || + size === undefined || + reason === undefined || + reason === placeholder + ) + unread.push(line); else allowances.set(worker, { bytes: Number(size.replaceAll(",", "")), reason }); } return { allowances, unread }; @@ -187,11 +225,16 @@ const report = ( productionBuild: Build, previewBuild: Build | undefined, base: ReadonlyMap | undefined, - description: string | undefined, + allowed: + | { + readonly allowances: ReadonlyMap; + readonly unread: ReadonlyArray; + } + | undefined, + baseline: string, ) => { const uploads = productionBuild.uploads; const failures = [...(previewBuild?.failures ?? []), ...productionBuild.failures]; - const allowed = description === undefined ? undefined : allowancesIn(description); const grown: Array = [...uploads.values()].flatMap((upload) => { const before = base?.get(upload.worker); const uploaded = before === undefined ? 0 : upload.bytes - before.bytes; @@ -218,7 +261,10 @@ const report = ( ? [] : grown.filter(({ added, allowance }) => allowance === undefined || allowance.bytes < added); const unread = [ - ...(allowed?.unread ?? []).map((line) => `\`${line}\` is not a Worker, a size and a reason`), + ...(allowed?.unread ?? []).map( + (line) => + `\`${line}\` is not a Worker, a size and a reason (write the real reason in place of ${placeholder})`, + ), ...[...(allowed?.allowances.keys() ?? [])] .filter((worker) => !uploads.has(worker)) .map((worker) => `\`${worker}\` is not a Worker this check builds`), @@ -247,12 +293,12 @@ const report = ( "Built as `alchemy deploy` builds them, against the budgets in " + "`apps/hosted/cloud/src/infrastructure/worker-build.ts`. Headroom is for the larger build." + (base === undefined - ? " Main's sizes for this base were not available, so no change is shown and growth is not checked." + ? ` The sizes of ${baseline} were not available, so no change is shown and growth is not checked.` : ""), "", previewBuild === undefined - ? `| Worker | \`${productionBuild.stage}\` | Change from main | Budget | Headroom |` - : `| Worker | \`${productionBuild.stage}\` | \`${previewBuild.stage}\` | Change from main | Budget | Headroom |`, + ? `| Worker | \`${productionBuild.stage}\` | Change from ${baseline} | Budget | Headroom |` + : `| Worker | \`${productionBuild.stage}\` | \`${previewBuild.stage}\` | Change from ${baseline} | Budget | Headroom |`, previewBuild === undefined ? "|---|---:|---:|---:|---:|" : "|---|---:|---:|---:|---:|---:|", ]; for (const worker of measured) { @@ -268,7 +314,7 @@ const report = ( const budgetOnMain = before === undefined || before.budget === worker.budget ? "" - : ` (main: ${bytes(before.budget)})`; + : ` (${baseline}: ${bytes(before.budget)})`; lines.push( `| ${worker.worker} | ${upload === undefined ? "n/a" : bytes(upload.bytes)} | ` + (previewBuild === undefined @@ -296,7 +342,7 @@ const report = ( "isolate starts. A module only a dynamic `import()` loads is uploaded but waits for the " + "request that needs it.", "", - "| Worker | Static imports | Change from main |", + `| Worker | Static imports | Change from ${baseline} |`, "|---|---:|---:|", ...[...uploads.values()].map((upload) => { const before = base?.get(upload.worker)?.staticBytes; @@ -446,14 +492,32 @@ const program = Effect.gen(function* () { message: `No Worker uploads in ${given ?? "the build's output"}`, }); const baseDirectory = flag("--base"); + // What `--base` holds, named in the report and the failures: `main`, or a branch at a commit. + const baseline = flag("--baseline") ?? "main"; const base = baseDirectory === undefined ? undefined : yield* readUploads(baseDirectory); const descriptionFile = flag("--description"); + const mergedFile = flag("--merged"); + const merged = + mergedFile === undefined + ? undefined + : yield* Schema.decodeUnknownEffect( + Schema.fromJsonString( + Schema.Array(Schema.Struct({ number: Schema.Number, body: Schema.String })), + ), + )(yield* fs.readFileString(mergedFile)); const { markdown, over, low, unallowed, unread, failures } = report( built.production, built.preview, base === undefined || base.size === 0 ? undefined : base, - descriptionFile === undefined ? undefined : yield* fs.readFileString(descriptionFile), + merged !== undefined + ? mergedAllowances(merged) + : descriptionFile === undefined + ? undefined + : allowancesIn(yield* fs.readFileString(descriptionFile)), + baseline, ); + // The pull requests a push to main merges, as the messages name them. + const pulls = (merged ?? []).map((pull) => `#${pull.number}`).join(", "); yield* Console.log(markdown); const summary = yield* Config.String("GITHUB_STEP_SUMMARY").pipe(Config.option); if (Option.isSome(summary)) @@ -468,19 +532,47 @@ const program = Effect.gen(function* () { ); for (const { worker, added, what, allowance } of unallowed) yield* Console.log( - `::error title=${worker} Worker grew without an allowance::${signed(added)} bytes${what === "upload" ? "" : ` of ${what}`} over main, about ${compileMs(added)} ms on each cold start; ` + - (allowance === undefined - ? `a pull request may add ${bytes(growthLimit)}.` - : `the description allows ${bytes(allowance.bytes)}.`) + - " Trim it or allow it in the pull request description. See the job summary for what grew and the lines to add.", + merged !== undefined + ? `::error title=${worker} Worker grew on main without an allowance::This push adds ${signed(added)} bytes${what === "upload" ? "" : ` of ${what}`} to ${worker} over ${baseline}, about ${compileMs(added)} ms on each cold start; ` + + (merged.length === 0 + ? "it merges no pull request, so no description allows it." + : allowance === undefined + ? `the pull requests it merges (${pulls}) allow none.` + : `the pull requests it merges (${pulls}) allow ${bytes(allowance.bytes)}.`) + + ` Put the line \`${worker} ${signed(added)}: ${placeholder}\` in a worker-growth block of the merged pull request's description, with your reason, and run this failed job again, which reads the descriptions afresh; or shrink ${worker} in a new pull request. See the job summary for what grew.` + : `::error title=${worker} Worker grew without an allowance::${signed(added)} bytes${what === "upload" ? "" : ` of ${what}`} over ${baseline}, about ${compileMs(added)} ms on each cold start; ` + + (allowance === undefined + ? `a pull request may add ${bytes(growthLimit)}.` + : `the description allows ${bytes(allowance.bytes)}.`) + + ` Trim it, or allow it: put the line \`${worker} ${signed(added)}: ${placeholder}\` in a worker-growth block of the pull request description, with your reason in place of ${placeholder}, then run this job again. See the job summary for what grew.`, + ); + // On main, an allowance a merged description writes in a way this cannot read is not growth. + if (merged !== undefined) + for (const problem of unread) + yield* Console.log( + `::notice title=Worker sizes::A merged pull request's allowance: ${problem}.`, + ); + if (unallowed.length > 0) + yield* Console.log( + [ + "To allow this growth, add this block to the pull request description, with a reason after each colon, then run this job again:", + "", + "```worker-growth", + ...unallowed.map(({ worker, added }) => `${worker} ${signed(added)}: ${placeholder}`), + "```", + ].join("\n"), ); const problems = [ ...(failures.length > 0 ? [`Worker builds failed: ${failures.join("; ")}`] : []), ...(over.length > 0 ? [`Over budget: ${over.map((worker) => worker.worker).join(", ")}`] : []), ...(unallowed.length > 0 - ? [`Grew without an allowance: ${unallowed.map((grew) => grew.worker).join(", ")}`] + ? [ + `Grew without an allowance: ${unallowed.map((grew) => grew.worker).join(", ")}. Lines to add to a worker-growth block of the description, each with its reason: ${unallowed.map(({ worker, added }) => `${worker} ${signed(added)}: `).join("; ")}`, + ] + : []), + ...(unread.length > 0 && merged === undefined + ? [`Allowances this check cannot read: ${unread.join("; ")}`] : []), - ...(unread.length > 0 ? [`Allowances this check cannot read: ${unread.join("; ")}`] : []), ]; if (problems.length > 0) return yield* new WorkerSizesFailed({ message: problems.join(". ") }); }); diff --git a/apps/hosted/cloud/scripts/worker-startup.ts b/apps/hosted/cloud/scripts/worker-startup.ts new file mode 100644 index 0000000000..69892e3a5a --- /dev/null +++ b/apps/hosted/cloud/scripts/worker-startup.ts @@ -0,0 +1,351 @@ +/** + * How long each budgeted Worker takes to start, in workerd, for two builds side by side. + * + * node apps/hosted/cloud/scripts/worker-startup.ts --base --head [--runs 21] + * [--workers mcp-server,api] + * + * Each directory is a built Cloud app (`apps/hosted/cloud` after `bun run hosted:cloud:worker-sizes`, + * which leaves `.generated/worker-uploads/` and Alchemy's `.alchemy/bundles/`), or a directory + * holding copies of those two as `worker-uploads/` and `bundles/`. A Worker is measured when both + * builds have an upload record for it and an entry chunk in its bundle. + * + * For each Worker it loads the chunks the entry imports statically (what Cloudflare compiles and + * evaluates when an isolate starts; a chunk only `import()` reaches is left out) into a fresh + * isolate through the Worker Loader of the workerd the repository pins, with no bindings and no + * network, and times two things from the loader's `get` to the first response: + * + * - `compile`: the chunks supplied, nothing imported. workerd compiles every module it is given. + * - `evaluate`: the same, with a main module that imports the entry, so the entry's static graph + * is also instantiated and evaluated. + * + * Base and head alternate within every run, each run in a new workerd process, and one unrecorded + * load warms each process first. It prints JSON: per Worker, every run's milliseconds, the median + * and the minimum, per side and mode. This is workerd on the machine that runs it, not Cloudflare's + * edge, and its noise is the machine's: pin it to idle cores (`taskset -c 2,3 node ...`) on a + * shared host. + * + * The static graph comes from V8 itself (`vm.SourceTextModule`'s requests: static imports and + * re-exports, never `import()` or text in strings), so the script needs Node's + * `--experimental-vm-modules`, and starts itself again with it when it is missing. + */ +import { spawn, spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { createServer } from "node:net"; +import { cpus, loadavg, tmpdir } from "node:os"; +import { join, posix } from "node:path"; +import vm from "node:vm"; + +if (!("SourceTextModule" in vm)) { + const again = spawnSync( + process.execPath, + ["--experimental-vm-modules", "--no-warnings", ...process.argv.slice(1)], + { stdio: "inherit" }, + ); + process.exit(again.status ?? 1); +} + +const argument = (name: string) => { + const index = process.argv.indexOf(`--${name}`); + return index === -1 ? undefined : process.argv[index + 1]; +}; +const base = argument("base"); +const head = argument("head"); +const runs = Number(argument("runs") ?? 21); +const only = argument("workers")?.split(","); +if (base === undefined || head === undefined || !Number.isInteger(runs) || runs < 1) { + console.error( + "Usage: worker-startup.ts --base --head [--runs 21] [--workers a,b]\n" + + "Each directory is a built apps/hosted/cloud, or holds worker-uploads/ and bundles/.", + ); + process.exit(2); +} + +/** The upload records and the bundles of one build. */ +const layout = (directory: string) => { + const nested = { + uploads: join(directory, ".generated", "worker-uploads"), + bundles: join(directory, ".alchemy", "bundles"), + }; + const flat = { uploads: join(directory, "worker-uploads"), bundles: join(directory, "bundles") }; + const found = [nested, flat].find((each) => existsSync(each.uploads) && existsSync(each.bundles)); + if (found === undefined) + throw new Error( + `${directory} has neither .generated/worker-uploads with .alchemy/bundles nor worker-uploads with bundles`, + ); + return found; +}; + +/** A chunk name that looks content-hashed (`name-<8 characters>.js`); an entry's name has no hash. */ +const hashed = /-[A-Za-z0-9_-]{8}\.js$/; + +/** The relative chunks a chunk imports statically, as V8 reads them. */ +const staticImports = (file: string, code: string) => + new vm.SourceTextModule(code, { identifier: file }).moduleRequests + .map(({ specifier }) => specifier) + .filter((specifier) => specifier.startsWith(".")) + .map((specifier) => posix.normalize(posix.join(posix.dirname(file), specifier))); + +/** The relative chunks a chunk loads with `import()` of a literal. */ +const dynamicImports = (file: string, code: string) => + [...code.matchAll(/import\(\s*["'`](\.{1,2}\/[^"'`]+)["'`]\s*\)/g)].map((match) => + posix.normalize(posix.join(posix.dirname(file), match[1]!)), + ); + +interface Graph { + readonly entry: string; + /** The entry's static chunk set, by file name. */ + readonly modules: Readonly>; +} + +/** The entry chunk of `worker` in a build's bundles, and the chunks it imports statically. */ +const staticGraph = (bundles: string, worker: string): Graph | undefined => { + for (const folder of readdirSync(bundles)) { + const directory = join(bundles, folder); + let files: string[]; + try { + files = readdirSync(directory).filter((file) => file.endsWith(".js")); + } catch { + continue; + } + const named = files.filter((file) => file.startsWith(`${worker}-`) || file === `${worker}.js`); + if (named.length === 0) continue; + // The entry is the one chunk named for the Worker without a content hash. A name such as + // `app-data-app-data.js` looks hashed, so failing that it is the one such chunk no other chunk + // imports (shared chunks may import from the entry, so that test alone is not enough). + const plain = named.filter((file) => !hashed.test(file)); + let entry = plain.length === 1 ? plain[0] : undefined; + if (entry === undefined) { + const referenced = new Set(); + for (const file of files) { + const code = readFileSync(join(directory, file), "utf8"); + for (const target of [...staticImports(file, code), ...dynamicImports(file, code)]) + referenced.add(target); + } + const roots = named.filter((file) => !referenced.has(file)); + entry = roots.length === 1 ? roots[0] : undefined; + } + if (entry === undefined) continue; + const modules: Record = {}; + const pending = [entry]; + for (let file = pending.pop(); file !== undefined; file = pending.pop()) { + if (modules[file] !== undefined) continue; + const code = readFileSync(join(directory, file), "utf8"); + modules[file] = code; + for (const target of staticImports(file, code)) + if (existsSync(join(directory, target))) pending.push(target); + } + return { entry, modules }; + } + return undefined; +}; + +const workersOf = (uploads: string) => + readdirSync(uploads) + .filter((file) => file.endsWith(".json")) + .map((file) => file.slice(0, -".json".length)); + +const sides = { base: layout(base), head: layout(head) } as const; +const names = workersOf(sides.base.uploads) + .filter((worker) => workersOf(sides.head.uploads).includes(worker)) + .filter((worker) => only === undefined || only.includes(worker)) + .sort(); + +const sets: Record = {}; +const measured: string[] = []; +const skipped: Record = {}; +for (const worker of names) { + const graphs = { + base: staticGraph(sides.base.bundles, worker), + head: staticGraph(sides.head.bundles, worker), + }; + if (graphs.base === undefined || graphs.head === undefined) { + skipped[worker] = "no entry chunk in one of the bundles"; + continue; + } + sets[`${worker}:base`] = graphs.base; + sets[`${worker}:head`] = graphs.head; + measured.push(worker); +} + +/** The pinned workerd's binary: the package's CommonJS main exports its path as `default`. */ +const workerdPackage = createRequire(import.meta.url)("workerd") as { readonly default: string }; +const workerdBinary = workerdPackage.default; + +const compatibility = { + date: "2026-09-08", + flags: ["nodejs_compat", "no_brotli_content_encoding"], +}; + +/** The Worker that loads each set through its Worker Loader and times it. */ +const harness = `import sets from "./sets.json"; +let loads = 0; +export default { + async fetch(request, env) { + const url = new URL(request.url); + const set = sets[url.searchParams.get("set")]; + const evaluate = url.searchParams.get("mode") === "evaluate"; + const main = evaluate + ? \`import * as entry from "./\${set.entry}"; export default { fetch() { return new Response(typeof entry); } };\` + : \`export default { fetch() { return new Response("compiled"); } };\`; + const started = Date.now(); + try { + const worker = env.LOADER.get(\`load-\${loads++}\`, () => ({ + compatibilityDate: ${JSON.stringify(compatibility.date)}, + compatibilityFlags: ${JSON.stringify(compatibility.flags)}, + mainModule: "harness-main.js", + modules: { "harness-main.js": main, ...set.modules }, + globalOutbound: null, + })); + const response = await worker.getEntrypoint().fetch("http://startup.invalid/"); + await response.text(); + } catch (error) { + return Response.json({ error: String(error).slice(0, 500) }); + } + return Response.json({ ms: Date.now() - started }); + }, +}; +`; + +const freePort = () => + new Promise((resolve, reject) => { + const server = createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + server.close(() => + typeof address === "object" && address !== null + ? resolve(address.port) + : reject(new Error("no port")), + ); + }); + }); + +const work = mkdtempSync(join(tmpdir(), "worker-startup-")); +writeFileSync(join(work, "sets.json"), JSON.stringify(sets)); +writeFileSync(join(work, "harness.js"), harness); + +/** Start one workerd process serving the harness; resolves its origin once it answers. */ +const startWorkerd = async () => { + const port = await freePort(); + writeFileSync( + join(work, "config.capnp"), + `using Workerd = import "/workerd/workerd.capnp"; +const config :Workerd.Config = ( + services = [(name = "harness", worker = ( + modules = [(name = "harness.js", esModule = embed "harness.js"), (name = "sets.json", json = embed "sets.json")], + compatibilityDate = "${compatibility.date}", + compatibilityFlags = ${JSON.stringify(compatibility.flags)}, + bindings = [(name = "LOADER", workerLoader = ())]))], + sockets = [(name = "http", address = "127.0.0.1:${port}", http = (), service = "harness")]); +`, + ); + const child = spawn(workerdBinary, ["serve", join(work, "config.capnp"), "--experimental"], { + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr = (stderr + chunk.toString()).slice(-2000); + }); + const origin = `http://127.0.0.1:${port}`; + for (let attempt = 0; attempt < 100; attempt++) { + if (child.exitCode !== null) throw new Error(`workerd exited: ${stderr}`); + try { + await fetch(origin); + return { origin, stop: () => child.kill() }; + } catch { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + child.kill(); + throw new Error(`workerd did not answer: ${stderr}`); +}; + +const load = async (origin: string, set: string, mode: "compile" | "evaluate") => { + const response = await fetch(`${origin}/?set=${encodeURIComponent(set)}&mode=${mode}`); + const body = (await response.json()) as { ms?: number; error?: string }; + if (body.ms === undefined) throw new Error(`${set} ${mode}: ${body.error ?? response.status}`); + return body.ms; +}; + +type Side = "base" | "head"; +type Mode = "compile" | "evaluate"; +const samples: Record>> = Object.fromEntries( + measured.map((worker) => [ + worker, + { compile: { base: [], head: [] }, evaluate: { base: [], head: [] } }, + ]), +); + +const loadBefore = loadavg(); +try { + for (let run = 0; run < runs; run++) { + const server = await startWorkerd(); + try { + // The first load of a process also warms the harness itself; it is not recorded. + const first = measured[0]; + if (first !== undefined) await load(server.origin, `${first}:base`, "compile"); + const order: Side[] = run % 2 === 0 ? ["base", "head"] : ["head", "base"]; + for (const mode of ["compile", "evaluate"] as const) + for (const worker of measured) + for (const side of order) + samples[worker]![mode][side].push(await load(server.origin, `${worker}:${side}`, mode)); + } finally { + server.stop(); + } + } +} finally { + rmSync(work, { recursive: true, force: true }); +} + +const median = (values: readonly number[]) => { + const sorted = [...values].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2; +}; +const summary = (values: readonly number[]) => ({ + runs: values, + median: median(values), + min: Math.min(...values), +}); +const bytes = (graph: Graph) => + Object.values(graph.modules).reduce((total, code) => total + Buffer.byteLength(code), 0); + +console.log( + JSON.stringify( + { + workerd: JSON.parse( + readFileSync(new URL(import.meta.resolve("workerd/package.json")), "utf8"), + ).version, + runs, + machine: { cpus: cpus().length, loadBefore, loadAfter: loadavg() }, + workers: Object.fromEntries( + measured.map((worker) => [ + worker, + { + entry: { base: sets[`${worker}:base`]!.entry, head: sets[`${worker}:head`]!.entry }, + staticModules: { + base: Object.keys(sets[`${worker}:base`]!.modules).length, + head: Object.keys(sets[`${worker}:head`]!.modules).length, + }, + staticBytes: { + base: bytes(sets[`${worker}:base`]!), + head: bytes(sets[`${worker}:head`]!), + }, + compile: { + base: summary(samples[worker]!.compile.base), + head: summary(samples[worker]!.compile.head), + }, + evaluate: { + base: summary(samples[worker]!.evaluate.base), + head: summary(samples[worker]!.evaluate.head), + }, + }, + ]), + ), + skipped, + }, + null, + 2, + ), +); diff --git a/apps/hosted/cloud/src/contracts/billing.ts b/apps/hosted/cloud/src/contracts/billing.ts index 2d5fe7bcf2..6e2e115ae7 100644 --- a/apps/hosted/cloud/src/contracts/billing.ts +++ b/apps/hosted/cloud/src/contracts/billing.ts @@ -7,7 +7,6 @@ import { import { ApiError } from "@executor-js/utils/api-error"; import { Context, Effect, Schema } from "effect"; import { HttpApiEndpoint, HttpApiGroup } from "effect/http-api"; -import { RequireBrowserSession } from "@executor-js/hosted-server"; /** The cloud projection of Autumn's catalog. Prices come from the selected provider catalog. */ export const BillingPlan = Schema.Struct({ @@ -90,7 +89,10 @@ export class Billing extends Context.Service< } >()("cloud/Billing") {} -/** Billing is deliberately absent from the shared and self-hosted API. */ +/** + * Billing is deliberately absent from the shared and self-hosted API. Owners and admins reach it + * from the dashboard or an agent; checkout and the portal return a link the person opens to pay. + */ export const billingGroup = HttpApiGroup.make("billing") .add( HttpApiEndpoint.get("overview", "/api/organizations/:organization/billing", { @@ -121,5 +123,4 @@ export const billingGroup = HttpApiGroup.make("billing") error: [BillingUnavailable, OrganizationForbidden], }), ) - .middleware(RequireOrganization) - .middleware(RequireBrowserSession); + .middleware(RequireOrganization); diff --git a/apps/hosted/cloud/src/contracts/edge-contract.json b/apps/hosted/cloud/src/contracts/edge-contract.json index 6031aed948..ce16279c74 100644 --- a/apps/hosted/cloud/src/contracts/edge-contract.json +++ b/apps/hosted/cloud/src/contracts/edge-contract.json @@ -32,6 +32,7 @@ "/authors/*", "/og-image.png", "/pattern-graph-paper.svg", + "/.well-known/security.txt", "/favicon.png", "/.well-known/agent-skills/*", "/api/00e2e1f082a6ef17/*", @@ -198,6 +199,11 @@ "target": "/.well-known/agent-skills/index.json", "forwards": true }, + { + "method": "GET", + "target": "/.well-known/security.txt", + "forwards": true + }, { "method": "POST", "target": "/api/00e2e1f082a6ef17/e/?ip=0", diff --git a/apps/hosted/cloud/src/contracts/site-paths.ts b/apps/hosted/cloud/src/contracts/site-paths.ts index 0f4ccf8749..5bcfa2fd2a 100644 --- a/apps/hosted/cloud/src/contracts/site-paths.ts +++ b/apps/hosted/cloud/src/contracts/site-paths.ts @@ -38,6 +38,12 @@ export const sitePages: ReadonlyArray = [ */ export const v1SitePages: ReadonlyArray = ["/google-oauth", "/privacy", "/terms"]; +/** + * How to report a security problem (RFC 9116), from the site's assets. Listed in `siteFiles` + * for the edge; the browser and MCP hosts serve it too, as the file's `Canonical` lines say. + */ +export const securityTxt: SitePattern = "/.well-known/security.txt"; + /** * Static files the site's pages load, which every host serves from its assets, and the logo the * OAuth client metadata document names: the document is on the edge, so its logo must be too. @@ -50,6 +56,7 @@ export const siteFiles: ReadonlyArray = [ "/apple-touch-icon.png", "/og-image.png", "/pattern-graph-paper.svg", + securityTxt, // v1's dashboard has no `/favicon.png`, so v1 forwards it to v2 like the document. clientMetadataLogoPath, ]; diff --git a/apps/hosted/cloud/src/implementation/billing.ts b/apps/hosted/cloud/src/implementation/billing.ts index ab96dae157..21a8714f4f 100644 --- a/apps/hosted/cloud/src/implementation/billing.ts +++ b/apps/hosted/cloud/src/implementation/billing.ts @@ -1,14 +1,16 @@ import { BillingMeter } from "../contracts/billing-meter.ts"; import { billingMembers } from "../infrastructure/billing-members.ts"; import { + BearerAuthentication, SessionAuthentication, + currentOrganizationSlug, organizationOwner, requireOrganizationAdmin, type OrganizationId, } from "@executor-js/hosted-server"; import { makeExecutionMemo } from "alchemy/Runtime/ExecutionMemo"; import { Cause, Effect, Fiber, Layer, Schema } from "effect"; -import { FetchHttpClient, HttpServerRequest } from "effect/http"; +import { FetchHttpClient } from "effect/http"; import { SqlClient } from "effect/sql"; import { AutumnClient, type AutumnRequestFailed } from "../contracts/autumn.ts"; import { autumnLive, autumnReads } from "./autumn-client.ts"; @@ -340,12 +342,10 @@ export const billingHandlers = HttpApiBuilder.group(ExecutorCloudApi, "billing", const billing = yield* Billing; const meter = yield* BillingMeter; const auth = yield* SessionAuthentication; - const destination = (organization: OrganizationId) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const slug = yield* auth.organizationSlug(new Headers(request.headers), organization); - return new URL(`/org/${encodeURIComponent(slug)}/billing`, auth.origin); - }); + const api = yield* BearerAuthentication; + const destination = currentOrganizationSlug(auth, api).pipe( + Effect.map((slug) => new URL(`/org/${encodeURIComponent(slug)}/billing`, auth.origin)), + ); return handlers .handle("overview", () => Effect.gen(function* () { @@ -361,17 +361,13 @@ export const billingHandlers = HttpApiBuilder.group(ExecutorCloudApi, "billing", .handle("checkout", ({ payload }) => Effect.gen(function* () { const { organization } = yield* requireOrganizationAdmin; - return yield* billing.checkout( - organization, - payload.plan, - yield* destination(organization), - ); + return yield* billing.checkout(organization, payload.plan, yield* destination); }), ) .handle("portal", () => Effect.gen(function* () { const { organization } = yield* requireOrganizationAdmin; - return yield* billing.portal(organization, yield* destination(organization)); + return yield* billing.portal(organization, yield* destination); }), ); }), diff --git a/apps/hosted/cloud/src/implementation/r2-read.ts b/apps/hosted/cloud/src/implementation/r2-read.ts new file mode 100644 index 0000000000..845e2ddd79 --- /dev/null +++ b/apps/hosted/cloud/src/implementation/r2-read.ts @@ -0,0 +1,30 @@ +import { Effect, Schedule } from "effect"; +import { providerFailureCode } from "./provider-failure.ts"; + +/** + * R2's internal error (10001) and temporary unavailability (10043); both ask callers to retry + * (developers.cloudflare.com/r2/api/error-codes). Shared with writes (r2-write.ts). + */ +export const transientR2Failure = (error: unknown) => { + const code = providerFailureCode(error); + return code === "r2.10001" || code === "r2.10043"; +}; + +/** + * Read an R2 object, listing or head again after a transient R2 failure. Reads change nothing, so + * repeating one is safe. Stop after three attempts; other errors, defects and cancellation + * propagate. A body stream that fails after the read opened is not repeated here. + */ +export const readR2Object = (read: Effect.Effect) => + read.pipe( + Effect.tapError((error) => + transientR2Failure(error) + ? Effect.annotateCurrentSpan("storage.blob.transient", true) + : Effect.void, + ), + Effect.retry({ + times: 2, + schedule: Schedule.exponential("100 millis"), + while: transientR2Failure, + }), + ); diff --git a/apps/hosted/cloud/src/implementation/r2-write.ts b/apps/hosted/cloud/src/implementation/r2-write.ts index 559517689a..903ccddad7 100644 --- a/apps/hosted/cloud/src/implementation/r2-write.ts +++ b/apps/hosted/cloud/src/implementation/r2-write.ts @@ -1,20 +1,69 @@ +import { BlobStoreError } from "@executor-js/sdk/core"; import { Effect, Schedule } from "effect"; import { providerFailureCode } from "./provider-failure.ts"; +import { transientR2Failure } from "./r2-read.ts"; -/** Persist an immutable object across explicit R2 rate-limit rejections. - * The caller must retain the same key and complete bytes for every attempt. +/** + * R2 refuses writes to one key faster than one a second (10058, HTTP 429), and asks callers to + * retry its internal error (10001) and temporary unavailability (10043) + * (developers.cloudflare.com/r2/api/error-codes, /r2/platform/limits). + */ +const rateLimited = (error: unknown) => providerFailureCode(error) === "r2.10058"; +const repeatable = (error: unknown) => rateLimited(error) || transientR2Failure(error); + +/** Persist an immutable object across R2 rate-limit and transient rejections. + * The caller must retain the same key and complete bytes for every attempt: a put publishes the + * whole object, so writing identical bytes again leaves the same object, whether or not the + * refused attempt was stored. * Stop after three attempts; other errors, defects and cancellation propagate. */ export const persistR2Object = (write: Effect.Effect) => write.pipe( Effect.tapError((error) => - providerFailureCode(error) === "r2.10058" + rateLimited(error) ? Effect.annotateCurrentSpan("storage.blob.rate_limited", true) - : Effect.void, + : transientR2Failure(error) + ? Effect.annotateCurrentSpan("storage.blob.transient", true) + : Effect.void, ), Effect.retry({ times: 2, schedule: Schedule.exponential("1 second"), - while: (error) => providerFailureCode(error) === "r2.10058", + while: repeatable, }), ); + +/** + * R2's refusals of a write for what it carries, not for storage's state: an invalid key (10020, + * invalid characters or over 1,024 bytes) or an object over 5 GiB (100100), both HTTP 400 + * (developers.cloudflare.com/r2/api/error-codes). Writing the same key or bytes again cannot + * succeed, so the failure keeps R2's status for the caller to tell it from a storage failure. + */ +const refusedWrite = (error: unknown) => { + const code = providerFailureCode(error); + return code === "r2.10020" || code === "r2.100100" ? { status: 400 } : {}; +}; + +/** + * Cloud's blob store write: repeated as `persistR2Object` says, then a `BlobStoreError`. The write's + * span (the SDK's `storage.blob.put`) records how many attempts it took, each a whole upload. + */ +export const writeR2Object = (write: Effect.Effect) => + Effect.suspend(() => { + let attempts = 0; + return Effect.suspend(() => { + attempts += 1; + return write; + }).pipe( + persistR2Object, + Effect.ensuring( + Effect.suspend(() => Effect.annotateCurrentSpan("storage.blob.attempts", attempts)), + ), + ); + }).pipe( + Effect.asVoid, + Effect.tapError((error) => + Effect.annotateCurrentSpan({ "storage.blob.failure.code": providerFailureCode(error) }), + ), + Effect.mapError((error) => new BlobStoreError({ operation: "put", ...refusedWrite(error) })), + ); diff --git a/apps/hosted/cloud/src/implementation/test-stage-branch.ts b/apps/hosted/cloud/src/implementation/test-stage-branch.ts new file mode 100644 index 0000000000..3021c9646b --- /dev/null +++ b/apps/hosted/cloud/src/implementation/test-stage-branch.ts @@ -0,0 +1,183 @@ +/** + * Removes a deployed E2E run's PlanetScale branch when the stage's Alchemy destroy left it. + * + * Alchemy's destroy deletes the branch only when its state records it as the stage's own. If the + * deploy stopped after PlanetScale created the branch but before Alchemy recorded it (a create + * that never became ready, a killed process), destroy logs that the branch "exists in the cloud but + * is not owned by this stack" and leaves it, billing, while the run looks finished. + * + * What this may delete, and nothing else: + * - one branch, named exactly `test-` for the stage being destroyed, where the slug is a + * deployed E2E run's (`e2e-` and 1-13 lowercase letters or digits, `e2e/sdk/deployment.ts`). The + * name is derived from the slug; it is never taken from a branch list or matched by a pattern; + * - on the test database the job names (`TEST_STAGE_DATABASE`), with the job's own PlanetScale + * token, the one the stage's deploy and destroy use; + * - only a development branch: the branch is read first, and a production branch (one created at a + * paid size) is never sent a delete. The stage token cannot delete those anyway; they need someone + * with an organization admin token. + * + * It first waits up to 30 s for the destroy's own delete to land, and sends a delete only if the + * branch is still there; after a delete it waits up to 2 minutes for PlanetScale to answer 404. + * Each call has a 20 s time-out. Every outcome says whether a delete was sent and what PlanetScale + * answered (`branchOutcomeMessage`). + */ +import { Config, Duration, Effect, Option, Redacted, Schedule, Schema } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/http"; +import { planetScaleApi } from "./test-stage-orphans.ts"; + +/** A deployed E2E run's slug (`e2e/sdk/deployment.ts`). */ +const e2eSlug = /^e2e-[a-z0-9]{1,13}$/u; + +/** + * The only branch a stage's teardown may delete by name: `test-` for a deployed E2E run's + * slug. Anything else is refused with the reason, before any request is made. + */ +export const stageBranchFor = ( + slug: string, +): { readonly branch: string } | { readonly refused: string } => + e2eSlug.test(slug) + ? { branch: `test-${slug}` } + : { refused: `"${slug}" is not a deployed E2E run's slug (e2e- and 1-13 letters or digits)` }; + +/** Whether `name` is exactly the branch `stageBranchFor(slug)` derives. */ +export const isStageBranch = (name: string, slug: string) => { + const derived = stageBranchFor(slug); + return "branch" in derived && derived.branch === name; +}; + +export type BranchOutcome = + /** The name was refused; nothing was sent. */ + | { readonly _tag: "Refused"; readonly reason: string } + /** PlanetScale answered 404 before any delete was sent: nothing to remove. */ + | { readonly _tag: "Absent" } + /** A production branch; no delete was sent. */ + | { readonly _tag: "Production" } + /** The branch could not be read (an answer other than 200 or 404, or none in time); no delete was sent. */ + | { readonly _tag: "Unreadable"; readonly status: number | undefined } + /** A delete was sent and no answer came back in time. */ + | { readonly _tag: "DeleteUnanswered" } + /** A delete was sent and PlanetScale refused it. */ + | { readonly _tag: "DeleteRefused"; readonly status: number } + /** A delete was sent and accepted, and PlanetScale then answered 404. */ + | { readonly _tag: "Deleted"; readonly status: number } + /** A delete was sent and accepted, but PlanetScale still answered 200 at the end of the wait. */ + | { readonly _tag: "StillThere"; readonly status: number; readonly waited: string }; + +/** Whether the outcome leaves no branch, so the stage's lease may go. */ +export const branchGone = (outcome: BranchOutcome) => + outcome._tag === "Absent" || outcome._tag === "Deleted"; + +/** One line for the log and the failure: the branch, whether a delete was sent, the answer, who can act. */ +export const branchOutcomeMessage = (branch: string, outcome: BranchOutcome) => { + const remover = + "Its stage lease is kept, so scheduled cleanup tries again; if it stays, someone with permission must delete it in PlanetScale"; + switch (outcome._tag) { + case "Refused": + return `Not deleting PlanetScale branch ${branch}: ${outcome.reason}. No request was sent.`; + case "Absent": + return `PlanetScale branch ${branch} is gone (PlanetScale answers 404). No delete was needed.`; + case "Deleted": + return `PlanetScale branch ${branch} was left by the destroy; a delete was sent (PlanetScale answered ${outcome.status}) and it is gone (404).`; + case "Production": + return `PlanetScale branch ${branch} is a production branch (created at a paid size). No delete was sent: the stage token cannot delete production branches. ${remover}, with an organization admin token.`; + case "Unreadable": + return `PlanetScale branch ${branch} could not be read (${outcome.status === undefined ? `no answer within ${callTimeout}` : `PlanetScale answered ${outcome.status}`}). No delete was sent. ${remover}.`; + case "DeleteUnanswered": + return `PlanetScale branch ${branch} was left by the destroy. A delete was sent; no answer came back within ${callTimeout}. ${remover}.`; + case "DeleteRefused": + return `PlanetScale branch ${branch} was left by the destroy. A delete was sent and PlanetScale refused it (${outcome.status}). ${remover}.`; + case "StillThere": + return `PlanetScale branch ${branch} was left by the destroy. A delete was sent and accepted (${outcome.status}), but PlanetScale still lists it after ${outcome.waited}. ${remover}.`; + } +}; + +const Branch = Schema.Struct({ name: Schema.String, production: Schema.Boolean }); + +/** Each call to PlanetScale gets at most this long; one that runs out counts as no answer. */ +const callTimeout = "20 seconds"; +/** + * How often the branch is read while waiting. Before any delete, it is read for up to 6 polls + * (30 s by default): the deploy tool's own delete of a branch it recorded returns before + * PlanetScale answers 404 (it took 0.4-6.1 s in the deployed runs' logs), and a branch that goes in + * that time needs no second delete. After a delete, up to 24 polls (2 minutes by default). + */ +const pollInterval = Config.Duration("TEST_STAGE_BRANCH_POLL").pipe( + Config.withDefault(Duration.seconds(5)), +); +const settlePolls = 6; +const confirmPolls = 24; + +/** + * Remove the stage's branch if it is still there once the destroy's own delete has had time to + * land. `Option.none()` when the job names no PlanetScale test database. + */ +export const removeStageBranch = (slug: string) => + Effect.gen(function* () { + const derived = stageBranchFor(slug); + if ("refused" in derived) + return Option.some({ _tag: "Refused", reason: derived.refused }); + const database = yield* Config.option(Config.NonEmptyString("TEST_STAGE_DATABASE")); + if (Option.isNone(database)) return Option.none(); + const organization = yield* Config.NonEmptyString("PLANETSCALE_ORGANIZATION"); + const id = yield* Config.Redacted("PLANETSCALE_API_TOKEN_ID"); + const token = yield* Config.Redacted("PLANETSCALE_API_TOKEN"); + const poll = yield* pollInterval; + const api = yield* planetScaleApi; + const http = yield* HttpClient.HttpClient; + const url = `${api}/v1/organizations/${encodeURIComponent(organization)}/databases/${encodeURIComponent(database.value)}/branches/${encodeURIComponent(derived.branch)}`; + /** One call: its answer, or none when it failed or ran out of time. */ + const call = (request: HttpClientRequest.HttpClientRequest) => + http + .execute( + request.pipe( + HttpClientRequest.setHeader( + "authorization", + `${Redacted.value(id)}:${Redacted.value(token)}`, + ), + ), + ) + .pipe(Effect.timeout(callTimeout), Effect.option); + const read = call(HttpClientRequest.get(url)); + /** Read until PlanetScale answers 404, at most `polls` more times; whether it did. */ + const gone = (polls: number) => + read.pipe( + Effect.filterOrFail((answer) => Option.isSome(answer) && answer.value.status === 404), + Effect.retry({ schedule: Schedule.spaced(poll), times: polls }), + Effect.isSuccess, + ); + + const before = yield* read; + if (Option.isNone(before)) + return Option.some({ _tag: "Unreadable", status: undefined }); + if (before.value.status === 404) return Option.some({ _tag: "Absent" }); + // Only a 200 can describe the branch; a body that merely looks like one under another status + // (an error page, a proxy's answer) is not trusted. + if (before.value.status !== 200) + return Option.some({ _tag: "Unreadable", status: before.value.status }); + const found = yield* before.value.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Branch)), + Effect.timeout(callTimeout), + Effect.option, + ); + // The answer must be the branch asked for; anything else is not deleted. + if (Option.isNone(found) || found.value.name !== derived.branch) + return Option.some({ _tag: "Unreadable", status: 200 }); + if (found.value.production) return Option.some({ _tag: "Production" }); + // The destroy's own delete may still be landing: give it time before sending one. + if (yield* gone(settlePolls)) return Option.some({ _tag: "Absent" }); + + const deleted = yield* call(HttpClientRequest.delete(url)); + if (Option.isNone(deleted)) return Option.some({ _tag: "DeleteUnanswered" }); + const status = deleted.value.status; + if (status < 200 || status >= 300) + return Option.some({ _tag: "DeleteRefused", status }); + return Option.some( + (yield* gone(confirmPolls)) + ? { _tag: "Deleted", status } + : { + _tag: "StillThere", + status, + waited: Duration.format(Duration.times(poll, confirmPolls)), + }, + ); + }); diff --git a/apps/hosted/cloud/src/implementation/test-stage-commands.ts b/apps/hosted/cloud/src/implementation/test-stage-commands.ts index 6fc7f8bdb1..c6ae1881cd 100644 --- a/apps/hosted/cloud/src/implementation/test-stage-commands.ts +++ b/apps/hosted/cloud/src/implementation/test-stage-commands.ts @@ -19,6 +19,7 @@ import { ReleaseRefusal, releaseRefusalReport } from "../contracts/release-guard import { withStageAdmin } from "./test-stage-inventory.ts"; import { discoverTestStages } from "./test-stage-discovery.ts"; import { findOrphanBranches } from "./test-stage-orphans.ts"; +import { branchGone, branchOutcomeMessage, removeStageBranch } from "./test-stage-branch.ts"; import { awaitStageRollout } from "./test-stage-rollout.ts"; import { destroyUnreadableStage, @@ -167,6 +168,26 @@ const destroy = (stageSlug: string, options: DestroyOptions) => }, ), ).pipe(Effect.timeout("10 minutes")); + // Alchemy deletes the branch only when its state records it. A create that stopped before + // Alchemy recorded it leaves the branch "not owned by this stack"; remove it by its exact + // name (test-stage-branch.ts). The lease stays until the branch is confirmed gone, so the + // next scheduled run retries what this one could not finish. + if (lease?.database === "planetscale") { + const outcome = yield* removeStageBranch(stageSlug).pipe( + Effect.catch((error) => + Effect.fail( + failure( + `Could not check PlanetScale branch test-${stageSlug} after the destroy (${"message" in error ? error.message : "configuration or request failed"}). No delete was sent. Its stage lease is kept, so scheduled cleanup tries again.`, + ), + ), + ), + ); + if (Option.isSome(outcome)) { + const message = branchOutcomeMessage(`test-${stageSlug}`, outcome.value); + if (!branchGone(outcome.value)) return yield* failure(message); + yield* Console.log(message); + } + } // Failed destruction retains the lease, so the next scheduled run retries it. yield* admin.remove(stageSlug); }), @@ -408,10 +429,10 @@ const cleanup = Command.make("cleanup", {}, () => (stage) => destroy(stage.slug, { automatic: true, expectedOwner: Option.none(), byName: false }).pipe( Effect.as(true), - Effect.catch(() => - Console.error(`Cleanup failed for ${stage.slug}; its lease remains for retry.`).pipe( - Effect.as(false), - ), + Effect.catch((error) => + Console.error( + `Cleanup failed for ${stage.slug}; its lease remains for retry.${Schema.is(TestStageFailed)(error) ? ` ${error.message}` : ""}`, + ).pipe(Effect.as(false)), ), ), { concurrency: 4 }, diff --git a/apps/hosted/cloud/src/implementation/test-stage-orphans.ts b/apps/hosted/cloud/src/implementation/test-stage-orphans.ts index 53471c61c0..83f122966b 100644 --- a/apps/hosted/cloud/src/implementation/test-stage-orphans.ts +++ b/apps/hosted/cloud/src/implementation/test-stage-orphans.ts @@ -63,7 +63,7 @@ const maxPages = 50; * PlanetScale's API, or a stand-in on this machine (`test-stage-orphans.spec.ts`). The override * accepts only a loopback origin, so the job's PlanetScale credentials cannot be sent elsewhere. */ -const planetScaleApi = Effect.gen(function* () { +export const planetScaleApi = Effect.gen(function* () { const configured = yield* Config.option(Config.NonEmptyString("TEST_STAGE_PLANETSCALE_API")); if (Option.isNone(configured)) return "https://api.planetscale.com"; const url = yield* Effect.try({ diff --git a/apps/hosted/cloud/src/infrastructure/blobs.ts b/apps/hosted/cloud/src/infrastructure/blobs.ts index b7d7771bc3..5b96b86068 100644 --- a/apps/hosted/cloud/src/infrastructure/blobs.ts +++ b/apps/hosted/cloud/src/infrastructure/blobs.ts @@ -12,7 +12,8 @@ import * as Cloudflare from "alchemy/Cloudflare"; import { Effect, Option, Stream } from "effect"; import { testStage } from "./stage.ts"; import { providerFailureCode } from "../implementation/provider-failure.ts"; -import { persistR2Object } from "../implementation/r2-write.ts"; +import { readR2Object } from "../implementation/r2-read.ts"; +import { writeR2Object } from "../implementation/r2-write.ts"; import type { WorkspaceObjects } from "../implementation/workspace-cache.ts"; /** Configured stages keep retained builds; a destroyed test stage leaves nothing behind. */ @@ -47,6 +48,7 @@ const listFolder = ( names: page.objects.map((object) => object.key.slice(folder.length + 1)), truncated: page.truncated, })), + readR2Object, Effect.provide(RuntimeContext.phantom), Effect.tapError((error) => Effect.annotateCurrentSpan({ "storage.blob.failure.code": providerFailureCode(error) }), @@ -71,6 +73,7 @@ const streamObject = (bucket: ReadClient, key: BlobKey, range: BlobRange | undef body: object.body.pipe(Stream.mapError(() => new BlobStoreError({ operation: "get" }))), }); }).pipe( + readR2Object, Effect.provide(RuntimeContext.phantom), Effect.tapError((error) => Effect.annotateCurrentSpan({ "storage.blob.failure.code": providerFailureCode(error) }), @@ -99,6 +102,7 @@ export const cloudBlobs = Effect.gen(function* () { if (object === null) return Option.none(); return Option.some(new Uint8Array(yield* object.arrayBuffer())); }).pipe( + readR2Object, Effect.provide(RuntimeContext.phantom), Effect.tapError((error) => Effect.annotateCurrentSpan({ "storage.blob.failure.code": providerFailureCode(error) }), @@ -108,6 +112,7 @@ export const cloudBlobs = Effect.gen(function* () { exists: (key) => bucket.head(key).pipe( Effect.map((object) => object !== null), + readR2Object, Effect.provide(RuntimeContext.phantom), Effect.tapError((error) => Effect.annotateCurrentSpan({ "storage.blob.failure.code": providerFailureCode(error) }), @@ -118,13 +123,8 @@ export const cloudBlobs = Effect.gen(function* () { bucket.put(key, body).pipe( // Builds own UUID keys; onboarding icons use content hashes. Rejected // writes can repeat the same complete bytes without replaying app work. - persistR2Object, - Effect.asVoid, Effect.provide(RuntimeContext.phantom), - Effect.tapError((error) => - Effect.annotateCurrentSpan({ "storage.blob.failure.code": providerFailureCode(error) }), - ), - Effect.mapError(() => new BlobStoreError({ operation: "put" })), + writeR2Object, ), remove: (key) => removeObjects(bucket, key), // One R2 request deletes up to 1,000 objects; `removeAllKeys` keeps callers within it. @@ -144,6 +144,7 @@ export const cloudBuildReader = Effect.gen(function* () { (operation: "get" | "exists") => (effect: Effect.Effect) => effect.pipe( + readR2Object, Effect.provide(RuntimeContext.phantom), Effect.tapError((error) => Effect.annotateCurrentSpan({ "storage.blob.failure.code": providerFailureCode(error) }), @@ -189,7 +190,7 @@ export const cloudWorkspaceObjects = Effect.gen(function* () { const object = yield* bucket.get(key(code)); if (object === null) return Option.none(); return Option.some({ etag: object.etag, body: yield* object.bytes() }); - }).pipe(Effect.provide(RuntimeContext.phantom), observe("get")), + }).pipe(readR2Object, Effect.provide(RuntimeContext.phantom), observe("get")), put: (code, body) => bucket.put(key(code), body).pipe( // Only a failed precondition returns null, and this write has none. diff --git a/apps/hosted/cloud/src/infrastructure/executor.ts b/apps/hosted/cloud/src/infrastructure/executor.ts index 2e6b02f431..4ca61a6fb0 100644 --- a/apps/hosted/cloud/src/infrastructure/executor.ts +++ b/apps/hosted/cloud/src/infrastructure/executor.ts @@ -62,7 +62,11 @@ export const cloudEgress = Effect.gen(function* () { */ const cloudFirstPartyOAuthClients = Effect.gen(function* () { const emulated = Option.isSome(yield* cloudEmulators); - const clients = yield* firstPartyOAuthClients({ testDefinitions: emulated }); + // Cloud's credentials are Executor's own registrations, limited to what services approved. + const clients = yield* firstPartyOAuthClients({ + testDefinitions: emulated, + registrations: "executor", + }); const production = Option.getOrUndefined(yield* stageName) === productionStage; if (production && !clients.some((client) => client.id === "github")) return yield* Effect.die(new Error(missingFirstPartyGitHubClient)); diff --git a/apps/hosted/cloud/src/infrastructure/runtime.ts b/apps/hosted/cloud/src/infrastructure/runtime.ts index 620acaca1c..8e76aeff52 100644 --- a/apps/hosted/cloud/src/infrastructure/runtime.ts +++ b/apps/hosted/cloud/src/infrastructure/runtime.ts @@ -1,6 +1,7 @@ /** Cloud's app runtime: the shared runner in the AppData Worker, with R2 and the Cache API as build store. */ import { traceHeaders } from "@executor-js/telemetry"; import { + BlobStoreError, BuildId, RuntimeBuildFailed, BuildMemoryExceeded, @@ -219,13 +220,15 @@ export const cloudRuntime = Effect.fn(function* ( }).pipe( Effect.tapError((error) => Effect.annotateCurrentSpan({ - "build.stage": Schema.is(BuildMemoryExceeded)(error) - ? "compile" - : Schema.is(RuntimeProtocolUnsupported)(error) - ? "protocol" - : Schema.is(RuntimeAppsDependencyMissing)(error) - ? "dependencies" - : error.stage, + "build.stage": Schema.is(BlobStoreError)(error) + ? "retain" + : Schema.is(BuildMemoryExceeded)(error) + ? "compile" + : Schema.is(RuntimeProtocolUnsupported)(error) + ? "protocol" + : Schema.is(RuntimeAppsDependencyMissing)(error) + ? "dependencies" + : error.stage, // The message quotes the deployer's source and its compiler errors, which only the // deployer receives; the span records which failure it was. "build.error": error._tag, diff --git a/apps/hosted/cloud/src/infrastructure/workflows.ts b/apps/hosted/cloud/src/infrastructure/workflows.ts index ea15ed17b0..669eb365b2 100644 --- a/apps/hosted/cloud/src/infrastructure/workflows.ts +++ b/apps/hosted/cloud/src/infrastructure/workflows.ts @@ -3,18 +3,36 @@ import { cloudSentry, reportCloudFailure } from "../implementation/error-reporti /** One native Cloudflare workflow routes every run to its retained Dynamic Worker app build. */ import { cloudAnalytics, recordBackgroundUsage } from "../implementation/product-analytics.ts"; import * as Cloudflare from "alchemy/Cloudflare"; -import { Clock, Effect, Exit, Schema } from "effect"; +import { Cause, Clock, Config, Effect, Exit, Option, Schema } from "effect"; import { HostedExecutor } from "@executor-js/hosted-server"; import { WorkflowHost, WorkflowRunId, type WorkflowDriver, + EngineFaults, + engineFaultError, + engineFaultFor, + type EngineFault, + nativeEngineFailure, workflowFailureMessage, } from "@executor-js/sdk/core"; +import { cloudEmulators } from "./emulators.ts"; import { cloudProduct } from "./product.ts"; import { appDataSupervisors } from "./app-data.ts"; import { safe } from "./workflow-runtime.ts"; +/** + * A sleep the native engine refused or failed: the platform answered, so the failure names the + * platform's error class and classified code. A failure carrying our own marker passes through. + */ +const nativeSleep = (effect: Effect.Effect) => + effect.pipe( + Effect.catchCause((cause) => { + if (Cause.hasInterrupts(cause)) return Effect.interrupt; + return Effect.fail(nativeEngineFailure(Cause.squash(cause))); + }), + ); + /** A retryable step failure. The engine serializes its encoded message, so a replay decodes the same. */ class RetryableStepFailure extends Schema.TaggedError()( "RetryableStepFailure", @@ -23,9 +41,31 @@ class RetryableStepFailure extends Schema.TaggedError()( const encode = workflowFailureMessage; -const runWorkflow = (input: { - run: string; -}): Effect.Effect => +/** + * E2E runs only: engine call faults (`engine-faults.ts`), read only where emulated providers run, + * which only loopback Cloud and `test-e2e-` stages accept. + */ +const engineFaults = Effect.gen(function* () { + if (Option.isNone(yield* cloudEmulators.pipe(Effect.orDie))) return null; + const configured = yield* Config.option(Config.String("EXECUTOR_TEST_WORKFLOW_ENGINE_FAULTS")); + if (Option.isNone(configured)) return null; + return yield* Schema.decodeUnknownEffect(Schema.fromJsonString(EngineFaults))( + configured.value, + ).pipe(Effect.orDie); +}); + +/** A sleep failed by an E2E fault, before the engine sees it or after it waited. */ +const faulted = (fault: EngineFault | undefined, sleep: Effect.Effect) => + fault === undefined + ? sleep + : fault.when === "before" + ? Effect.die(engineFaultError(fault)) + : Effect.flatMap(sleep, () => Effect.die(engineFaultError(fault))); + +const runWorkflow = ( + input: { run: string }, + faults: ReadonlyArray | null, +): Effect.Effect => Effect.gen(function* () { const { NonRetryableError } = yield* Effect.promise(() => import("cloudflare:workflows")); const run = yield* Schema.decodeUnknownEffect(WorkflowRunId)(input.run); @@ -33,8 +73,11 @@ const runWorkflow = (input: { const services = yield* Effect.context(); const executor = yield* Effect.flatten(HostedExecutor); const driver: WorkflowDriver = { - do: (name, options, work) => - safe( + do: (name, options, work) => { + const fault = engineFaultFor(faults, run, name); + if (fault?.when === "before") + return Effect.fail(nativeEngineFailure(engineFaultError(fault))); + const call = safe( step.do({ name, ...options, @@ -58,9 +101,17 @@ const runWorkflow = (input: { ), ), }), - ), - sleep: (name, duration) => safe(step.sleep(name, duration)), - sleepUntil: (name, timestamp) => safe(step.sleepUntil(name, timestamp)), + ); + return fault === undefined + ? call + : call.pipe( + Effect.flatMap(() => Effect.fail(nativeEngineFailure(engineFaultError(fault)))), + ); + }, + sleep: (name, duration) => + nativeSleep(faulted(engineFaultFor(faults, run, name), step.sleep(name, duration))), + sleepUntil: (name, timestamp) => + nativeSleep(faulted(engineFaultFor(faults, run, name), step.sleepUntil(name, timestamp))), }; const started = yield* Clock.currentTimeMillis; return yield* executor[WorkflowHost].execute(run, driver).pipe( @@ -86,11 +137,12 @@ export class AppWorkflows extends Cloudflare.Workflow()( ); const analytics = yield* cloudAnalytics; const report = yield* cloudSentry; + const faults = yield* engineFaults; return (input: { run: string }) => Effect.scoped( report( analytics.wrap( - runWorkflow(input).pipe( + runWorkflow(input, faults).pipe( Effect.provide(executor), Effect.tapCause(reportCloudFailure), Effect.withSpan("workflow.attempt", { diff --git a/apps/hosted/cloud/src/main.ts b/apps/hosted/cloud/src/main.ts index 69b63c4685..528c03734e 100644 --- a/apps/hosted/cloud/src/main.ts +++ b/apps/hosted/cloud/src/main.ts @@ -101,7 +101,7 @@ import { cloudEntryApi, cloudEntryDocument, resolveCloudEntry } from "./implemen import { browserReturnTo } from "@executor-js/hosted-server/browser/contracts"; import { HttpServerRequest } from "effect/http"; import { homepage, staticDocument } from "./implementation/homepage.ts"; -import { sitePageRoutes } from "./contracts/site-paths.ts"; +import { securityTxt, sitePageRoutes } from "./contracts/site-paths.ts"; import { accountCallback, apiGoneOnEdge, @@ -597,6 +597,8 @@ export default Api.make( const siteRoutes = HttpRouter.addAll( sitePageRoutes.map((page) => HttpRouter.route("GET", page, sitePage)), ); + // `/.well-known/*` runs through the Worker, so the hosts that serve it read it from the assets. + const securityTxtRoute = HttpRouter.add("GET", securityTxt, staticDocument()); // The site's first-party analytics proxy and error tunnel. const siteTelemetryRoutes = Layer.mergeAll( HttpRouter.add("*", "/api/:channel/*", analytics.proxy), @@ -634,6 +636,7 @@ export default Api.make( ), publishedSkillRoutes(authoring), siteRoutes, + securityTxtRoute, siteTelemetryRoutes, browserTelemetry.pipe(HttpRouter.provideRequest(auth.identity)), HttpRouter.add("GET", "/", homepage(auth.cookiePrefix, dashboard(null))), @@ -691,6 +694,7 @@ export default Api.make( authorizationServerRoutes, // ChatGPT verifies the domain of the app's MCP URL, now `mcp.`, at this path. HttpRouter.add("GET", "/.well-known/openai-apps-challenge", openAiAppsChallenge), + securityTxtRoute, ).pipe( Layer.provide(HttpServer.layerServices), toRequestHttpEffect, diff --git a/apps/hosted/self-host/Dockerfile b/apps/hosted/self-host/Dockerfile index 02ab6754b5..b7c790203a 100644 --- a/apps/hosted/self-host/Dockerfile +++ b/apps/hosted/self-host/Dockerfile @@ -17,11 +17,44 @@ COPY apps ./apps COPY playground ./playground COPY e2e/viewer/package.json ./e2e/viewer/package.json RUN bun install --frozen-lockfile --filter @executor-js/hosted-self-host --filter @executor-js/hosted-self-host-web --filter @executor-js/telemetry --filter apps --filter @executor-js/app-live-inbox -RUN bun run --cwd packages/telemetry build:motel RUN bun run --cwd apps/hosted/self-host/web build RUN bun run --cwd packages/apps build RUN bun run --cwd apps/hosted/self-host package:runtime +# The telemetry collector, built from its crate by the same script and toolchain as every release +# target (packages/telemetry/collector/README.md, Release builds). Only the crate, its toolchain +# file and the build script are copied in, so a change elsewhere reuses these layers. +FROM node:24.22.1-bookworm-slim AS collector +ARG TARGETARCH +# gcc compiles the bundled SQLite and links; its binutils give the platform checks objdump. git +# lists the crate's files for the build's identity (an empty repository lists them as untracked). +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl gcc libc6-dev git && rm -rf /var/lib/apt/lists/* +ENV RUSTUP_HOME=/opt/rust/rustup CARGO_HOME=/opt/rust/cargo PATH=/opt/rust/cargo/bin:$PATH +# rustup-init by version and SHA-256: the pin in .github/actions/rustup/action.yml. Change both +# together. The toolchain itself comes from the crate's rust-toolchain.toml. +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) triple=x86_64-unknown-linux-gnu; sha=b1c8cb14e30889e7e2999e6b69e62d9e9b9ffc0f4df8180564d306d9680e6c55 ;; \ + arm64) triple=aarch64-unknown-linux-gnu; sha=9f608c142e0e38a78957d0661ba1dfaf0b6fb37184cfaa4b9a0aa57af6184282 ;; \ + *) echo "No pinned rustup-init for $TARGETARCH." >&2; exit 1 ;; \ + esac; \ + curl --proto '=https' --tlsv1.2 -sSfL -o /tmp/rustup-init "https://static.rust-lang.org/rustup/archive/1.29.3/$triple/rustup-init"; \ + echo "$sha /tmp/rustup-init" | sha256sum -c -; \ + chmod +x /tmp/rustup-init; \ + /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain none; \ + rm /tmp/rustup-init; \ + rustup set auto-self-update disable +WORKDIR /build/packages/telemetry/collector +COPY packages/telemetry/collector/rust-toolchain.toml ./ +RUN rustup toolchain install +COPY packages/telemetry/collector/Cargo.toml packages/telemetry/collector/Cargo.lock ./ +# Its own step, so an outage at crates.io is named as one. +RUN cargo fetch --locked +COPY packages/telemetry/collector ./ +COPY packages/telemetry/scripts/build-collector.ts /build/packages/telemetry/scripts/ +WORKDIR /build +RUN git init --quiet && node packages/telemetry/scripts/build-collector.ts build check + FROM golang:1.27.2-bookworm AS native WORKDIR /build COPY apps/hosted/self-host/native/go.mod apps/hosted/self-host/native/go.sum ./ @@ -49,6 +82,9 @@ COPY --from=build --chown=1000:1000 /build/apps/hosted/self-host/dist/workerd ./ COPY --from=native /executor-host /usr/local/bin/executor-host COPY --from=native /usr/local/go/LICENSE /usr/share/doc/golang/LICENSE COPY --from=native /licenses/ /usr/share/doc/executor-host/ +COPY --from=collector /build/packages/telemetry/dist/collector/executor-collector ./executor-collector +COPY --from=collector /build/packages/telemetry/dist/collector/NOTICE /build/packages/telemetry/dist/collector/cargo-packages.txt /usr/share/doc/executor-collector/ +COPY --from=collector /build/packages/telemetry/dist/collector/licenses/ /usr/share/doc/executor-collector/licenses/ VOLUME /app/data COPY --chmod=755 apps/hosted/self-host/entrypoint.sh /usr/local/bin/executor-entrypoint ENTRYPOINT ["executor-entrypoint"] diff --git a/apps/hosted/self-host/README.md b/apps/hosted/self-host/README.md index 4840242811..c60ab3c77e 100644 --- a/apps/hosted/self-host/README.md +++ b/apps/hosted/self-host/README.md @@ -166,21 +166,20 @@ still fail now and then; only three failures in a row count. Give your own probe The Docker build produces a standalone server bundle. It prepares the trusted Worker host and framework once; authored apps still compile inside workerd. -The image retains PGlite, one native workerd binary, the dashboard, and a small native host for Git, keys, locks, and HTTP. Executor, -authored apps, workflows, and Motel share one workerd executable. Motel runs as -its own workerd process, so its indexing no longer blocks the product's JavaScript -thread; the two still share the container's CPU, memory and disk. Neither Bun nor -Node ships in the runtime image. Build tools -and the workspace dependency tree stay in the build stage. +The image retains PGlite, one native workerd binary, the dashboard, a small native host for Git, keys, locks, and HTTP, and the +telemetry collector. Executor, authored apps and workflows share one workerd executable. The +collector is `executor-collector` (`packages/telemetry/collector`), a separate Rust process the +native host supervises on a private Unix socket; it shares the container's CPU, memory and disk with +the product. Its `collector` build stage builds it from the crate with the toolchain the crate pins, +and its notices are under `/usr/share/doc/executor-collector/`. The image does not serve the +trace viewer: the collector has no address a browser can reach. Neither Bun, Node nor Rust ships in +the runtime image. Build tools and the workspace dependency tree stay in the build stages. The image runs Executor's build of workerd from [UsefulSoftwareCo/workerd](https://github.com/UsefulSoftwareCo/workerd), pinned by release and SHA-256 in `workerd.json`; packaging downloads it and fails on any other digest. It adds idle-isolate and memory-pressure garbage collection and TCMalloc -memory release, set by `memory` in the generated `workerd.capnp`, and for Motel's -process in `motel.capnp`: Motel collects its isolate after 2 seconds idle and -returns the freed memory. With 30 apps on per-minute schedules that held it near -90 MB, where it had risen to about 430 MB. Cloud and local development keep +memory release, set by `memory` in the generated `workerd.capnp`. Cloud and local development keep upstream workerd. The runtime uses Debian 12 distroless with Git's native commands and HTTP backend, @@ -252,15 +251,15 @@ wildcard DNS as described below. The named `pglite-data` volume stores the database, app source, builds, app data, and generated `auth-secret.key` and `encryption.key` files under `/app/data`. Keep one server instance per data volume. -Motel uses a separate store at `/app/motel-data`. The Compose configuration mounts -the named `motel-data` volume there, retaining traces when its container is replaced. -Standalone image runs need their own separate mount to retain telemetry. -Adding a volume does not copy telemetry from an existing container into it. If -those traces are needed, export them or stop the container and copy its complete -Motel directory into the new volume before replacement. Keep the product and -Motel volumes separate; the entrypoint prepares their ownership for the non-root -server user. Do not remove either volume during updates. -Product upgrades do not import old Motel data. +The Rust telemetry collector keeps its store in `EXECUTOR_MOTEL_DATA_DIR`, by default +`/app/motel-data` (the setting retains its historical name). The fork Compose configuration +mounts the separate named `motel-data` volume there, retaining traces on replacement. +Standalone image runs need their own separate mount to retain telemetry. Adding a volume +does not copy an existing container's telemetry: preserve and verify those bytes before replacement. +Keep product and telemetry volumes separate; do not remove either during updates. +The collector neither reads nor removes the old nested `motel/` directory. Preserve it +and the exact old image to read pre-cutover history; new Rust traces are separate, not an +import, and cannot be read by Motel. Never reset or restore product data to repair telemetry. The database records the organization setup created as the instance's root. An upgrade from an older release records it when the database has exactly one @@ -301,8 +300,8 @@ there is no feedback table, so durable retention belongs to the receiver for now ### Trace sampling -Recording every span made Motel the busiest process under load, so the product -samples traces before they reach it. `EXECUTOR_TRACE_SAMPLE_RATE` (0 to 1, +Recording every span made the telemetry collector the busiest process under load, +so the product samples traces before they reach it. `EXECUTOR_TRACE_SAMPLE_RATE` (0 to 1, default `0.1`) is the fraction of ordinary traces kept; the decision comes from the trace ID, so every request of a trace, including the product's requests to itself through the app process, agrees. A request whose `traceparent` has the @@ -314,8 +313,8 @@ until the trace is decided, and drops them when its last local root ends fast and successful. Kept-by-tail roots carry `executor.trace.sampled_by` (`error` or `slow`). Logs and metrics are never sampled. `EXECUTOR_TRACE_SAMPLE_RATE=1` records everything, as before. The native host passes both variables to the -product with the other `EXECUTOR_*` settings; the app and Motel processes export -no spans of their own. +product with the other `EXECUTOR_*` settings; the app and telemetry collector +processes export no spans of their own. The product database is PGlite (PostgreSQL). Bun development stores it at `/app/data/hosted.pglite`; the native @@ -460,7 +459,11 @@ sessions on the old addresses end. App code can reach only public addresses. Requests to the dashboard origin go straight to the server, so the built-in Executor app works when that origin resolves to a private address. Set `EXECUTOR_APPS_ALLOW_PRIVATE_FETCH=true` to -let app code reach other private network addresses. +let app code reach other private network addresses. App builds follow the same +setting for every download off the registry's origin: a dependency declared by +tarball URL, and a tarball or redirect the registry's metadata sends elsewhere. +Downloads from the registry's own origin, including a private +`EXECUTOR_NPM_REGISTRY`, work either way. Host-side MCP import and OAuth requests have a separate exact-origin allowlist. Set `EXECUTOR_URL_ALLOW_HTTP_ORIGINS` to a JSON array of HTTP or HTTPS origins, diff --git a/apps/hosted/self-host/compose.yaml b/apps/hosted/self-host/compose.yaml index 641c048455..971d32b51a 100644 --- a/apps/hosted/self-host/compose.yaml +++ b/apps/hosted/self-host/compose.yaml @@ -53,7 +53,7 @@ services: EXECUTOR_APP_UI_HOST_FORMAT: # Optional organization whose apps use .. EXECUTOR_APP_UI_DEFAULT_ORGANIZATION: - # Unset passthrough entries stay absent; the default is bundled Motel. + # Unset passthrough entries stay absent; the default is the bundled Rust collector. OTEL_EXPORTER_OTLP_ENDPOINT: OTEL_EXPORTER_OTLP_HEADERS: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: diff --git a/apps/hosted/self-host/native/main.go b/apps/hosted/self-host/native/main.go index 2881a34e59..60d492b9d2 100644 --- a/apps/hosted/self-host/native/main.go +++ b/apps/hosted/self-host/native/main.go @@ -498,40 +498,42 @@ func serve(mode string) error { return err } defer os.RemoveAll(temporary) - motel, err := filepath.Abs(setting("EXECUTOR_MOTEL_DATA_DIR", "/app/motel-data")) + // The telemetry collector's data directory. The variable's name is historical: it named the + // directory of Motel, the collector earlier images ran, and operators may have set it. + telemetry, err := filepath.Abs(setting("EXECUTOR_MOTEL_DATA_DIR", "/app/motel-data")) if err != nil { return err } - if motel == directory || strings.HasPrefix(motel, directory+string(os.PathSeparator)) || strings.HasPrefix(directory, motel+string(os.PathSeparator)) { + if telemetry == directory || strings.HasPrefix(telemetry, directory+string(os.PathSeparator)) || strings.HasPrefix(directory, telemetry+string(os.PathSeparator)) { return errors.New("EXECUTOR_MOTEL_DATA_DIR must be separate from the product data directory") } if mode == "export" { - motel = filepath.Join(temporary, "motel") + telemetry = filepath.Join(temporary, "telemetry") } - if err = os.MkdirAll(motel, 0700); err != nil { + if err = os.MkdirAll(telemetry, 0700); err != nil { return err } productRoot, err := filepath.EvalSymlinks(directory) if err != nil { return err } - motelRoot, err := filepath.EvalSymlinks(motel) + telemetryRoot, err := filepath.EvalSymlinks(telemetry) if err != nil { return err } - if motelRoot == productRoot || strings.HasPrefix(motelRoot, productRoot+string(os.PathSeparator)) || strings.HasPrefix(productRoot, motelRoot+string(os.PathSeparator)) { - return errors.New("Motel and product data cannot share a datastore") + if telemetryRoot == productRoot || strings.HasPrefix(telemetryRoot, productRoot+string(os.PathSeparator)) || strings.HasPrefix(productRoot, telemetryRoot+string(os.PathSeparator)) { + return errors.New("Telemetry and product data cannot share a datastore") } productInfo, err := os.Stat(productRoot) if err != nil { return err } - motelInfo, err := os.Stat(motelRoot) + telemetryInfo, err := os.Stat(telemetryRoot) if err != nil { return err } - if os.SameFile(productInfo, motelInfo) { - return errors.New("Motel and product data cannot share a datastore") + if os.SameFile(productInfo, telemetryInfo) { + return errors.New("Telemetry and product data cannot share a datastore") } paths := map[string]string{"product-data": filepath.Join(directory, "product"), "legacy-data": filepath.Join(directory, "hosted.pglite"), "app-data": filepath.Join(directory, "workerd"), "workflow-data": filepath.Join(directory, "workerd", "workflows"), "builds": filepath.Join(directory, "builds")} if mode == "export" { @@ -665,11 +667,11 @@ func serve(mode string) error { if err != nil { return err } - // The collector listens only on a private Unix socket. The product reaches it through its MOTEL + // The collector listens only on a private Unix socket. The product reaches it through its COLLECTOR // binding and operators through `executor-host telemetry`; no address an app can fetch reaches it. // The socket lives in this host's own temporary directory, so containers that share TMPDIR and // mount their product volumes at the same path never share or remove each other's collector. - collector := filepath.Join(temporary, "motel.sock") + collector := filepath.Join(temporary, "collector.sock") // App code runs in its own workerd process unless EXECUTOR_APP_PROCESS=shared, or while exporting. // One app holding a process's only JavaScript thread then stops that process alone: its // watchdog restarts it, and the product, dashboard and MCP keep answering. @@ -694,7 +696,7 @@ func serve(mode string) error { } configPath = filepath.Join(temporary, "product.capnp") } - args := []string{"serve", configPath, "-I/", "--experimental", "--external-addr=native=unix:" + socket, "--external-addr=motel=unix:" + collector, "--socket-addr=http=unix:" + filepath.Join(temporary, "product.sock")} + args := []string{"serve", configPath, "-I/", "--experimental", "--external-addr=native=unix:" + socket, "--external-addr=collector=unix:" + collector, "--socket-addr=http=unix:" + filepath.Join(temporary, "product.sock")} appPaths := map[string]bool{"app-data": true, "workflow-data": true} for name, path := range paths { if separate && appPaths[name] { @@ -769,18 +771,6 @@ func serve(mode string) error { } return exportDatabase(filepath.Join(temporary, "export.sock"), os.Args[2]) } - motelConfig, err := os.ReadFile(filepath.Join(runtime, "motel.capnp")) - if err != nil { - command.Process.Kill() - <-stopped - return err - } - motelConfigPath := filepath.Join(temporary, "motel.capnp") - if err = os.WriteFile(motelConfigPath, bytes.ReplaceAll(motelConfig, []byte("@@RUNTIME@@"), []byte(strings.Trim(strconv.Quote(runtime), "\""))), 0600); err != nil { - command.Process.Kill() - <-stopped - return err - } forget, err := recordTelemetrySocket(directory, collector) if err != nil { command.Process.Kill() @@ -788,18 +778,21 @@ func serve(mode string) error { return err } defer forget() - stopMotel := supervise(func() *exec.Cmd { - // A collector that exited leaves its socket file behind; the next one binds the same path. + stopCollector := supervise(func() *exec.Cmd { + // A collector that was killed leaves its socket file behind, and the collector never + // removes a path it did not create; this directory is the host's own, so the next one + // binds the same path. os.Remove(collector) - command := exec.Command(filepath.Join(runtime, "workerd"), "serve", motelConfigPath, "-I/", "--experimental", "--directory-path=motel-data="+motel, "--socket-addr=motel=unix:"+collector) + command := exec.Command(filepath.Join(runtime, "executor-collector"), append([]string{"serve", "--data=" + telemetry, "--listen=unix:" + collector}, collectorBounds...)...) configureChild(command) + // Its one stdout line says it is listening; its log is JSON lines on stderr. command.Stdout = os.Stderr command.Stderr = os.Stderr command.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + temporary} command.Dir = runtime return command }, time.Second, 30*time.Second, time.After) - defer stopMotel() + defer stopCollector() proxy := productProxy(filepath.Join(temporary, "product.sock"), workerdIdleTimeout, proxies...) if trustedProxy != nil { @@ -869,7 +862,7 @@ func serve(mode string) error { // already stopped by a terminal's process-group signal is not restarted meanwhile. telemetryStopped := make(chan struct{}) go func() { - stopMotel() + stopCollector() close(telemetryStopped) }() command.Process.Signal(syscall.SIGTERM) @@ -883,6 +876,10 @@ func serve(mode string) error { return err } +// collectorBounds are the bounds the image gives its telemetry collector, the ones it gave Motel: +// four exports at once of at most 16 MiB each, a week of telemetry in at most 1 GiB. +var collectorBounds = []string{"--max-pending=4", "--max-export-bytes=16777216", "--max-store-mb=1024", "--retention-hours=168"} + // startupProgressInterval is how often the log says the product is still opening. const startupProgressInterval = 15 * time.Second @@ -1387,7 +1384,7 @@ func queryTelemetry(args []string, output io.Writer) error { }} defer transport.CloseIdleConnections() client := http.Client{Transport: transport, Timeout: 30 * time.Second} - response, err := client.Get("http://motel.internal" + args[0]) + response, err := client.Get("http://collector.internal" + args[0]) if err != nil { return errors.New("the telemetry collector is not running") } diff --git a/apps/hosted/self-host/scripts/package-runtime.ts b/apps/hosted/self-host/scripts/package-runtime.ts index e2b79b8002..2e3d8543c2 100644 --- a/apps/hosted/self-host/scripts/package-runtime.ts +++ b/apps/hosted/self-host/scripts/package-runtime.ts @@ -208,20 +208,6 @@ const packageRuntime = Effect.gen(function* () { yield* fs.writeFile(path.join(output, "workerd"), workerd.binary); yield* fs.chmod(path.join(output, "workerd"), 0o755); yield* fs.copy(web, path.join(output, "web"), { overwrite: true }); - yield* fs.copy(path.join(root, "packages/telemetry/dist/motel"), path.join(output, "motel"), { - overwrite: true, - }); - // The collector shares the container's memory limit. Bound what it holds for exports in - // flight (4 x 16 MiB) and what it stores; beyond either it refuses and counts. - const motelBounds = Object.entries({ - MOTEL_OTEL_MAX_PENDING_INGEST: 4, - MOTEL_OTEL_MAX_INGEST_BYTES: 16 * 1024 * 1024, - MOTEL_OTEL_MAX_SPANS: 1_000_000, - MOTEL_OTEL_MAX_DB_SIZE_MB: 1024, - MOTEL_OTEL_RETENTION_HOURS: 168, - }) - .map(([name, value]) => `(name=${JSON.stringify(name)},text=${JSON.stringify(String(value))})`) - .join(","); // Every workflow run is its own Engine durable object, and pinning engines kept every finished // run resident. workerd unloads an engine about 70 s after its last call once no caller holds // it, even with a step in flight; only a run started by create() has a caller holding it. The @@ -273,7 +259,7 @@ const packageRuntime = Effect.gen(function* () { const productWorker = (bindings: { readonly apps: string }) => `(name="product",worker=( compatibilityDate="2026-09-01",compatibilityFlags=["nodejs_compat","no_brotli_content_encoding","enable_request_signal"], modules=[(name="product.mjs",esModule=embed "@@RUNTIME@@/product.mjs"),(name="executor:pglite.wasm",wasm=embed "@@RUNTIME@@/pglite.wasm"),(name="executor:initdb.wasm",wasm=embed "@@RUNTIME@@/initdb.wasm"),(name="executor:pglite.data",data=embed "@@RUNTIME@@/pglite.data")], - bindings=[(name="PRODUCT",durableObjectNamespace="ExecutorProduct"),(name="NATIVE",service="native"),(name="LEGACY_DATABASE",service="legacy-data"),(name="BLOBS",service="builds"),(name="DASHBOARD",service="dashboard"),(name="PUBLIC_FETCH",service="public"),(name="PRIVATE_FETCH",service="internet"),(name="MOTEL",service="motel"),(name="SELF",service=(name="product",entrypoint="SelfOrigin")),(name="APPS",service=${bindings.apps}),(name="UNSAFE_EVAL",unsafeEval=void)], + bindings=[(name="PRODUCT",durableObjectNamespace="ExecutorProduct"),(name="NATIVE",service="native"),(name="LEGACY_DATABASE",service="legacy-data"),(name="BLOBS",service="builds"),(name="DASHBOARD",service="dashboard"),(name="PUBLIC_FETCH",service="public"),(name="PRIVATE_FETCH",service="internet"),(name="COLLECTOR",service="collector"),(name="SELF",service=(name="product",entrypoint="SelfOrigin")),(name="APPS",service=${bindings.apps}),(name="UNSAFE_EVAL",unsafeEval=void)], durableObjectNamespaces=[(className="ExecutorProduct",uniqueKey="executor-product",enableSql=true,preventEviction=true)],durableObjectStorage=(localDisk="product-data") ))`; const appsWorker = (bindings: { @@ -292,7 +278,7 @@ const packageRuntime = Effect.gen(function* () { const networks = `(name="public",network=(allow=["public"],tlsOptions=(trustBrowserCas=true,trustedCertificates=[@@EXTRA_CA_CERTIFICATES@@]))), (name="internet",network=(allow=["public","private","local"],tlsOptions=(trustBrowserCas=true,trustedCertificates=[@@EXTRA_CA_CERTIFICATES@@])))`; const productStores = `(name="native",external=(address="unix:/tmp/executor-native.sock",http=())), - (name="motel",external=(address="unix:/tmp/executor-motel.sock",http=())), + (name="collector",external=(address="unix:/tmp/executor-collector.sock",http=())), (name="dashboard",disk=(path="@@RUNTIME@@/web")), (name="product-data",disk=(path="/app/data/product",writable=true,allowDotfiles=true)), (name="legacy-data",disk=(path="/app/data/hosted.pglite",allowDotfiles=true)), @@ -342,41 +328,6 @@ const config :Workerd.Config = ( ], sockets=[(name="apps",address="unix:/tmp/executor-apps.sock",http=(),service="apps")] ); -`; - // workerd runs a process's JavaScript on one thread. The collector indexes every span it stores, - // about a thousand per catalog-wide search, so in the product's process that work delayed MCP - // requests by seconds. It runs in its own workerd process and listens only on a private Unix - // socket, which the host passes to it and to the product's MOTEL binding. With no TCP listener, - // an app's fetch cannot reach its ingest or query routes even when private fetch is allowed. - // The collector's process needs its own memory settings. Every export it stores leaves garbage in - // its isolate, and without these settings workerd does not collect an idle isolate or return - // freed memory to the system: with 30 apps' per-minute schedules the collector grew by about - // 100 MB an hour to about 430 MB, of which its heap held about 14 MB of live objects. The product's 10 s idle delay would never fire here, - // because the collector's retention alarm runs every 10 s. After 2 s without JavaScript activity - // the isolate runs a full collection, and TCMalloc returns the memory it frees. The same load then - // held the collector at about 90 MB, for about a fifth more of its CPU time. A collection stops - // the collector's thread while it runs, so an export that arrives during one waits for it. - const motelMemory = [ - "maintenanceIntervalMs=1000", - "idleIsolateGcDelayMs=2000", - "idleIsolateGcMode=full", - `tcmallocBackgroundReleaseBytesPerSecond=${8 * 1024 * 1024}`, - "releaseMemoryAfterGc=true", - ].join(","); - const motelConfig = `using Workerd = import "/workerd/workerd.capnp"; -const config :Workerd.Config = ( - memory=(${motelMemory}), - services=[ - (name="motel",worker=( - compatibilityDate="2026-09-01",compatibilityFlags=["nodejs_compat"],modules=[(name="motel.mjs",esModule=embed "@@RUNTIME@@/motel/motel.mjs")], - bindings=[(name="STORE",durableObjectNamespace="MotelCollector"),(name="ASSETS",service="motel-assets"),${motelBounds}], - durableObjectNamespaces=[(className="MotelCollector",uniqueKey="motel",enableSql=true)],durableObjectStorage=(localDisk="motel-data") - )), - (name="motel-assets",disk=(path="@@RUNTIME@@/motel/web/dist")), - (name="motel-data",disk=(path="/app/motel-data",writable=true,allowDotfiles=true)) - ], - sockets=[(name="motel",address="unix:/tmp/executor-motel.sock",http=(),service="motel")] -); `; // Retain dependency notices with the executable image, including the embedded engines. const packageDirectories = new Set([pg + "/..", alchemyRoot]); @@ -432,7 +383,6 @@ const config :Workerd.Config = ( yield* fs.writeFileString(path.join(output, "workerd.capnp"), config); yield* fs.writeFileString(path.join(output, "product.capnp"), productConfig); yield* fs.writeFileString(path.join(output, "apps.capnp"), appsConfig); - yield* fs.writeFileString(path.join(output, "motel.capnp"), motelConfig); // Count real files once, including native assets, and reject any link back to // the build tree. This budget covers both supported Linux architectures. const runtimeRoot = yield* fs.realPath(output); diff --git a/apps/hosted/self-host/src/implementation/executor-services.ts b/apps/hosted/self-host/src/implementation/executor-services.ts index f8a521505f..1a06f80748 100644 --- a/apps/hosted/self-host/src/implementation/executor-services.ts +++ b/apps/hosted/self-host/src/implementation/executor-services.ts @@ -88,7 +88,10 @@ export const selfHostExecutorServices = ( oauthRedirectUri: Option.getOrUndefined(accountCallback), }); // Only E2E hosts add test client definitions; see `firstPartyOAuthClients`. + // A self-host operator's credentials are its own registrations, which Executor's service + // approvals do not limit. const firstPartyClients = yield* firstPartyOAuthClients({ + registrations: "operator", testDefinitions: (yield* Config.String("EXECUTOR_ENVIRONMENT").pipe(Config.withDefault(""))) === "e2e", }); diff --git a/apps/hosted/self-host/src/implementation/workerd/bindings.ts b/apps/hosted/self-host/src/implementation/workerd/bindings.ts index 35de46c81f..3fec4ad802 100644 --- a/apps/hosted/self-host/src/implementation/workerd/bindings.ts +++ b/apps/hosted/self-host/src/implementation/workerd/bindings.ts @@ -192,7 +192,7 @@ export const bindingHttpClient = ( }); /** The bundled collector's internal origin; it names a service binding, never a network address. */ -export const collectorOrigin = "http://motel.internal"; +export const collectorOrigin = "http://collector.internal"; /** * Telemetry exports to the bundled collector go through its service binding, which the host diff --git a/apps/hosted/self-host/src/implementation/workerd/storage-migration.ts b/apps/hosted/self-host/src/implementation/workerd/storage-migration.ts index 3e7fd4f53d..1eb018e95f 100644 --- a/apps/hosted/self-host/src/implementation/workerd/storage-migration.ts +++ b/apps/hosted/self-host/src/implementation/workerd/storage-migration.ts @@ -39,7 +39,7 @@ const diskUrl = (path: string) => /** * Retry only an unfinished bootstrap. A completed import is immutable, and normal * opens never read the legacy directory again. The host holds the exclusive - * product volume lock while this operation runs. Motel is not involved. + * product volume lock while this operation runs. The telemetry collector is not involved. */ export const prepareProductFilesystem = (storage: DurableObjectStorage, legacy: LegacyDatabase) => Effect.gen(function* () { diff --git a/apps/hosted/self-host/src/worker.ts b/apps/hosted/self-host/src/worker.ts index db6de6c129..d3af1e4085 100644 --- a/apps/hosted/self-host/src/worker.ts +++ b/apps/hosted/self-host/src/worker.ts @@ -70,7 +70,7 @@ interface Environment { readonly PUBLIC_FETCH: HttpBinding; readonly PRIVATE_FETCH: HttpBinding; /** The bundled collector's private Unix socket, which the host connects. */ - readonly MOTEL: HttpBinding; + readonly COLLECTOR: HttpBinding; /** This product's own `SelfOrigin` entrypoint, for requests to the dashboard origin. */ readonly SELF: HttpBinding; readonly APPS: Fetcher; @@ -141,7 +141,7 @@ const prepare = (state: DurableObjectState, env: Environment) => logs: { url: `${collectorOrigin}/v1/logs` }, }).pipe( Layer.provideMerge( - Layer.effect(CurrentTelemetryClient, bindingTelemetryClient(env.MOTEL)), + Layer.effect(CurrentTelemetryClient, bindingTelemetryClient(env.COLLECTOR)), ), ) : telemetryLayer(telemetry), diff --git a/apps/hosted/server/src/contracts/organization.ts b/apps/hosted/server/src/contracts/organization.ts index 15ae828bb3..530e073363 100644 --- a/apps/hosted/server/src/contracts/organization.ts +++ b/apps/hosted/server/src/contracts/organization.ts @@ -17,6 +17,7 @@ import { App, AppId, OwnerId, + ProviderId, HttpUrl, StorageError, publishingDomain, @@ -194,6 +195,16 @@ export class OrganizationIcons extends Context.Service< export const Inventory = Schema.Struct({ /** The accounts the caller linked to each listed app and may use. */ links: Schema.Array(Schema.Struct({ app: AppId, account: AccountId })), + /** The app links of each listed app, shared by everyone using it, oldest first. */ + appLinks: Schema.Array( + Schema.Struct({ + app: AppId, + requirement: Schema.String, + account: AccountId, + provider: ProviderId, + label: Schema.String, + }), + ), apps: Schema.Array(App), /** Each account's checks by the apps listed here; reading them never runs a check. */ accounts: Schema.Array( diff --git a/apps/hosted/server/src/contracts/resource-access.ts b/apps/hosted/server/src/contracts/resource-access.ts index 75abb3e62e..92c62ccfb0 100644 --- a/apps/hosted/server/src/contracts/resource-access.ts +++ b/apps/hosted/server/src/contracts/resource-access.ts @@ -8,6 +8,7 @@ import { Provider, AppId, AccountId, + ProviderId, AccountScope, AccountConnectionId, ProviderNotFound, @@ -107,6 +108,18 @@ export const ResourceDirectory = Schema.Struct({ access: AppAccess, /** The accounts the caller linked to the app and may use; empty when they cannot use it. */ linkedAccounts: Schema.Array(AccountId), + /** + * The app's app links, shared by everyone using it, oldest first, with the label its + * accounts page shows; empty when the caller cannot use it. + */ + appLinks: Schema.Array( + Schema.Struct({ + requirement: Schema.String, + account: AccountId, + provider: ProviderId, + label: Schema.String, + }), + ), }), ), /** The caller's own accounts; every saved account is personal to its owner. */ diff --git a/apps/hosted/server/src/implementation/access.ts b/apps/hosted/server/src/implementation/access.ts index 3ce642f992..9408588917 100644 --- a/apps/hosted/server/src/implementation/access.ts +++ b/apps/hosted/server/src/implementation/access.ts @@ -13,9 +13,10 @@ import { policyDatabase, resourceAuthorityForApp, resourceAuthorityForLinks, - resourceAuthorityForLinksOfApps, + type AppAccountReads, type ResourceAuthority, } from "./resource-policy.ts"; +import type { AppAccess } from "../contracts/resource-access.ts"; import type { App, AppAccounts, @@ -30,15 +31,18 @@ import type { ToolIdentity } from "@executor-js/authorization"; import type { ToolAuthorization } from "./authorization.ts"; import { Context, Effect, type Exit, Result } from "effect"; import { HttpServerRequest } from "effect/http"; -import type { AppAccess } from "../contracts/resource-access.ts"; -import { CurrentUserId, Forbidden } from "../contracts/auth.ts"; +import { + BearerAuthentication, + CurrentUserId, + Forbidden, + SessionAuthentication, +} from "../contracts/auth.ts"; import { refuse } from "./auth-telemetry.ts"; import { appAccountReads, appLinkedAccounts, callerAccounts, versionAccounts, - type CallerLinks, } from "./app-accounts.ts"; import { CurrentOrganization, @@ -53,6 +57,19 @@ export const browserOnly = Effect.gen(function* () { if (request.headers.authorization !== undefined) return yield* refuse("bearer_not_allowed", new Forbidden()); }); +/** The current organization's slug, from the presented grant or else the browser session. */ +export const currentOrganizationSlug = ( + auth: SessionAuthentication["Service"], + api: BearerAuthentication["Service"], +) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const headers = new Headers(request.headers); + const { organization } = yield* CurrentOrganization; + return headers.has("authorization") + ? (yield* api.authenticate(headers, organization)).organizationSlug + : yield* auth.organizationSlug(headers, organization); + }); /** Membership was checked by middleware; administrative actions require the current role. */ export const requireOrganizationAdmin = Effect.gen(function* () { const organization = yield* CurrentOrganization; @@ -270,6 +287,23 @@ export const selectedVersion = ( linked, ); }); +/** + * The app's active version for an operation that names `version`, or none, with the accounts it + * runs with: a version other than the active one is not found, and an app without one is not + * deployed. + */ +export const pinnedActiveVersion = ( + selected: { readonly app: App; readonly accounts: AppAccounts }, + version: VersionId | undefined, +) => + Effect.gen(function* () { + const { app, accounts } = selected; + if (version !== undefined && version !== app.activeVersion) + return yield* new VersionNotFound({ app: app.id, version }); + if (app.activeVersion === null) return yield* new AppNotDeployed({ app: app.id }); + // Pin the authorized version so a concurrent promotion cannot silently change the call. + return { version: app.activeVersion, accounts }; + }); /** New tool discovery and calls use the active build; retained execution is reserved for saved invocations. */ export const selectedActiveVersion = ( executor: Executor, @@ -277,72 +311,24 @@ export const selectedActiveVersion = ( input: { readonly app: AppId; readonly version?: VersionId | undefined }, checked?: CheckedApp, ) => - Effect.gen(function* () { - const { app, accounts } = yield* selectedApp(executor, owner, input.app, undefined, checked); - if (input.version !== undefined && input.version !== app.activeVersion) - return yield* new VersionNotFound({ app: app.id, version: input.version }); - if (app.activeVersion === null) return yield* new AppNotDeployed({ app: app.id }); - // Pin the authorized version so a concurrent promotion cannot silently change the call. - return { version: app.activeVersion, accounts }; - }); - + Effect.flatMap(selectedApp(executor, owner, input.app, undefined, checked), (selected) => + pinnedActiveVersion(selected, input.version), + ); /** - * {@link selectedActiveVersion} for apps this request checked, such as the apps one catalog search - * reads: the links of every app with accounts are read with the caller's access in one statement, - * and each app decides as {@link selectedActiveVersion} would with them. An app it cannot decide - * that way, or that one of the checks refuses, is `None`: its own read reports why. + * {@link selectedApp} for an app decided from reads made for many apps at once, such as every app + * a search reads: the app as the request's app list read it, the caller's access to it, and the + * caller's and the app's links (see `resourceAuthorityForAppsWithLinks`). An app the caller may + * not use, or without a policy, is refused as `selectedApp` refuses it; otherwise its accounts are + * resolved from those links by the same rules. */ -export const selectedActiveVersions = ( - owner: OwnerId, - inputs: ReadonlyArray<{ readonly app: AppId; readonly version?: VersionId | undefined }>, - checked: ReadonlyArray, +export const selectedAppFrom = ( + app: App, + access: typeof AppAccess.Type | undefined, + links: AppAccountReads, ) => Effect.gen(function* () { - const organization = (yield* CurrentOrganization).organization; - const user = yield* CurrentUserId; - const known = new Map( - checked.filter((entry) => entry.app.owner === owner).map((entry) => [entry.app.id, entry]), - ); - const accountless = (app: App) => Object.keys(app.requirements.accounts).length === 0; - const withAccounts = [ - ...new Set( - inputs.flatMap((input) => { - const app = known.get(input.app)?.app; - return app === undefined || accountless(app) ? [] : [app.id]; - }), - ), - ]; - const none: ReadonlyMap< - AppId, - { readonly access: typeof AppAccess.Type; readonly accounts: CallerLinks } - > = new Map(); - const links = - withAccounts.length === 0 - ? none - : yield* resourceAuthorityForLinksOfApps(organization, user, withAccounts).pipe( - Effect.orElseSucceed(() => none), - ); - const sql = yield* policyDatabase; - return yield* Effect.forEach(inputs, (input) => - Effect.gen(function* () { - const found = known.get(input.app); - if (found === undefined || user === undefined) return yield* Effect.fail(undefined); - const { access, accounts } = accountless(found.app) - ? // As for one app: the caller's access was read with the app, and a version that - // declares no account reads no links. - { - access: found.access, - accounts: appAccountReads(sql, { subject: user, app: found.app.id }), - } - : (links.get(found.app.id) ?? (yield* Effect.fail(undefined))); - if (!access.canUse) return yield* Effect.fail(undefined); - const resolved = yield* callerAccounts(found.app, accounts); - if (input.version !== undefined && input.version !== found.app.activeVersion) - return yield* Effect.fail(undefined); - if (found.app.activeVersion === null) return yield* Effect.fail(undefined); - return { version: found.app.activeVersion, accounts: resolved }; - }).pipe(Effect.option), - ); + if (access === undefined || !access.canUse) return yield* new OrganizationForbidden(); + return { app, accounts: yield* callerAccounts(app, Effect.succeed(links)) }; }); /** diff --git a/apps/hosted/server/src/implementation/accounts.ts b/apps/hosted/server/src/implementation/accounts.ts index 4ace47f29d..f00f9dac7b 100644 --- a/apps/hosted/server/src/implementation/accounts.ts +++ b/apps/hosted/server/src/implementation/accounts.ts @@ -41,13 +41,9 @@ import { SessionAuthentication, CurrentPrincipal, } from "../contracts/auth.ts"; -import { - CurrentOrganization, - OrganizationSlug, - organizationOwner, -} from "../contracts/organization.ts"; +import { OrganizationSlug, organizationOwner } from "../contracts/organization.ts"; import { HostedExecutor } from "../contracts/executor.ts"; -import { currentOwner, ownedConnection } from "./access.ts"; +import { currentOrganizationSlug, currentOwner, ownedConnection } from "./access.ts"; import { appLinkApps, appLinkedAccounts, callerLinkedAccounts } from "./app-accounts.ts"; import { AccountLinkedToApp } from "../contracts/accounts.ts"; import { visibleAccountRefusal } from "./webhooks.ts"; @@ -252,12 +248,7 @@ export const connectionUrl = ( connection: AccountConnectionId, ) => Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const headers = new Headers(request.headers); - const organization = yield* CurrentOrganization; - const slug = headers.has("authorization") - ? (yield* api.authenticate(headers, organization.organization)).organizationSlug - : yield* auth.organizationSlug(headers, organization.organization); + const slug = yield* currentOrganizationSlug(auth, api); return `${auth.origin}/org/${encodeURIComponent(slug)}/connections/${encodeURIComponent(connection)}`; }); /** Connection metadata never grants access to another organization's request or app. */ diff --git a/apps/hosted/server/src/implementation/first-party-oauth.ts b/apps/hosted/server/src/implementation/first-party-oauth.ts index def2a6cd39..8505759a53 100644 --- a/apps/hosted/server/src/implementation/first-party-oauth.ts +++ b/apps/hosted/server/src/implementation/first-party-oauth.ts @@ -15,18 +15,23 @@ export const FirstPartyOAuthClientDefinition = FirstPartyOAuthClient.mapFields( export type FirstPartyOAuthClientDefinition = typeof FirstPartyOAuthClientDefinition.Encoded; /** - * Every scope Executor for Google may ask for and a grant may hold. This is v1's + * Every scope Executor for Google may ask for. Google lets the production client ask a user for a + * sensitive or restricted scope only once its OAuth verification approves that scope for the + * client, and shows "This app is blocked" otherwise, so this list must hold only the scopes + * approved for it and non-sensitive ones, which need no approval. Add a scope only after Google approves + * it for the production client; until then an app asking for it is refused before Google's consent + * screen, and its user signs in with their own client. + * + * Sources, kept by the operator: Google's verification approvals for the production client of + * 2026-10-05 (the Workspace scopes, `https://mail.google.com/` and `gmail.settings.basic` among + * them) and 2026-10-25 (YouTube, Google Ads, BigQuery and Apps Script). The list began as v1's * `GOOGLE_ALLOWED_SCOPES` (`apps/cloud/src/engine/first-party-oauth-clients.ts` in - * UsefulSoftwareCo/executor): the scopes v1's catalog requested for its 13 first-party Google - * presets (Calendar, Meet, Gmail, Sheets, Drive, Docs, Slides, Forms, Tasks, People, Photos - * Library, Photos Picker and Search Console), each with `openid email profile`, and - * `gmail.modify`, which Gmail connections made before the full-Gmail review keep. It is the scope - * set approved for the production app, extended on 2026-10-25 for YouTube, Google Ads, - * BigQuery and Apps Script. Apps Script metrics is non-sensitive. Google reports `email` and - * `profile` as granted under their `userinfo` names, and a grant holds only allowed scopes, so those two are allowed - * as well. + * UsefulSoftwareCo/executor): the scopes of v1's 13 first-party Google presets (Calendar, Meet, + * Gmail, Sheets, Drive, Docs, Slides, Forms, Tasks, People, Photos Library, Photos Picker and + * Search Console). `openid`, `email` and `profile`, their `userinfo` names, under which Google + * reports `email` and `profile` granted, and Apps Script metrics are non-sensitive. */ -const googleAllowedScopes = [ +const googleSignInScopes = [ "openid", "email", "profile", @@ -38,7 +43,6 @@ const googleAllowedScopes = [ "https://www.googleapis.com/auth/meetings.space.settings", "https://mail.google.com/", "https://www.googleapis.com/auth/gmail.settings.basic", - "https://www.googleapis.com/auth/gmail.modify", "https://www.googleapis.com/auth/spreadsheets", "https://www.googleapis.com/auth/drive.file", "https://www.googleapis.com/auth/drive", @@ -71,6 +75,25 @@ const googleAllowedScopes = [ "https://www.googleapis.com/auth/script.metrics", ]; +/** + * Scopes Executor for Google grants hold that Google has not approved for the production client, + * so no sign-in on Cloud asks for them. The grants keep renewing; a sign-in or reconnect asking for + * one is refused, leading with the user's own client and naming the approved scope that covers it, + * which grants more. A self-host operator's own Google client asks for them as before + * (`FirstPartyRegistrations`). Move a scope from here to `googleSignInScopes` once Google approves + * it; either list is one line per scope. + * + * - `gmail.modify`: v1 asked for it, and Gmail connections made then hold it. Google has not + * approved it for the production client; the full Gmail scope, `https://mail.google.com/`, is + * approved and covers it. + */ +const googleHeldScopes = [ + { + scope: "https://www.googleapis.com/auth/gmail.modify", + instead: ["https://mail.google.com/"], + }, +]; + /** * The user scopes Executor for Slack may ask for and a grant may hold: v1's `slackMcpUserScopes` * (`packages/react/src/lib/slack-mcp-oauth.ts` in UsefulSoftwareCo/executor), the user scopes @@ -121,7 +144,8 @@ const slackMcpUserScopes = [ * `uploads.github.com`, where GitHub takes release asset uploads; and `api.githubcopilot.com`, * GitHub's MCP server. GitHub accepts GitHub App user tokens on all three, and v1 sent this * client's tokens to every GitHub REST operation, uploads included, and to GitHub's MCP server. - * - `google` is Executor for Google, v1's Google client. Like every Google + * - `google` is Executor for Google, v1's Google client. It asks only for the scopes Google approved + * for it, and its grants may also hold the held scopes, which it no longer asks for. Like every Google * sign-in, it sends `access_type=offline` and `prompt=consent`, so each returns a refresh token. It has no revocation URL: Google * revokes every grant a user gave the client at once, so removing one account would end the * user's other Google accounts in every organization. Its tokens go to Google's API hosts. @@ -163,7 +187,8 @@ export const firstPartyOAuthClientDefinitions: readonly FirstPartyOAuthClientDef }, tokenEndpointAuthMethod: "client_secret_post", defaultScopes: ["openid", "email", "profile"], - allowedScopes: googleAllowedScopes, + allowedScopes: googleSignInScopes, + heldScopes: googleHeldScopes, placement: { hosts: ["*.googleapis.com"] }, }, { @@ -232,16 +257,48 @@ const testDefinitions = (allowed: boolean) => ); }); +/** + * Whose OAuth registrations the reviewed clients' credentials are. Cloud's are Executor's own, + * which a service may approve for only some scopes, as Google approved Executor for Google: their + * held scopes are never asked for. A self-host operator sets the credentials of its own + * registrations, which no approval of Executor's limits, so each reviewed client asks for its held + * scopes as allowed ones, as it did before they were held. + */ +export type FirstPartyRegistrations = "executor" | "operator"; + +/** A reviewed definition as `registrations` may use it. */ +const forRegistrations = ( + definition: FirstPartyOAuthClientDefinition, + registrations: FirstPartyRegistrations, +): FirstPartyOAuthClientDefinition => { + const { heldScopes, ...menu } = definition; + if (registrations === "executor" || heldScopes === undefined) return definition; + return { + ...menu, + allowedScopes: [ + ...(menu.allowedScopes ?? menu.defaultScopes), + ...heldScopes.map((held) => held.scope), + ], + }; +}; + /** * The operator's clients whose credentials are set, checked at startup. A client ID without its * secret, or a secret without its ID, stops the host and names the missing setting; no value is - * ever repeated. `testDefinitions` lets an E2E host add `EXECUTOR_TEST_OAUTH_CLIENT_DEFINITIONS`; - * every other host passes `false` and refuses to start with it set. + * ever repeated. `registrations` says whose registrations the reviewed clients are (see + * `FirstPartyRegistrations`); test definitions are used as declared. `testDefinitions` lets an E2E + * host add `EXECUTOR_TEST_OAUTH_CLIENT_DEFINITIONS`; every other host passes `false` and refuses + * to start with it set. */ -export const firstPartyOAuthClients = (options: { readonly testDefinitions: boolean }) => +export const firstPartyOAuthClients = (options: { + readonly testDefinitions: boolean; + readonly registrations: FirstPartyRegistrations; +}) => Effect.gen(function* () { const reviewed = yield* Schema.decodeUnknownEffect(Definitions)( - firstPartyOAuthClientDefinitions, + firstPartyOAuthClientDefinitions.map((definition) => + forRegistrations(definition, options.registrations), + ), ).pipe(Effect.orDie); const definitions = [...reviewed, ...(yield* testDefinitions(options.testDefinitions))]; const seen = new Set(); diff --git a/apps/hosted/server/src/implementation/mcp.ts b/apps/hosted/server/src/implementation/mcp.ts index 7af8d72ee6..2b47488f16 100644 --- a/apps/hosted/server/src/implementation/mcp.ts +++ b/apps/hosted/server/src/implementation/mcp.ts @@ -9,13 +9,19 @@ import { authorizeApp, authorizeSavedTool, authorizeTool } from "./authorization import { callerLinkedAccounts, linkedAccounts, requireConnectionAccounts } from "./app-accounts.ts"; import { GrantForbidden, untargetedLinkRefusal } from "@executor-js/mcp-auth"; import { + permitsApp, permittedAppIds, permittedToolAppIds, requiresToolMetadata, } from "@executor-js/authorization"; import { GroupDatabase } from "../contracts/groups.ts"; import { CurrentUserId } from "../contracts/auth.ts"; -import { policyDatabase, visibleAppAccesses, requireAppAccess } from "./resource-policy.ts"; +import { + policyDatabase, + resourceAuthorityForAppsWithLinks, + visibleAppAccesses, + requireAppAccess, +} from "./resource-policy.ts"; /** Hosted catalog and execution policy for the shared MCP engine; no HTTP transport or credentials. */ import { CurrentMcpExecution, @@ -32,15 +38,18 @@ import { InvocationSelection, CurrentCaller, linkedRequirements, + PrefetchHost, + SearchablePrefetch, type AppId, type ToolInvocationOptions, } from "@executor-js/sdk/core"; -import { Context, Effect, Option, Schema } from "effect"; +import { Array as Arr, Context, Effect, Exit, Option, Schema } from "effect"; import { checkedToolCall, currentOwner, selectedActiveVersion, checkInvocationAccounts, + selectedAppFrom, SelectionReadAhead, } from "./access.ts"; import { ConcurrentStatements, HostedBackground, HostedExecutor } from "../contracts/executor.ts"; @@ -148,6 +157,93 @@ export const hostedMcpBackend = Effect.gen(function* () { Effect.provideContext(context), ), ); + /** + * The caller's access to every app the request lists for discovery, their links to them and + * the apps' links, read in one statement once per request, when discovery first needs it, so + * discovery does not check each app in a statement of its own. Each app is still decided on its + * own from it, as its own reads would decide it. Undefined when the read did not answer: each + * app then reads for itself. + */ + const discoveryAccess = yield* Effect.cached( + Effect.gen(function* () { + const apps = (yield* discoveryApps).filter((app) => permitsApp(policy, app.id)); + const ids = apps.map((app) => app.id); + if (!Arr.isReadonlyArrayNonEmpty(ids)) return undefined; + const read = yield* resourceAuthorityForAppsWithLinks(organization.organization, user, ids); + return { ...read, apps }; + }).pipe( + Effect.catch(() => Effect.succeed(undefined)), + Effect.provideContext(context), + ), + ); + /** + * What search needs for the apps of one search batch: each app's selection decided from + * {@link discoveryAccess}, then the SDK's reads for the target of every app that selected + * (`PrefetchHost.searchable`), a fixed number of statements however many apps the batch has. + * Only the batch's apps are read: a search must not wait on another app's rows, which a slow or + * locked listing can hold. Undefined when the access read did not answer. + */ + const searchReadsFor = (batch: ReadonlySet) => + Effect.gen(function* () { + const read = yield* discoveryAccess; + if (read === undefined) return undefined; + const selections = new Map( + yield* Effect.forEach( + read.apps.filter((app) => batch.has(app.id)), + (app) => + Effect.map( + Effect.exit( + selectedAppFrom( + app, + read.access.find((policy) => policy.app === app.id), + read.links, + ), + ), + (selected) => [app.id, selected] as const, + ), + ), + ); + const targets = [...selections.values()].flatMap((selected) => + Exit.isSuccess(selected) && selected.value.app.activeVersion !== null + ? [ + { + app: selected.value.app.id, + version: selected.value.app.activeVersion, + accounts: selected.value.accounts, + }, + ] + : [], + ); + const prefetched = yield* (yield* sdk)[PrefetchHost].searchable({ + owner: organization.owner, + targets, + }); + return { selections, prefetched }; + }).pipe( + Effect.catch(() => Effect.succeed(undefined)), + Effect.provideContext(context), + ); + type SearchReads = Effect.Success>; + /** + * Each app's search batch reads, made once per batch, when the first of its apps needs them. An + * app belongs to the first batch that names it; a later batch naming it again reads the rest. + */ + const searchBatches = new Map>(); + const searchReads = (app: AppId, batch: ReadonlyArray | undefined) => + Effect.gen(function* () { + if (batch === undefined || !batch.includes(app)) return undefined; + const claimed = searchBatches.get(app); + if (claimed !== undefined) return yield* claimed; + const members = new Set(); + const reads = yield* Effect.cached(Effect.suspend(() => searchReadsFor(members))); + for (const member of batch) + if (!searchBatches.has(member)) { + members.add(member); + searchBatches.set(member, reads); + } + // Another read of the same batch may have claimed this app while the reads were made. + return yield* searchBatches.get(app) ?? reads; + }); /** * An operation resolves the caller's links narrowed to the grant's targets, so a plain user * requirement whose links the grant does not target fails as `AccountRequired`. When the caller @@ -343,22 +439,35 @@ export const hostedMcpBackend = Effect.gen(function* () { "read", ), searchableTools: (input, options) => - observe( - "searchableTools", - grantRefusal( - Effect.flatMap(listedApp(input.app), (checked) => - searchableTools(input, options, checked), - ), - ), - "read", - ), - // A search reads every app's rows: those kept for apps this request checked are read at once. + Effect.gen(function* () { + const reads = yield* searchReads(input.app, options?.searchBatch); + const selected = reads?.selections.get(input.app); + return yield* reads === undefined || selected === undefined + ? searchableTools(input, options) + : searchableTools(input, options, selected).pipe( + Effect.provideService(SearchablePrefetch, reads.prefetched), + ); + }).pipe((work) => observe("searchableTools", grantRefusal(work), "read")), searchableKept: (inputs) => observe( "searchableKept", - Effect.orElseSucceed(discoveryChecked, () => []).pipe( - Effect.flatMap((checked) => searchableKept(inputs, checked)), - ), + Effect.gen(function* () { + const first = inputs[0]; + if (first === undefined) return []; + const reads = yield* searchReads( + first.app, + inputs.map((input) => input.app), + ); + if (reads === undefined) return inputs.map(() => undefined); + const selections = new Map( + [...reads.selections].flatMap(([app, selected]) => + Exit.isSuccess(selected) ? [[app, selected.value] as const] : [], + ), + ); + return yield* searchableKept(inputs, selections).pipe( + Effect.provideService(SearchablePrefetch, reads.prefetched), + ); + }), "read", ), searchTools: (input) => observe("searchTools", searchTools(input), "read"), diff --git a/apps/hosted/server/src/implementation/organization.ts b/apps/hosted/server/src/implementation/organization.ts index aa5a8c5309..ee1433d1f0 100644 --- a/apps/hosted/server/src/implementation/organization.ts +++ b/apps/hosted/server/src/implementation/organization.ts @@ -28,11 +28,16 @@ import { import { McpAuthentication } from "../contracts/mcp.ts"; import { apiCaller, personCaller } from "./caller.ts"; import { Context } from "effect"; -import { currentResourceAuthority, visibleAccountsAs, visibleAppsAs } from "./resource-policy.ts"; +import { + currentResourceAuthority, + policyDatabase, + visibleAccountsAs, + visibleAppsAs, +} from "./resource-policy.ts"; import { readOrganizationIconUpload } from "./organization-icons.ts"; import { OrganizationTombstones } from "../contracts/organization-removal.ts"; import { requireOrganizationAdmin } from "./access.ts"; -import { callerLinkedAccounts } from "./app-accounts.ts"; +import { appLinkedAccounts, callerLinkedAccounts } from "./app-accounts.ts"; import { CurrentPrincipal, CurrentUserId } from "../contracts/auth.ts"; import { recordCaller, @@ -340,30 +345,51 @@ export const inventory = (owner: OwnerId) => .list({ owner }) .pipe(Effect.flatMap((listed) => visibleAccountsAs(actor, listed))) : []; - const linked = yield* callerLinkedAccounts(apps.map((app) => app.id)); + const ids = apps.map((app) => app.id); + const linked = yield* callerLinkedAccounts(ids); // An account linked to several requirements of one app is listed once for it. const links = [ ...new Map( linked.map((link) => [`${link.app}\u0000${link.id}`, { app: link.app, account: link.id }]), ).values(), ]; - // Checks are reported for the apps the caller linked each account to. + // App links follow app access, so every listed app's are shown, with the label the app's + // accounts page shows its users. + const appLinks = (yield* appLinkedAccounts(yield* policyDatabase, ids)).map((link) => ({ + app: link.app, + requirement: link.requirement, + account: link.id, + provider: link.provider, + label: link.label, + })); + // Checks are reported for the apps each account is linked to, in either scope. + const checked = [ + ...new Map( + [...links, ...appLinks].map((link) => [ + `${link.app}\u0000${link.account}`, + { app: link.app, account: link.account }, + ]), + ).values(), + ]; const health = new Map( - (accounts.length === 0 ? [] : yield* executor.accounts.listHealth({ owner, links })).map( - (entry) => [entry.account, entry], - ), + (accounts.length === 0 + ? [] + : yield* executor.accounts.listHealth({ owner, links: checked }) + ).map((entry) => [entry.account, entry]), ); const withHealth = (listedAccounts: typeof accounts) => listedAccounts.map((account) => { const checks = health.get(account.id); return checks === undefined ? account : { ...account, health: checks }; }); - if (policy.tools.kind === "all") return { apps, accounts: withHealth(accounts), links }; + if (policy.tools.kind === "all") + return { apps, accounts: withHealth(accounts), links, appLinks }; const selected = new Set(links.map((link) => link.account)); return { apps, accounts: withHealth(accounts.filter((account) => selected.has(account.id))), links, + appLinks, }; }); /** Organization routes do not own app/account operations. */ diff --git a/apps/hosted/server/src/implementation/resource-access.ts b/apps/hosted/server/src/implementation/resource-access.ts index a522fbeb1b..4dbbd1725f 100644 --- a/apps/hosted/server/src/implementation/resource-access.ts +++ b/apps/hosted/server/src/implementation/resource-access.ts @@ -15,7 +15,7 @@ import { type AccessRevision, } from "../contracts/resource-access.ts"; import { requireGroupSharing } from "./group-sharing.ts"; -import { callerLinkedAccounts } from "./app-accounts.ts"; +import { appLinkedAccounts, callerLinkedAccounts } from "./app-accounts.ts"; import { policyDatabase, currentResourceAuthority, @@ -47,26 +47,45 @@ export const resourceDirectory = (view: "available" | "managed" = "available") = }); const usableApps = listedApps.filter(({ access }) => access.canUse).map(({ app }) => app.id); const links = yield* callerLinkedAccounts(usableApps); + // App links follow app access, so every usable app's are listed. + const appLinks = yield* appLinkedAccounts(yield* policyDatabase, usableApps); const appEntries = listedApps.map(({ app, access }) => ({ app, access, linkedAccounts: [ ...new Set(links.filter((link) => link.app === app.id).map((link) => link.id)), ], + appLinks: appLinks.flatMap((link) => + link.app === app.id + ? [ + { + requirement: link.requirement, + account: link.id, + provider: link.provider, + label: link.label, + }, + ] + : [], + ), })); const linked = new Set(links.map((link) => link.id)); const accounts = (yield* executor.accounts.list({ owner })).filter( (account) => policy.tools.kind === "all" || linked.has(account.id), ); const providers = new Map(); - // Checks are reported for the apps the caller linked each account to. + // Checks are reported for the apps each account is linked to, in either scope. + const checked = [ + ...new Map( + [...links, ...appLinks].map((link) => [ + `${link.app}\u0000${link.id}`, + { account: link.id, app: link.app }, + ]), + ).values(), + ]; const health = new Map( (accounts.length === 0 ? [] - : yield* executor.accounts.listHealth({ - owner, - links: links.map((link) => ({ account: link.id, app: link.app })), - }) + : yield* executor.accounts.listHealth({ owner, links: checked }) ).map((entry) => [entry.account, entry]), ); // Owners both use and manage their accounts, so both views list the same ones. diff --git a/apps/hosted/server/src/implementation/resource-lifecycle.ts b/apps/hosted/server/src/implementation/resource-lifecycle.ts index 0b299bff98..10c3dbc44a 100644 --- a/apps/hosted/server/src/implementation/resource-lifecycle.ts +++ b/apps/hosted/server/src/implementation/resource-lifecycle.ts @@ -4,15 +4,15 @@ import { AccountConnectionTargetChanged, AccountId, AccountScope, + AppId, StorageError, StorageHost, type Account, - type AppId, type Executor, type ResourceLifecycle, type OwnerId, } from "@executor-js/sdk/core"; -import { Context, Effect, Option, Schema } from "effect"; +import { Context, Effect, Option, Result, Schema } from "effect"; import { SqlClient } from "effect/sql"; import { CurrentUserId } from "../contracts/auth.ts"; import { OrganizationId } from "../contracts/organization.ts"; @@ -87,19 +87,32 @@ const organizationOf = (owner: OwnerId) => ) : Effect.fail(new StorageError()); -/** The rows that let a subject use an app now, as a member: exactly one when they may. */ -const appUse = ( +/** + * The rows that let a subject use these apps now, as a member: exactly one for each app they may + * use. + */ +const appsUse = ( sql: SqlClient.SqlClient, - input: { readonly id: AppId; readonly organization: OrganizationId; readonly actor: string }, + input: { + readonly ids: readonly AppId[]; + readonly organization: OrganizationId; + readonly actor: string; + }, ) => sql`select p.id from hosted_app_access p join member m on m."organizationId" = p.organization_id and m."userId" = ${input.actor} - where p.id = ${input.id} and p.organization_id = ${input.organization} + where ${sql.in("p.id", input.ids)} and p.organization_id = ${input.organization} and ((p.audience = 'private' and p.creator_id = m."userId") or p.audience = 'everyone' or (p.audience = 'groups' and (p.creator_id = m."userId" or m.role = 'owner' or exists(select 1 from hosted_app_groups g join hosted_group_members gm on gm.group_id = g.group_id where g.app_id = p.id and gm.member_id = m.id))))`; +/** The same membership policy for one app. */ +const appUse = ( + sql: SqlClient.SqlClient, + input: { readonly id: AppId; readonly organization: OrganizationId; readonly actor: string }, +) => appsUse(sql, { ids: [input.id], organization: input.organization, actor: input.actor }); + /** Whether a subject is a member who may use the app now, as any of its operations requires. */ export const actorUsesApp = ( sql: SqlClient.SqlClient, @@ -139,6 +152,14 @@ const AppRecheck = Schema.Struct({ accounts: Schema.fromJsonString(Schema.Array(AllowedAccount)), appLinks: Schema.fromJsonString(Schema.Array(AllowedUse)), }); +/** {@link AppRecheck} for several apps: the row of each app the actor may use, and each app link. */ +const AppsRecheck = Schema.Struct({ + access: Schema.fromJsonString(Schema.Array(AppId)), + accounts: AppRecheck.fields.accounts, + appLinks: Schema.fromJsonString( + Schema.Array(Schema.Struct({ ...AllowedUse.fields, app: AppId })), + ), +}); // Each account must still belong to the owner the invocation resolved it under. const allowedOf = (accounts: readonly AccountOwner[], allowed: ReadonlyArray) => { const owners = new Map(accounts.map((account) => [account.id, account.owner])); @@ -300,24 +321,71 @@ export const hostedResourceLifecycle = Effect.gen(function* () { SchemaError: () => new StorageError(), }), ), - // Every app's recheck in one statement, one row per check; each decides as `appResolving`. - appsResolving: (checks) => + // `appResolving` for every app a search reads, in one statement: the rows that let the actor + // use each app, the actor's use of every user account the apps run with, and the app links + // of every app requirement's account. Each app passes or is refused as it would be alone. + appsResolving: (owner, apps) => Effect.gen(function* () { - if (checks.length === 0) return []; - const rechecks = yield* Effect.forEach(checks, appRecheck); - const rows = yield* sql`${sql.join( - " union all ", - false, - )( - rechecks.map((check, index) => sql`select ${index}::int as recheck, ${check.columns}`), - )}`; - const checked = yield* Schema.decodeUnknownEffect( - Schema.Array(Schema.Struct({ ...AppRecheck.fields, recheck: Schema.Int })), - )(rows); - const byCheck = new Map(checked.map((row) => [row.recheck, row])); - return rechecks.map((check, index) => { - const row = byCheck.get(index); - return row === undefined ? undefined : check.passed(row); + if (apps.length === 0) return []; + const user = yield* CurrentUserId; + const organization = yield* organizationOf(owner); + const ids = [...new Set(apps.map((app) => app.id))]; + const owned = + user === undefined ? [] : organizationAccounts(apps.flatMap((app) => app.accounts)); + // An app account is the app owner's; one resolved under another owner is refused. + const usesOf = (app: (typeof apps)[number]) => + app.appUses.filter((use) => use.account.owner === owner); + const used = apps.filter((app) => usesOf(app).length > 0); + const access = + user === undefined + ? sql`select p.id from hosted_app_access p + where ${sql.in("p.id", ids)} and p.organization_id = ${organization}` + : appsUse(sql, { ids, organization, actor: user }); + const users = + user === undefined || owned.length === 0 + ? sql.literal(noRows) + : sql`(select coalesce(json_agg(allowed), '[]'::json)::text + from (${accountAccess(sql, owned, user)}) allowed)`; + const linkRows = + used.length === 0 + ? sql.literal(noRows) + : sql`(select coalesce(json_agg(json_build_object('app', l.app, + 'requirement', l.requirement, 'id', a.id)), '[]'::json)::text + from ${usableAppLinks( + sql, + used.map((app) => app.id), + )} + and ${sql.in("l.account", [ + ...new Set(used.flatMap((app) => usesOf(app).map((use) => use.account.id))), + ])})`; + const rows = yield* sql`select + (select coalesce(json_agg(access.id), '[]'::json)::text from (${access}) access) + as access, + ${users} as accounts, ${linkRows} as "appLinks"`; + const checked = (yield* Schema.decodeUnknownEffect(Schema.Array(AppsRecheck))(rows))[0]; + if (checked === undefined) return yield* new StorageError(); + return apps.map((app): Result.Result, StorageError> => { + // Exactly one row lets the actor use the app, as `appResolving` requires. + if (checked.access.filter((id) => id === app.id).length !== 1) + return Result.fail(new StorageError()); + const uses = usesOf(app); + return Result.succeed( + passedEvery( + app.accounts, + app.appUses, + allowedOf( + user === undefined ? [] : organizationAccounts(app.accounts), + checked.accounts, + ), + new Set( + checked.appLinks + .filter( + (row) => row.app === app.id && uses.some((use) => use.account.id === row.id), + ) + .map((row) => useKey(row.requirement, row.id)), + ), + ), + ); }); }).pipe( Effect.catchTags({ diff --git a/apps/hosted/server/src/implementation/resource-policy.ts b/apps/hosted/server/src/implementation/resource-policy.ts index f2e6e91649..ce52a56498 100644 --- a/apps/hosted/server/src/implementation/resource-policy.ts +++ b/apps/hosted/server/src/implementation/resource-policy.ts @@ -325,6 +325,56 @@ export const resourceAuthorityForLinks = ( Effect.catchTags({ SqlError: () => new StorageError(), SchemaError: () => new StorageError() }), ); +const MemberPoliciesLinks = Schema.Struct({ + ...Membership.fields, + policies: Schema.fromJsonString(Schema.Array(AppPolicy)), + userLinks: Schema.fromJsonString(Schema.Array(LinkedAccount)), + appLinks: Schema.fromJsonString(Schema.Array(LinkedAccount)), +}); +/** + * {@link resourceAuthorityForLinks} for many apps, such as every app a search reads: membership, + * each app's policy, the user's links to the apps whose accounts they may use now and the apps' + * usable app links, in one statement, each app's links oldest first. Apps without a policy in the + * organization are omitted, and callers deny them, as {@link resourceAuthorityForApps} does. + */ +export const resourceAuthorityForAppsWithLinks = ( + organization: OrganizationId, + user: string | undefined, + apps: readonly [AppId, ...AppId[]], +) => + Effect.gen(function* () { + if (user === undefined) return yield* new OrganizationForbidden(); + const sql = yield* policyDatabase; + const json = sql.literal(linkedAccountJson); + const rows = yield* sql`select m.id, m.role, + (select coalesce(json_agg(json_build_object('app', p.id, 'creator', p.creator_id, + 'audience', p.audience, 'revision', p.revision, + 'groups', array(select g.group_id from hosted_app_groups g where g.app_id = p.id + order by g.group_id), + 'granted', exists(select 1 from hosted_app_groups g + join hosted_group_members gm on gm.group_id = g.group_id + where g.app_id = p.id and gm.member_id = m.id))), '[]'::json)::text + from hosted_app_access p + where ${sql.in("p.id", apps)} and p.organization_id = m."organizationId") as policies, + (select coalesce(json_agg(${json} order by l.created_at, a.id), '[]'::json)::text + from ${usableLinks(sql, { subject: user, apps })}) as "userLinks", + (select coalesce(json_agg(${json} order by l.created_at, a.id), '[]'::json)::text + from ${usableAppLinks(sql, apps)}) as "appLinks" + from member m + where m."organizationId" = ${organization} and m."userId" = ${user}`; + const found = yield* Schema.decodeUnknownEffect(Schema.Array(MemberPoliciesLinks))(rows); + const row = found[0]; + if (found.length !== 1 || row === undefined) return yield* new OrganizationForbidden(); + const actor = { organization, user, member: row.id, role: row.role }; + return { + actor, + access: row.policies.map((policy) => appAccessOf(policy, actor)), + links: { user: row.userLinks, app: row.appLinks } satisfies AppAccountReads, + }; + }).pipe( + Effect.catchTags({ SqlError: () => new StorageError(), SchemaError: () => new StorageError() }), + ); + /** * {@link resourceAuthorityForLinks} for several apps, in one statement: the caller's access to * each app with a policy, with the caller's links to it and its app links. An app without a policy diff --git a/apps/hosted/server/src/implementation/tools.ts b/apps/hosted/server/src/implementation/tools.ts index bd0b875c9f..b6171d0d5e 100644 --- a/apps/hosted/server/src/implementation/tools.ts +++ b/apps/hosted/server/src/implementation/tools.ts @@ -12,6 +12,8 @@ import { ToolApprovalIssuer, ToolApprovalNotFound, ToolNotFound, + type App, + type AppAccounts, type AppId, type ApprovalRequestId, type Executor, @@ -24,7 +26,7 @@ import { type BrowserApprovalView, type BrowserToolRun, } from "@executor-js/mcp/browser"; -import { Effect, Option, Schema } from "effect"; +import { Effect, Schema } from "effect"; import { HttpApiBuilder } from "effect/http-api"; import { HostedApi } from "../contracts/api.ts"; import { CallFailures, RunFailures } from "../contracts/tools.ts"; @@ -45,8 +47,8 @@ import { checkedToolCall, checkInvocationAccounts, currentOwner, + pinnedActiveVersion, selectedActiveVersion, - selectedActiveVersions, usableApp, type CheckedApp, } from "./access.ts"; @@ -74,17 +76,24 @@ export const listTools = ( ), }; }); -/** Keep the target's current catalog as search rows after the same checks as discovery. */ -export const searchableTools = ( +/** + * Keep the target's current catalog as search rows after the same checks as discovery. `selected` + * is the caller's access to the app and its accounts when a read made for many apps at once + * decided them (see `selectedAppFrom`); otherwise they are read for this app alone. + */ +export const searchableTools = ( input: Parameters[0], options?: ToolListOptions, - checked?: CheckedApp, + selected?: Effect.Effect<{ readonly app: App; readonly accounts: AppAccounts }, E>, ) => Effect.gen(function* () { const policy = yield* authorizeApp(input.app); const owner = yield* currentOwner; const executor = yield* Effect.flatten(HostedExecutor); - const { version, accounts } = yield* selectedActiveVersion(executor, owner, input, checked); + const { version, accounts } = + selected === undefined + ? yield* selectedActiveVersion(executor, owner, input) + : yield* Effect.flatMap(selected, (found) => pinnedActiveVersion(found, input.version)); const searchable = yield* executor.tools.searchable({ ...input, version, accounts }, options); // Failed routers list no tools, so the policy decides them without the catalog. return { @@ -101,19 +110,21 @@ export const searchableTools = ( */ export const searchableKept = ( inputs: ReadonlyArray[0]>, - checked: ReadonlyArray, + selections: ReadonlyMap, ) => Effect.gen(function* () { const policy = yield* CurrentPolicy; - const owner = yield* currentOwner; const executor = yield* Effect.flatten(HostedExecutor); - const permitted = inputs.filter((input) => permitsApp(policy, input.app)); - const selected = yield* selectedActiveVersions(owner, permitted, checked); - const targets = permitted.flatMap((input, index) => { - const found = selected[index]; - return found === undefined || Option.isNone(found) + const targets = inputs.flatMap((input) => { + if (!permitsApp(policy, input.app)) return []; + const found = selections.get(input.app); + const version = found?.app.activeVersion; + return found === undefined || + version === null || + version === undefined || + (input.version !== undefined && input.version !== version) ? [] - : [{ input, read: { ...input, ...found.value } }]; + : [{ input, read: { ...input, version, accounts: found.accounts } }]; }); const kept = yield* executor.tools.searchableKept(targets.map(({ read }) => read)); const served = new Map( diff --git a/apps/hosted/server/src/implementation/webhooks.ts b/apps/hosted/server/src/implementation/webhooks.ts index 3616bb0a70..6be461b4d1 100644 --- a/apps/hosted/server/src/implementation/webhooks.ts +++ b/apps/hosted/server/src/implementation/webhooks.ts @@ -1,7 +1,6 @@ -import { CurrentOrganization, OrganizationForbidden } from "../contracts/organization.ts"; +import { OrganizationForbidden } from "../contracts/organization.ts"; /** Shared product policy; local does not acquire organizations to reuse webhook execution. */ import { SessionAuthentication, BearerAuthentication, CurrentUserId } from "../contracts/auth.ts"; -import { HttpServerRequest } from "effect/http"; import { HttpServerResponse } from "effect/http"; import { Effect } from "effect"; import { HttpApiBuilder } from "effect/http-api"; @@ -26,6 +25,7 @@ import { actorManagerOwner, appReaderOwner, checkAccounts, + currentOrganizationSlug, currentOwner, selectedApp, usableApp, @@ -158,12 +158,7 @@ export const hostedWebhookHandlers = HttpApiBuilder.group(HostedApi, "webhooks", const executor = yield* Effect.flatten(HostedExecutor); yield* managedSubscription(executor, params); yield* executor.webhookSetup.read(params); - const request = yield* HttpServerRequest.HttpServerRequest; - const headers = new Headers(request.headers); - const slug = headers.has("authorization") - ? (yield* api.authenticate(headers, (yield* CurrentOrganization).organization)) - .organizationSlug - : yield* auth.organizationSlug(headers, (yield* CurrentOrganization).organization); + const slug = yield* currentOrganizationSlug(auth, api); return { url: `${auth.origin}/org/${encodeURIComponent(slug)}/webhooks/${encodeURIComponent(params.app)}/${encodeURIComponent(params.subscription)}`, }; diff --git a/apps/hosted/server/src/index.ts b/apps/hosted/server/src/index.ts index 7f27cf583a..f90f511472 100644 --- a/apps/hosted/server/src/index.ts +++ b/apps/hosted/server/src/index.ts @@ -64,7 +64,11 @@ export { lookupOrganizationSlug, noOrganizationRemovals, } from "./implementation/organization.ts"; -export { requireOrganizationAdmin, requireOrganizationOwner } from "./implementation/access.ts"; +export { + currentOrganizationSlug, + requireOrganizationAdmin, + requireOrganizationOwner, +} from "./implementation/access.ts"; export { hostedMiddlewareLive } from "./implementation/middleware.ts"; export * from "./contracts/organization-removal.ts"; export { diff --git a/apps/hosted/testing/database-pool-fixture.ts b/apps/hosted/testing/database-pool-fixture.ts index 94f4ab334e..8d688c899a 100644 --- a/apps/hosted/testing/database-pool-fixture.ts +++ b/apps/hosted/testing/database-pool-fixture.ts @@ -24,6 +24,13 @@ * snapshot inside `PgPool.held`, and refuses to write `BEGIN` outside a hold. A batch whose * pool cannot connect (both attempts closed at once, as by a restarting pooler) returns each * leaf's failure rather than failing itself. + * - `commit-unanswered`: a batch leaf's statement after the one its batch sent is not written while + * the batch's `COMMIT` is unanswered (through Hyperdrive such a statement is never answered); it + * is written, and answered, once the `COMMIT` is. A proxy holds the server's answer to the + * `COMMIT` for a moment and counts what the client writes meanwhile. + * - `commit-rollback`: the same when a statement inside the batch fails, so its `COMMIT` is + * answered as a rollback: the later statement still waits for that answer, then is written and + * answered. */ import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -45,6 +52,7 @@ import { Fiber, FileSystem, Layer, + Option, Redacted, Schedule, Schema, @@ -64,6 +72,8 @@ const cases = [ "sibling", "batch", "connect-abandoned", + "commit-unanswered", + "commit-rollback", ] as const; /** The protocol code of a `CancelRequest`, the first message on its own connection. */ @@ -117,6 +127,78 @@ const proxy = (target: URL, dropCancels: boolean) => }), ); +/** + * A TCP proxy to the database that holds the server's answer to each `COMMIT` (its + * `CommandComplete`, `COMMIT` or, for a transaction that failed, `ROLLBACK`, and every message + * after it) for `holdMillis`, and counts the client's writes that arrive while it holds one: writes + * made while a `COMMIT` is unanswered, as the client sees it. + */ +const commitHoldingProxy = (target: URL, holdMillis: number) => + Effect.acquireRelease( + Effect.callback<{ readonly server: Net.Server; readonly behindCommit: { count: number } }>( + (resume) => { + const behindCommit = { count: 0 }; + const sockets = new Set(); + const server = Net.createServer((client) => { + sockets.add(client); + const upstream = Net.connect(Number(target.port || 5432), target.hostname); + sockets.add(upstream); + let holding = false; + let held: Array = []; + // The server's messages, cut at their boundaries: a type byte and a length that counts + // itself. The startup reply begins with messages too. + let rest = Buffer.alloc(0); + upstream.on("data", (chunk: Buffer) => { + rest = Buffer.concat([rest, chunk]); + while (rest.length >= 5) { + const length = rest.readInt32BE(1) + 1; + if (rest.length < length) break; + const message = rest.subarray(0, length); + rest = rest.subarray(length); + const tag = + message[0] === 0x43 ? message.subarray(5, length - 1).toString("latin1") : ""; + const commit = tag === "COMMIT" || tag === "ROLLBACK"; + if (commit && !holding) { + holding = true; + setTimeout(() => { + holding = false; + const out = Buffer.concat(held); + held = []; + if (!client.destroyed) client.write(out); + }, holdMillis); + } + if (holding) held.push(Buffer.from(message)); + else client.write(message); + } + }); + client.on("data", (chunk) => { + if (holding) behindCommit.count++; + upstream.write(chunk); + }); + upstream.once("close", () => { + sockets.delete(upstream); + client.destroy(); + }); + client.once("close", () => { + sockets.delete(client); + upstream.destroy(); + }); + upstream.on("error", () => client.destroy()); + client.on("error", () => upstream.destroy()); + }); + server.on("close", () => { + for (const socket of sockets) socket.destroy(); + }); + server.listen(0, "127.0.0.1", () => resume(Effect.succeed({ server, behindCommit }))); + }, + ), + ({ server }) => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + server.emit("close"); + }), + ); + const command = Command.make("database-pool-fixture", { configuration: Flag.String("configuration"), case: Flag.Literals("case", cases), @@ -510,6 +592,55 @@ const command = Command.make("database-pool-fixture", { }, }; } + case "commit-unanswered": + case "commit-rollback": { + const holding = yield* commitHoldingProxy(target, 300); + const address = holding.server.address(); + if (address === null || typeof address === "string") + return yield* Effect.die("The proxy has no port"); + const url = new URL(target.toString()); + url.hostname = "127.0.0.1"; + url.port = String(address.port); + // A client of the session objects' shape, whose statements `pipelined` claims. + const client = yield* Layer.build( + PgClient.layer({ + url: Redacted.make(url.toString()), + applicationName, + maxConnections: 2, + multiplex: true, + multiplexConcurrency: 1, + prepare: false, + }), + ).pipe(Effect.orDie); + const sql = Context.get(client, SqlClient.SqlClient); + // The first leaf reads again once its first statement answers, as a bearer recheck + // whose key was contended does: its batch's `COMMIT` has not answered yet. It runs + // outside the batch's transaction, so a failure inside the transaction does not + // reach it. + let later = "not read"; + const recheck = Effect.gen(function* () { + yield* sql`SELECT 1 AS first`; + const rows = yield* sql<{ readonly later: number }>`SELECT 2 AS later`; + later = rows[0]?.later === 2 ? "answered" : "wrong rows"; + }); + // A statement that fails inside the transaction aborts it, so its `COMMIT` is + // answered as a rollback. + const inside = args.case === "commit-rollback" ? sql`SELECT 1 / 0` : sql`SELECT 3`; + const exits = yield* PgPool.held( + PgConnection.pipelined([recheck, inside, sql`SELECT 4`], { + outside: 1, + begin: "ISOLATION LEVEL REPEATABLE READ READ ONLY", + }), + ).pipe(Effect.timeoutOption("10 seconds")); + return { + leaves: Option.match(exits, { + onNone: () => "timed out", + onSome: (exits) => exits.map((exit) => (Exit.isSuccess(exit) ? "ok" : "failed")), + }), + behindCommit: holding.behindCommit.count, + later, + }; + } case "batch": { // A client of the session objects' shape: its statements go through the borrower // that `pipelined` claims them from. diff --git a/apps/hosted/testing/scenario-server.ts b/apps/hosted/testing/scenario-server.ts index 74d2e86a75..d3320d5305 100644 --- a/apps/hosted/testing/scenario-server.ts +++ b/apps/hosted/testing/scenario-server.ts @@ -1,7 +1,7 @@ /** Local fixture control process. Database credentials stay in memory and never enter the Worker. */ import { createServer } from "node:http"; import { timingSafeEqual } from "node:crypto"; -import { Pool } from "pg"; +import { Pool, type PoolClient } from "pg"; import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; @@ -328,6 +328,38 @@ const main = Effect.gen(function* () { * The app domain status the product's location read reports for this scenario's team: the row * the coordinator wrote for its current slug, or `absent` before a pass has written one. */ + /** + * The stage's webhook source queue: how many sources wait, pending and unleased, for the sweep; + * how many team and member provisioning jobs, which link the default app's accounts and so queue + * sources, are still to finish; and the database's clock when it counted. Counts of the test + * stage's own rows, never which. + */ + const webhookQueue = Effect.gen(function* () { + const database = pool; + if (auth === undefined || database === undefined) + return yield* new TestAccountFailed({ stage: "configuration" }); + const result = yield* Effect.tryPromise({ + try: () => + database.query( + `select (select count(*)::int from executor_webhook_sources + where status = 'pending' and lease is null) as pending, + (select count(*)::int from hosted_provisioning + where kind in ('team', 'member') and status in ('queued', 'running')) as provisioning, + (extract(epoch from now()) * 1000)::bigint::text as at`, + ), + catch: () => new TestAccountFailed({ stage: "database" }), + }); + const [row] = yield* Schema.decodeUnknownEffect( + Schema.Tuple([ + Schema.Struct({ + pending: Schema.Int, + provisioning: Schema.Int, + at: Schema.NumberFromString, + }), + ]), + )(result.rows); + return row; + }); const domain = Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest; const { id } = yield* request.json.pipe( @@ -711,6 +743,120 @@ const main = Effect.gen(function* () { deletionMs, }; }); + /** + * A scenario's hold on the row of one of its own users' API keys, so its MCP calls' key checks + * contend as behind another request's use: `age` makes the key's last use the given number of + * seconds old, `hold` updates it to now in a transaction left open, which the calls' checks wait + * for, and `release` commits that. One hold per scenario; the fixture rolls back what it still + * holds when it ends. + */ + const KeyUse = Schema.Struct({ + id: Id, + key: Schema.NonEmptyString, + action: Schema.Literals(["age", "hold", "release", "blocked"]), + seconds: Schema.optional(Schema.Number.check(Schema.isBetween({ minimum: 0, maximum: 3600 }))), + }); + /** Each scenario's open hold: its connection and that connection's backend, which others wait on. */ + const holds = new Map(); + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + for (const { client } of holds.values()) { + await client.query("rollback").catch(() => undefined); + client.release(); + } + holds.clear(); + }), + ); + const keyUse = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const input = yield* request.json.pipe(Effect.flatMap(Schema.decodeUnknownEffect(KeyUse))); + const scenario = owned.get(input.id); + const database = pool; + if (scenario === undefined || database === undefined) + return yield* new TestAccountFailed({ stage: "fixture" }); + const users = [...scenario.users]; + const run = (text: string, params: ReadonlyArray) => + Effect.tryPromise({ + try: () => database.query(text, [...params]), + catch: () => new TestAccountFailed({ stage: "database" }), + }); + // Only a key of the scenario's own users. + const mine = `id = $1 and "referenceId" = any($2::text[])`; + switch (input.action) { + case "age": { + const updated = yield* run( + `update apikey set "lastRequest" = now() - make_interval(secs => $3) where ${mine}`, + [input.key, users, input.seconds ?? 0], + ); + return { updated: updated.rowCount ?? 0 }; + } + case "hold": { + if (holds.has(input.id)) return yield* new TestAccountFailed({ stage: "fixture" }); + const client = yield* Effect.tryPromise({ + try: () => database.connect(), + catch: () => new TestAccountFailed({ stage: "database" }), + }); + const held = yield* Effect.tryPromise({ + try: async () => { + await client.query("begin"); + const updated = await client.query( + `update apikey set "lastRequest" = now() where ${mine}`, + [input.key, users], + ); + const backend = await client.query<{ pid: number }>("select pg_backend_pid() as pid"); + return { rowCount: updated.rowCount, pid: Number(backend.rows[0]?.pid) }; + }, + catch: () => new TestAccountFailed({ stage: "database" }), + }).pipe( + Effect.tapError(() => + Effect.promise(async () => { + await client.query("rollback").catch(() => undefined); + client.release(); + }), + ), + ); + holds.set(input.id, { client, pid: held.pid }); + return { updated: held.rowCount ?? 0 }; + } + case "release": { + const hold = holds.get(input.id); + if (hold === undefined) return { released: false }; + const client = hold.client; + holds.delete(input.id); + yield* Effect.tryPromise({ + try: () => client.query("commit"), + catch: () => new TestAccountFailed({ stage: "database" }), + }).pipe(Effect.ensuring(Effect.sync(() => client.release()))); + return { released: true }; + } + case "blocked": { + // Sessions waiting on the scenario's hold: those whose blockers include its backend. + // Postgres queues later lockers behind the first waiter, so a session counts when the hold + // blocks it or blocks a session that blocks it. Only the bearer key check's statement + // (`loadPat`) counts as a key check; any other waiter is listed by how its statement + // begins (this database is a test stage's or local Cloud's). + const hold = holds.get(input.id); + if (hold === undefined) return { keyChecks: 0, others: [] }; + const rows = yield* run( + `with recursive waiting(pid) as ( + select pid from pg_stat_activity where $1::int = any(pg_blocking_pids(pid)) + union + select a.pid from pg_stat_activity a join waiting w on w.pid = any(pg_blocking_pids(a.pid)) + ) + select a.query like '%from apikey where key =%' as key_check, + left(regexp_replace(a.query, '\\s+', ' ', 'g'), 40) as statement + from waiting w join pg_stat_activity a on a.pid = w.pid`, + [hold.pid], + ); + return { + keyChecks: rows.rows.filter((row) => row.key_check === true).length, + others: rows.rows + .filter((row) => row.key_check !== true) + .map((row) => String(row.statement ?? "")), + }; + } + } + }); const routes = Layer.mergeAll( HttpRouter.add("POST", "/configure", endpoint(configure)), HttpRouter.add("POST", "/actors", endpoint(provision)), @@ -719,6 +865,8 @@ const main = Effect.gen(function* () { HttpRouter.add("POST", "/remove", endpoint(remove)), HttpRouter.add("POST", "/settle", endpoint(settle)), HttpRouter.add("POST", "/domain", endpoint(domain)), + HttpRouter.add("POST", "/webhook-queue", endpoint(webhookQueue)), + HttpRouter.add("POST", "/api-key-use", endpoint(keyUse)), HttpRouter.add( "GET", "/health", diff --git a/apps/hosted/web/src/contracts/app-accounts.ts b/apps/hosted/web/src/contracts/app-accounts.ts index 2caaeace9a..1c434098fc 100644 --- a/apps/hosted/web/src/contracts/app-accounts.ts +++ b/apps/hosted/web/src/contracts/app-accounts.ts @@ -150,7 +150,7 @@ const sameLink = (row: AppAccountLink, key: LinkKey) => /** * Publish a confirmed link or unlink to the scope's list and the requirement's choices, which * reads both again for a plain requirement's replaced account, then read what else it changed: the - * inventory's user links, the account's apps, and the app's discovery. + * inventory's links, the account's apps, and the app's discovery. */ const linkChanged = (get: Atom.FnContext, key: LinkKey, saved: AppAccountLink | undefined) => { const app = new AppKey({ organization: key.organization, app: key.app }); @@ -174,7 +174,7 @@ const linkChanged = (get: Atom.FnContext, key: LinkKey, saved: AppAccountLink | const others = rows.filter((row) => !sameLink(row, key)); return saved === undefined ? others : [...others, saved]; }); - if (key.scope === "user") refreshResourceDirectory(get, key.organization); + refreshResourceDirectory(get, key.organization); get.refresh(accountAtom({ organization: key.organization, account: key.account })); refreshAppDiscovery(get, key); }; diff --git a/apps/hosted/web/src/contracts/apps.ts b/apps/hosted/web/src/contracts/apps.ts index 4dccf74912..610cd2467a 100644 --- a/apps/hosted/web/src/contracts/apps.ts +++ b/apps/hosted/web/src/contracts/apps.ts @@ -178,9 +178,16 @@ const removeApp = Atom.family((key: AppKey) => ...data, apps: data.apps.filter((app) => app.id !== key.app), links: data.links.filter((link) => link.app !== key.app), + appLinks: data.appLinks.filter((link) => link.app !== key.app), })); + const shared = Option.isSome(current) + ? current.value.appLinks.flatMap((link) => (link.app === key.app ? [link.account] : [])) + : []; if (Option.isSome(current)) - for (const account of linkedAccountIds(current.value.links, key.app)) + for (const account of new Set([ + ...linkedAccountIds(current.value.links, key.app), + ...shared, + ])) acknowledge( get, accountAtom({ organization: key.organization, account }), diff --git a/apps/hosted/web/src/contracts/resource-access.ts b/apps/hosted/web/src/contracts/resource-access.ts index ba1786bd7e..987f4f419d 100644 --- a/apps/hosted/web/src/contracts/resource-access.ts +++ b/apps/hosted/web/src/contracts/resource-access.ts @@ -129,18 +129,26 @@ const inventory = Atom.family((key: ListKey) => access.audience.groups.some((id) => id === key.group))), ) .map(({ app }) => app); - const links = data.apps - .filter(({ app }) => apps.some((item) => item.id === app.id)) - .flatMap(({ app, linkedAccounts }) => - linkedAccounts.map((account) => ({ app: app.id, account })), - ); + const listed = data.apps.filter(({ app }) => apps.some((item) => item.id === app.id)); + const links = listed.flatMap(({ app, linkedAccounts }) => + linkedAccounts.map((account) => ({ app: app.id, account })), + ); + const appLinks = listed.flatMap(({ app, appLinks }) => + appLinks.map((link) => ({ ...link, app: app.id })), + ); const accounts = data.accounts.map(({ account, provider, health }) => ({ ...account, providerName: provider.definition.name, providerUrl: providerDisplayUrl(provider.definition), ...(health === undefined ? {} : { health }), })); - return { apps, accounts, links, pendingApp: key.group !== "private" && data.pendingApp }; + return { + apps, + accounts, + links, + appLinks, + pendingApp: key.group !== "private" && data.pendingApp, + }; }), ), ); diff --git a/apps/local/desktop/README.md b/apps/local/desktop/README.md index a916feef9b..6829d95172 100644 --- a/apps/local/desktop/README.md +++ b/apps/local/desktop/README.md @@ -255,7 +255,7 @@ so the menu explains that they must be changed where they are set. reveals it. It contains a manifest (versions, platform, origin, port source, data directory) and the allowlisted files from `diagnostics/`: the rotating `executor-*.jsonl` logs and `collector.json`, at most 50 MiB each and 14 days old. Keys, `installation.json`, databases, -retained builds, the browser profile and Motel's trace store are never read. Logs are already +retained builds, the browser profile and the telemetry collector's store are never read. Logs are already redacted at the source; the export additionally masks bearer values, `pair`/`token`/`code`/`state` URL parameters, 64-digit hex runs and credential-named JSON fields. @@ -283,7 +283,7 @@ from this workspace launcher. Windows and Linux have not been verified. ## Persistent diagnostics -The background server runs the Motel collector and persists traces and logs under the +The background server runs the telemetry collector and persists traces and logs under the data directory's `diagnostics/`. The Electron parent records startup, connections, recovery and renderer messages in `executor-desktop.jsonl` beside them. See [telemetry](../../../notes/telemetry.md#frontend-lifetime-and-local-diagnostics) diff --git a/apps/local/desktop/src/implementation/diagnostics.ts b/apps/local/desktop/src/implementation/diagnostics.ts index d3b31aa9bc..95354c201d 100644 --- a/apps/local/desktop/src/implementation/diagnostics.ts +++ b/apps/local/desktop/src/implementation/diagnostics.ts @@ -2,7 +2,8 @@ * Diagnostics export: one zip in Downloads that a user can attach to a report. * * Only allowlisted log files from the data directory's `diagnostics/` are read. Keys, the - * installation record, databases, retained builds, the browser profile and Motel's trace store are + * installation record, databases, retained builds, the browser profile and the telemetry collector's + * store are * never opened. Logs already carry redacted values; the export also masks credential shapes * (bearer values, pairing and signed-link tokens, 64-character hex keys and credential-named * JSON fields) as a second barrier before anything leaves the machine. @@ -85,7 +86,7 @@ export const makeExportDiagnosticsAction = (options: { "keys and installation record", "databases and retained builds", "browser profile", - "Motel trace storage", + "telemetry collector storage", ], }; files["manifest.json"] = strToU8(`${JSON.stringify(manifest, null, 2)}\n`); diff --git a/apps/local/server/src/contracts/dashboard.ts b/apps/local/server/src/contracts/dashboard.ts index 0d94fdae17..307583b058 100644 --- a/apps/local/server/src/contracts/dashboard.ts +++ b/apps/local/server/src/contracts/dashboard.ts @@ -425,6 +425,16 @@ export const DashboardOverview = Schema.Struct({ apps: Schema.Array(App), /** The accounts linked to each app, oldest first. */ links: Schema.Array(Schema.Struct({ app: AppId, account: AccountId })), + /** The app links of each app, oldest first. */ + appLinks: Schema.Array( + Schema.Struct({ + app: AppId, + requirement: Schema.String, + account: AccountId, + provider: ProviderId, + label: Schema.String, + }), + ), accounts: Schema.Array(DashboardAccount), }); export type DashboardOverview = typeof DashboardOverview.Type; diff --git a/apps/local/server/src/implementation/app-accounts.ts b/apps/local/server/src/implementation/app-accounts.ts index 0285989e10..0a3b4956ca 100644 --- a/apps/local/server/src/implementation/app-accounts.ts +++ b/apps/local/server/src/implementation/app-accounts.ts @@ -626,31 +626,10 @@ const passing = ( export const localResourceLifecycle = ( executor: Effect.Effect, sql: SqlClient.SqlClient, -): ResourceLifecycle => ({ - userLinks: (input) => - Effect.gen(function* () { - const sdk = yield* executor; - const app = yield* storage(sdk.apps.get({ app: input.app })); - return (yield* linkedAccountsOf(sql, sdk, app, localActor)).map((link) => ({ - requirement: link.requirement, - account: link.account.id, - })); - }), - appLinks: (input) => - Effect.gen(function* () { - const sdk = yield* executor; - const app = yield* storage(sdk.apps.get({ app: input.app })); - return (yield* linkedAccountsOf(sql, sdk, app, null)).map((link) => ({ - requirement: link.requirement, - account: link.account.id, - })); - }), - // Nobody else links the account, so a source the app no longer uses stops. - sourceActorLost: (input) => - Effect.flatMap(executor, (sdk) => unlinkSource(sdk, input.app, input.account)), +): ResourceLifecycle => { // The local subject may use every account it resolves. An app requirement's account must still // be the app owner's and linked to the app for that requirement: unlinking refuses it at once. - appResolving: (app, accounts, appUses) => + const appResolving: ResourceLifecycle["appResolving"] = (app, accounts, appUses) => Effect.gen(function* () { if (appUses.length === 0) return passing(accounts, [], () => true); const sdk = yield* executor; @@ -674,34 +653,63 @@ export const localResourceLifecycle = ( use.account.owner === app.owner && current.has(`${use.requirement}\u0000${use.account.id}`), ); - }).pipe(Effect.catchTags(sqlFailures)), - // Removing what the app registered needs no current app link: unlinking is what removes its - // registrations. An app account passes while it is the app owner's, so the app could have linked - // it. - removalResolving: (app, accounts, appUses) => - Effect.succeed(passing(accounts, appUses, (use) => use.account.owner === app.owner)), - accountsResolving: (accounts) => Effect.succeed(new Set(accounts.map((account) => account.id))), - connectionCompleting: () => Effect.void, - appCreated: () => Effect.void, - appRemoving: () => Effect.void, - // The owner links in both scopes. The SDK has checked, under the app's lock, that the - // requirement still has the scope the connection was started for. - appAccountConnected: (input) => - Effect.gen(function* () { - const sdk = yield* executor; - const app = yield* storage(sdk.apps.get({ app: input.app })); - yield* addLink(sql, sdk, { - app, - requirement: input.requirement, - account: input.account, - subject: scopeSubject(input.scope), - }).pipe(Effect.mapError(() => new StorageError())); - }), - accountCreated: () => Effect.void, - // An app link is used by its app for every call; the route refuses with the apps that link it, - // and this refuses an app link made since. Its user links are removed with it. - accountRemoving: (account) => - Effect.flatMap(executor, (sdk) => appLinkApps(sql, sdk, account.id)).pipe( - Effect.flatMap((apps) => (apps.length > 0 ? new StorageError() : Effect.void)), - ), -}); + }).pipe(Effect.catchTags(sqlFailures)); + return { + userLinks: (input) => + Effect.gen(function* () { + const sdk = yield* executor; + const app = yield* storage(sdk.apps.get({ app: input.app })); + return (yield* linkedAccountsOf(sql, sdk, app, localActor)).map((link) => ({ + requirement: link.requirement, + account: link.account.id, + })); + }), + appLinks: (input) => + Effect.gen(function* () { + const sdk = yield* executor; + const app = yield* storage(sdk.apps.get({ app: input.app })); + return (yield* linkedAccountsOf(sql, sdk, app, null)).map((link) => ({ + requirement: link.requirement, + account: link.account.id, + })); + }), + // Nobody else links the account, so a source the app no longer uses stops. + sourceActorLost: (input) => + Effect.flatMap(executor, (sdk) => unlinkSource(sdk, input.app, input.account)), + appResolving, + // Local reads in process, so each app is checked on its own at no extra round trip. + appsResolving: (owner, apps) => + Effect.forEach(apps, (app) => + Effect.result(appResolving({ id: app.id, owner }, app.accounts, app.appUses)), + ), + // Removing what the app registered needs no current app link: unlinking is what removes its + // registrations. An app account passes while it is the app owner's, so the app could have linked + // it. + removalResolving: (app, accounts, appUses) => + Effect.succeed(passing(accounts, appUses, (use) => use.account.owner === app.owner)), + accountsResolving: (accounts) => Effect.succeed(new Set(accounts.map((account) => account.id))), + connectionCompleting: () => Effect.void, + appCreated: () => Effect.void, + appRemoving: () => Effect.void, + // The owner links in both scopes. The SDK has checked, under the app's lock, that the + // requirement still has the scope the connection was started for. + appAccountConnected: (input) => + Effect.gen(function* () { + const sdk = yield* executor; + const app = yield* storage(sdk.apps.get({ app: input.app })); + yield* addLink(sql, sdk, { + app, + requirement: input.requirement, + account: input.account, + subject: scopeSubject(input.scope), + }).pipe(Effect.mapError(() => new StorageError())); + }), + accountCreated: () => Effect.void, + // An app link is used by its app for every call; the route refuses with the apps that link it, + // and this refuses an app link made since. Its user links are removed with it. + accountRemoving: (account) => + Effect.flatMap(executor, (sdk) => appLinkApps(sql, sdk, account.id)).pipe( + Effect.flatMap((apps) => (apps.length > 0 ? new StorageError() : Effect.void)), + ), + }; +}; diff --git a/apps/local/server/src/implementation/dashboard.ts b/apps/local/server/src/implementation/dashboard.ts index 4907ed869a..504749ff79 100644 --- a/apps/local/server/src/implementation/dashboard.ts +++ b/apps/local/server/src/implementation/dashboard.ts @@ -247,7 +247,7 @@ export const dashboard = ( { concurrency: 4 }, ); // An account's checks are those of the apps that link it, in either scope; the inventory - // lists the user links, each app and account once. + // lists the user links, each app and account once, and the app links. const all = yield* accountLinks(sql, executor); const pairs = (subjectScoped: boolean) => { const seen = new Set(); @@ -259,6 +259,21 @@ export const dashboard = ( }); }; const links = pairs(true); + const listed = new Map(accounts.map((account) => [account.id, account])); + const appLinks = all.flatMap((link) => { + const account = listed.get(link.account); + return link.subject !== null || account === undefined + ? [] + : [ + { + app: link.app, + requirement: link.requirement, + account: account.id, + provider: account.provider, + label: account.label, + }, + ]; + }); const health = yield* executor.accounts.listHealth({ links: pairs(false) }); const checks = new Map(health.map((entry) => [entry.account, entry])); const definitions = new Map(providers.map((provider) => [provider.id, provider.definition])); @@ -275,7 +290,7 @@ export const dashboard = ( ...(health === undefined ? {} : { health }), }); } - return { apps, links, accounts: display }; + return { apps, links, appLinks, accounts: display }; }); const appDetail = (appId: AppId) => Effect.gen(function* () { diff --git a/apps/local/server/src/implementation/mcp.ts b/apps/local/server/src/implementation/mcp.ts index df47dd9241..39ad104f85 100644 --- a/apps/local/server/src/implementation/mcp.ts +++ b/apps/local/server/src/implementation/mcp.ts @@ -74,7 +74,6 @@ const grantedBackend = (sql: SqlClient.SqlClient, executor: Executor, principal: Effect.forEach(input.apps, (app) => Effect.result(linkedAccountsOf(sql, executor, { app }))), listTools: (input, options) => executor.tools.list(input, options), searchableTools: (input, options) => executor.tools.searchable(input, options), - // Local reads each app's rows with `searchableTools`, after resolving its accounts. searchableKept: (inputs) => Effect.succeed(inputs.map(() => undefined)), searchTools: (input) => executor.tools.search(input), callTool: (input, options?: ToolInvocationOptions) => executor.tools.call(input, options), diff --git a/apps/local/web/src/contracts/onboarding.ts b/apps/local/web/src/contracts/onboarding.ts index 6e50f0c502..eeb34fca32 100644 --- a/apps/local/web/src/contracts/onboarding.ts +++ b/apps/local/web/src/contracts/onboarding.ts @@ -28,9 +28,14 @@ export const deleteAppAtom = Atom.family((app: AppId) => acknowledge(get, overviewAtom, (data) => ({ ...data, apps: data.apps.filter((current) => current.id !== app), + links: data.links.filter((link) => link.app !== app), + appLinks: data.appLinks.filter((link) => link.app !== app), })); if (Option.isSome(current)) - for (const account of linkedAccountIds(current.value.links, app)) + for (const account of new Set([ + ...linkedAccountIds(current.value.links, app), + ...current.value.appLinks.flatMap((link) => (link.app === app ? [link.account] : [])), + ])) acknowledge(get, accountAtom(account), (data) => ({ ...data, apps: data.apps.filter((current) => current.id !== app), diff --git a/apps/marketing/public/.well-known/security.txt b/apps/marketing/public/.well-known/security.txt new file mode 100644 index 0000000000..0962905695 --- /dev/null +++ b/apps/marketing/public/.well-known/security.txt @@ -0,0 +1,8 @@ +Contact: mailto:security@executor.sh +Contact: https://github.com/UsefulSoftwareCo/executor/security/advisories/new +Expires: 2027-11-17T00:00:00Z +Preferred-Languages: en +Canonical: https://executor.sh/.well-known/security.txt +Canonical: https://app.executor.sh/.well-known/security.txt +Canonical: https://mcp.executor.sh/.well-known/security.txt +Policy: https://github.com/UsefulSoftwareCo/executor/blob/v2/SECURITY.md diff --git a/bun.lock b/bun.lock index 8b996a1a04..118c01f097 100644 --- a/bun.lock +++ b/bun.lock @@ -40,7 +40,7 @@ }, "apps/cli": { "name": "executor", - "version": "2.0.0-beta.50", + "version": "2.0.0-beta.59", }, "apps/docs": { "name": "@executor-js/docs", @@ -489,7 +489,7 @@ }, "packages/apps": { "name": "apps", - "version": "0.0.1-beta.138.shuv.1", + "version": "0.0.1-beta.141.shuv.1", "dependencies": { "@cfworker/json-schema": "4.1.1", "@csstools/css-tokenizer": "4.0.2", @@ -686,10 +686,8 @@ "@bufbuild/protobuf": "2.11.0", "@executor-js/utils": "workspace:*", "effect": "4.1.4", - "workerd": "1.20260918.1", }, "devDependencies": { - "@effect/platform-bun": "4.1.4", "@effect/platform-node": "4.1.4", }, }, diff --git a/e2e/README.md b/e2e/README.md index e8ecc5482e..ec16408745 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -22,7 +22,8 @@ bun run e2e:prepare bun run e2e:self-host ``` -`e2e:prepare` builds the dashboards, app framework, bundled Motel and shared +`e2e:prepare` builds the dashboards, app framework, the telemetry collector (with cargo; it needs +[rustup](https://rustup.rs), which installs the toolchain the crate pins) and its viewer, and the shared workerd runtime artifact. Run it again after changing these inputs. It caches each build step's outputs in `~/.cache/executor-e2e-prepare` (or `$EXECUTOR_E2E_CACHE`) under a hash of the step's inputs: the files git sees in its packages and their @@ -65,15 +66,20 @@ artifacts and run the focused native scenario with the same Effect/Vitest runner ```sh bun run e2e:prepare bun run --cwd apps/hosted/self-host package:runtime +node packages/telemetry/scripts/build-collector.ts check +cp packages/telemetry/dist/collector/executor-collector apps/hosted/self-host/dist/workerd/executor-collector +cmp packages/telemetry/dist/collector/executor-collector apps/hosted/self-host/dist/workerd/executor-collector mkdir -p .local/native-auth go build -C apps/hosted/self-host/native -o ../../../../.local/native-auth/executor-host . bunx vitest run --config e2e/self-host-native.config.ts --testNamePattern 'Native self-host OAuth rate limits' ``` The same artifacts run the native telemetry scenarios (`--testNamePattern 'Native self-host -(telemetry ingest|restarts its telemetry)'`). The packaged runtime must include `motel.capnp`. -The host rewrites that config into a temporary file and serves the collector on a private Unix -socket, recorded at `.executor-telemetry` in the product data directory. Nothing listens on TCP +(telemetry ingest|restarts its telemetry)'`). The native fixture needs `executor-collector` +beside `workerd.capnp`: unlike the Dockerfile's collector stage, `package:runtime` does not +install that executable. The copy and byte comparison above stage the verified build. +The host supervises the collector on a private Unix socket, recorded at `.executor-telemetry` +in the product data directory. Nothing listens on TCP port 4318. The scenarios check that collector ingest does not delay product health, that a crashed collector restarts on its retained store after the supervisor's one-second backoff, and that Ctrl-C stops it with a draining product without a restart. @@ -140,7 +146,8 @@ job; it adds a warning and a line to the run's summary with that command for the - `BrowserDriver`: scoped Playwright process, shared by a suite. - `Browser`: isolated context per case, with an Effect `use` boundary for SDK calls. - `Evidence`: named steps, screenshots, request timings and the final outcome. -- `Telemetry`: queries spans that actually reached Motel over HTTP. +- `Telemetry`: queries spans that actually reached the collector over HTTP. A trace the collector + answered in part fails the read; failure evidence keeps its `meta` marker. - `McpOAuth`: public discovery, registration, PKCE, recorded browser consent, refresh and revocation. - `McpClient`: scoped official MCP protocol clients with safe method/revision/trace evidence. - `ClaudeClient`: real Claude Code in a Terminal Control PTY, with an injected model API. @@ -177,7 +184,8 @@ tokens. Its header lists what it cannot catch. Read secrets with `reveal`, and c expect no page errors use `noPageErrors`, which lists each message, and the browser keeps every page and console error in `page-errors.json`. -`e2e:prepare` builds the dashboards, app framework, bundled Motel and shared +`e2e:prepare` builds the dashboards, app framework, the telemetry collector (with cargo; it needs +[rustup](https://rustup.rs), which installs the toolchain the crate pins) and its viewer, and the shared workerd runtime artifact. Run it again after changing these inputs. It caches each build step's outputs in `~/.cache/executor-e2e-prepare` (or `$EXECUTOR_E2E_CACHE`) under a hash of the step's inputs: the files git sees in its packages and their @@ -265,7 +273,8 @@ job; it adds a warning and a line to the run's summary with that command for the - `BrowserDriver`: scoped Playwright process, shared by a suite. - `Browser`: isolated context per case, with an Effect `use` boundary for SDK calls. - `Evidence`: named steps, screenshots, request timings and the final outcome. -- `Telemetry`: queries spans that actually reached Motel over HTTP. +- `Telemetry`: queries spans that actually reached the collector over HTTP. A trace the collector + answered in part fails the read; failure evidence keeps its `meta` marker. - `McpOAuth`: public discovery, registration, PKCE, recorded browser consent, refresh and revocation. - `McpClient`: scoped official MCP protocol clients with safe method/revision/trace evidence. - `ClaudeClient`: real Claude Code in a Terminal Control PTY, with an injected model API. @@ -530,13 +539,13 @@ refresh tokens. A slow renewal holds its claim while other calls wait, a renewal survives its caller disconnecting, and `docker kill` mid-renewal followed by an immediate `docker start` recovers the grant within an execute deadline. -| Command | Target | Current coverage | -| ----------------------- | --------------------------------------------------------------- | ------------------------------------------------------------- | -| `bun run e2e:self-host` | Fresh Node/PGlite self-host | Shared hosted scenario, password login, account volume, Motel | -| `bun run e2e:local` | Fresh Node/PGlite local product | Pairing, replay rejection, dashboard access and reload | -| `bun run e2e:cloud` | Managed local Cloud Worker + Postgres, or explicit attached URL | Cloud onboarding, shared hosted behavior and MCP | -| `bun run e2e:parity` | Self-host and Cloudflare | Identical role/account and MCP scenarios on both | -| `bun run e2e:all` | All three | All applicable tests in one combined report | +| Command | Target | Current coverage | +| ----------------------- | --------------------------------------------------------------- | ----------------------------------------------------------------- | +| `bun run e2e:self-host` | Fresh Node/PGlite self-host | Shared hosted scenario, password login, account volume, collector | +| `bun run e2e:local` | Fresh Node/PGlite local product | Pairing, replay rejection, dashboard access and reload | +| `bun run e2e:cloud` | Managed local Cloud Worker + Postgres, or explicit attached URL | Cloud onboarding, shared hosted behavior and MCP | +| `bun run e2e:parity` | Self-host and Cloudflare | Identical role/account and MCP scenarios on both | +| `bun run e2e:all` | All three | All applicable tests in one combined report | With no `E2E_CLOUD_URL`, the Cloud target starts the real Alchemy Worker and a throwaway Postgres container on fresh ports. It provisions its external services @@ -783,18 +792,18 @@ the recorded page. Query strings/fragments are omitted. Playwright renders the bar images and scoped ffmpeg processes compose the MP4; the tested page is never modified. Raw video and trace artifacts remain in the case directory. -The server log and complete Motel database remain under each target's run folder. +The server log and the collector's complete database remain under each target's run folder. Saved trace samples are not a full trace archive. The telemetry test checks actual server spans, status and the presence of SQL spans; client spans cannot satisfy it. While running, `data/diagnostics/collector.json` identifies the collector query URL. -Managed Cloud's collector is four Motels behind one router (`serveShardedOtlpCollector` in -`support/otlp-collector.ts`). One Motel stores about 1,500 spans a second, and six workers -provisioning their organizations export more than that; a single Motel refused thousands of -exports a run, which the product dropped once its three-second export budget ran out. The router -splits each export by trace ID, sends a trace's reads to its Motel, and merges searches across -all four. Each Motel keeps its database in `data/diagnostics/motel-` and its log in -`collector-.log`; `GET /api/ingest` on the router lists each one's refused exports. +Managed Cloud's collector is four collectors behind one router (`serveShardedOtlpCollector` in +`support/otlp-collector.ts`). The split dates from Motel, which stored about 1,500 spans a second +while six workers provisioning their organizations exported more; a single Motel refused thousands +of exports a run. Whether one Rust collector keeps up there has not been measured, so the split +stays. The router splits each export by trace ID, sends a trace's reads to its collector, and +merges searches across all four. Each keeps its database in `data/diagnostics/collector-` and +its log in `collector-.log`; `GET /api/ingest` on the router lists each one's refused exports. All runs use synthetic identities. Private run directories and session files are ignored by Git. They can still contain session cookies and should not be published. @@ -977,7 +986,7 @@ hold duration, never deployment credentials, into the test process. ## Existing failure The original 1,000-account run preserved every account and selection but lost later -server telemetry before it reached Motel. That delivery assertion remains enabled; +server telemetry before it reached the collector. That delivery assertion remains enabled; this runner migration does not relax its timeout or reduce the default data volume. ## Driver references @@ -1008,7 +1017,7 @@ a failed subscription followed by a linked retry, and live access revocation. Each scenario owns its app and organization. The retry scenario requires both the failed attempt and its native link to reach the collector. -Self-host reads actual delivery through Motel. For a dedicated deployed Cloud +Self-host reads actual delivery through the bundled collector. For a dedicated deployed Cloud stage, bind `E2E_AXIOM_TOKEN` through the credential launcher and set `E2E_AXIOM_DATASET=executor-next-test-traces` together with the normal attached stage URL and private synthetic actor file. The query adapter reads only the diff --git a/e2e/check-boundary.ts b/e2e/check-boundary.ts index 3e75871797..c851877cd2 100644 --- a/e2e/check-boundary.ts +++ b/e2e/check-boundary.ts @@ -80,6 +80,17 @@ const check = Effect.gen(function* () { // This adapter observes the real OS store, never an application implementation. if (label === `support${path.sep}os-credential.ts` && specifier === "@napi-rs/keyring") return; + // Node's fetch says whether a request's connection was reused only through the + // diagnostics channels its bundled HTTP client publishes (lost-connections.ts). + if ( + label === `support${path.sep}lost-connections.ts` && + specifier === "node:diagnostics_channel" + ) + return; + // The lost-connection cases stand a stage in over TLS, resetting the TCP connection under + // the TLS session as an edge would. + if (label === `tests${path.sep}lost-connections.spec.ts` && specifier === "node:tls") + return; // This external MCP fixture needs a real TLS listener, not a product implementation. if (label === `support${path.sep}mcp-auth-patterns.ts` && specifier === "node:https") return; diff --git a/e2e/check-description-gh.ts b/e2e/check-description-gh.ts new file mode 100644 index 0000000000..75cee78166 --- /dev/null +++ b/e2e/check-description-gh.ts @@ -0,0 +1,116 @@ +/** + * The `gh` that e2e/check-description.ts puts first on e2e/ci-selection.ts's PATH. It answers each + * read the selection makes from GitHub with the same text from this checkout, and refuses any + * other call, so a new read in the selection fails here loudly instead of being answered wrong. + * + * - `api repos//pulls/ --jq '.body // ""'`: the description being checked. + * - `pr list ... --base ...`: the real `gh` for a pull request given with --pr; otherwise + * none, as for a branch with no pull request above it. + * - `api repos//compare/
... --jq ...`: each file changed on HEAD since main, and a + * renamed file's old name. + * - `api --paginate repos//pulls//files --jq '.[].filename'`: the files changed on HEAD + * since the base, a renamed file by its new name and a deleted one by its old. + * - `api repos//contents/?ref=
-H 'Accept: application/vnd.github.raw+json'`: the + * file on main. + * - `api repos//contents/?ref=
--jq '.[] | "\(.sha) \(.name)"'` (or `.path`): + * the directory's files on main with their blob IDs. + */ +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { Console, Effect, FileSystem, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +const args = process.argv.slice(2); +const env = (name: string) => process.env[name] ?? ""; + +/** Runs a command and returns what it prints, failing the stand-in with its error output. */ +const run = (command: string, commandArgs: ReadonlyArray) => + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* processes.spawn(ChildProcess.make(command, commandArgs)); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + child.stdout.pipe(Stream.decodeText(), Stream.mkString), + child.stderr.pipe(Stream.decodeText(), Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + if (Number(exitCode) !== 0) { + yield* Console.error(stderr.trim()); + return yield* Effect.die(`${command} ${commandArgs.join(" ")} failed`); + } + return stdout; + }).pipe(Effect.scoped); + +/** `git diff --name-status` of `from...HEAD`, as `[status, path, renamed-to?]` rows. */ +const changes = (from: string) => + run("git", ["diff", "--name-status", "-M", `${from}...HEAD`]).pipe( + Effect.map((text) => + text + .split("\n") + .filter((line) => line !== "") + .map((line) => line.split("\t")), + ), + ); + +const refuse = Effect.gen(function* () { + yield* Console.error( + `e2e/check-description.ts does not know how to answer \`gh ${args.join(" ")}\`, a read e2e/ci-selection.ts makes. Teach e2e/check-description-gh.ts to answer it from the checkout.`, + ); + process.exitCode = 3; +}); + +const answer = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = args.find((arg) => arg.startsWith("repos/")); + const jq = args.includes("--jq") ? (args[args.indexOf("--jq") + 1] ?? "") : ""; + const main = env("CHECK_DESCRIPTION_MAIN"); + if (args[0] === "pr" && args[1] === "list") { + const real = env("CHECK_DESCRIPTION_GH"); + if (real !== "") yield* Console.log((yield* run(real, args)).trimEnd()); + return; + } + if (args[0] !== "api" || path === undefined) return yield* refuse; + const route = path.replace(/^repos\/[^/]+\/[^/]+\//, ""); + if (/^pulls\/[^/]+$/.test(route) && jq === '.body // ""') + return yield* Console.log(yield* fs.readFileString(env("CHECK_DESCRIPTION_FILE"))); + // GitHub lists at most 3,000 of a pull request's files, and 300 of a comparison's. + if (/^pulls\/[^/]+\/files$/.test(route) && jq === ".[].filename") + return yield* Console.log( + (yield* changes(env("CHECK_DESCRIPTION_BASE"))) + .slice(0, 3000) + .map(([status, from, to]) => (status!.startsWith("R") ? to : from)) + .join("\n"), + ); + if (/^compare\/[^/]+\.\.\.[^/]+$/.test(route)) + return yield* Console.log( + (yield* changes(main)) + .slice(0, 300) + .flatMap(([status, from, to]) => (status!.startsWith("R") ? [to!, from!] : [from!])) + .join("\n"), + ); + if (!route.startsWith("contents/")) return yield* refuse; + const target = route.slice("contents/".length).replace(/\?.*$/, ""); + // The file's text as GitHub serves it raw; a trailing newline more changes no reading of it. + if (args.includes("-H")) + return yield* Console.log(yield* run("git", ["show", `${main}:${target}`])); + const field = + jq === '.[] | "\\(.sha) \\(.name)"' + ? "name" + : jq === '.[] | "\\(.sha) \\(.path)"' + ? "path" + : ""; + if (field === "") return yield* refuse; + const rows = (yield* run("git", ["ls-tree", main, `${target}/`])) + .split("\n") + .filter((line) => line !== "") + .flatMap((line) => { + const [meta, file] = line.split("\t"); + const [, type, sha] = meta!.split(" "); + return type === "blob" ? [`${sha} ${field === "name" ? file!.split("/").at(-1) : file}`] : []; + }); + yield* Console.log(rows.join("\n")); +}); + +NodeRuntime.runMain(answer.pipe(Effect.provide(NodeServices.layer))); diff --git a/e2e/check-description.ts b/e2e/check-description.ts new file mode 100644 index 0000000000..77c8cbf604 --- /dev/null +++ b/e2e/check-description.ts @@ -0,0 +1,204 @@ +/** + * Check a pull request description's `e2e` block before pushing, as CI's `select` job will: + * + * bun run e2e:check-description --file description.md + * bun run e2e:check-description - (the description on stdin) + * bun run e2e:check-description --pr 4439 (an open pull request's description and base) + * + * It runs `node e2e/ci-selection.ts`, the program the `select` job runs, with the job's + * environment, and a stand-in `gh` first on its PATH (e2e/check-description-gh.ts) that answers + * that program's reads from this checkout instead of GitHub: the description from the file, stdin + * or the pull request; the changed files from `git diff` of HEAD against the base (`--base`, the + * pull request's base, or main), and main's files, each fetched from origin as it is now (or the + * commit `--main` names, for a test). With `--pr` the checkout must hold the pull request's head + * commit, and the pull request's head branch stands for this one. It prints the same messages, + * minus the advice only CI can follow, exits with the same code, and on success prints the same + * selection: the jobs, their scenario counts and the release lanes. It checks what is committed. + * A `skip` cannot be checked before the layer above is open; with a file or stdin it says so and + * passes. + */ +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { Console, Effect, FileSystem, Path, Schema, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +class CheckFailed extends Schema.TaggedError()("CheckFailed", { + message: Schema.String, +}) {} + +const usage = + "Usage: bun run e2e:check-description (--file | - | --pr ) [--base ] [--main ]"; + +/** Runs a command and returns what it prints, its error output and its exit code. */ +const run = (command: string, args: ReadonlyArray, stdin?: string) => + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* processes.spawn( + ChildProcess.make(command, args, stdin === undefined ? {} : { stdin: "pipe" }), + ); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + child.stdout.pipe(Stream.decodeText(), Stream.mkString), + child.stderr.pipe(Stream.decodeText(), Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + return { stdout, stderr, exitCode: Number(exitCode) }; + }).pipe(Effect.scoped); + +const git = (...args: ReadonlyArray) => + run("git", args).pipe( + Effect.flatMap(({ stdout, stderr, exitCode }) => + exitCode === 0 + ? Effect.succeed(stdout.trim()) + : Effect.fail( + new CheckFailed({ message: `git ${args.join(" ")} failed: ${stderr.trim()}` }), + ), + ), + ); + +const program = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const args = process.argv.slice(2); + const option = (name: string) => { + const index = args.indexOf(name); + return index === -1 ? undefined : (args[index + 1] ?? ""); + }; + const file = option("--file"); + const pr = option("--pr"); + const stdin = args.includes("-"); + if ( + [file, pr].filter((given) => given !== undefined).length + (stdin ? 1 : 0) !== 1 || + file === "" || + pr === "" + ) + return yield* new CheckFailed({ message: usage }); + + const ghPath = (yield* run("sh", ["-c", "command -v gh"])).stdout.trim(); + const repository = + process.env["GITHUB_REPOSITORY"] ?? + /github\.com[:/]([^/]+\/[^/.]+)/.exec(yield* git("remote", "get-url", "origin"))?.[1]; + if (repository === undefined) + return yield* new CheckFailed({ + message: "Cannot tell the repository from the origin remote; set GITHUB_REPOSITORY.", + }); + + let description: string; + let baseBranch = option("--base") ?? "main"; + let headRef = yield* git("rev-parse", "--abbrev-ref", "HEAD"); + const head = yield* git("rev-parse", "HEAD"); + if (pr !== undefined) { + if (ghPath === "") + return yield* new CheckFailed({ + message: "--pr reads the pull request with gh, which is not on PATH.", + }); + const viewed = yield* run(ghPath, [ + "pr", + "view", + pr, + "--repo", + repository, + "--json", + "body,baseRefName,headRefName,headRefOid", + ]); + if (viewed.exitCode !== 0) + return yield* new CheckFailed({ + message: `gh pr view ${pr} failed: ${viewed.stderr.trim()}`, + }); + const read = yield* Schema.decodeUnknownEffect( + Schema.fromJsonString( + Schema.Struct({ + body: Schema.String, + baseRefName: Schema.String, + headRefName: Schema.String, + headRefOid: Schema.String, + }), + ), + )(viewed.stdout); + // The changed files come from this checkout, so it must hold the pull request's head. + if (head !== read.headRefOid) + return yield* new CheckFailed({ + message: `This checkout's HEAD is ${head}, but pull request ${pr}'s head is ${read.headRefOid}, so its changed files would not be the pull request's. Check out that commit first: git fetch origin ${read.headRefName} && git checkout --detach ${read.headRefOid}`, + }); + description = read.body; + baseBranch = option("--base") ?? read.baseRefName; + headRef = read.headRefName; + } else description = yield* fs.readFileString(stdin ? "/dev/stdin" : file!); + + // Main and the base as the remote has them, fetched into refs of this check's own, or the + // commit `--main` names (which also stands for the base unless `--base` names a ref). + const given = option("--main"); + const main = given ?? "refs/check-description/main"; + const base = given === undefined ? "refs/check-description/base" : (option("--base") ?? given); + if (given === undefined) { + const fetched = yield* run("git", [ + "fetch", + "--quiet", + "origin", + "+refs/heads/main:refs/check-description/main", + `+refs/heads/${baseBranch}:refs/check-description/base`, + ]); + if (fetched.exitCode !== 0) + return yield* new CheckFailed({ + message: `Could not fetch main and ${baseBranch} from origin, which the check compares with as CI does: ${fetched.stderr.trim()}`, + }); + } + yield* Console.error( + `Checking against main at ${yield* git("rev-parse", "--short", `${main}^{commit}`)} and the base ${given === undefined ? baseBranch : base} at ${yield* git("rev-parse", "--short", `${base}^{commit}`)}${given === undefined ? ", as origin has them" : ""}, for HEAD ${head.slice(0, 9)}.`, + ); + + const here = path.dirname(yield* path.fromFileUrl(new URL(import.meta.url))); + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "executor-check-description-" }); + const bin = path.join(directory, "bin"); + yield* fs.makeDirectory(bin); + yield* fs.writeFileString(path.join(directory, "description.md"), description); + yield* fs.writeFileString( + path.join(bin, "gh"), + `#!/bin/sh\nexec node ${JSON.stringify(path.join(here, "check-description-gh.ts"))} "$@"\n`, + ); + yield* fs.chmod(path.join(bin, "gh"), 0o755); + const outputs = path.join(directory, "outputs"); + yield* fs.writeFileString(outputs, ""); + yield* fs.writeFileString(path.join(directory, "summary"), ""); + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* processes.spawn( + ChildProcess.make("node", [path.join(here, "ci-selection.ts")], { + stdout: "inherit", + stderr: "inherit", + extendEnv: true, + env: { + PATH: `${bin}:${process.env["PATH"] ?? ""}`, + E2E_PULL_REQUEST: pr ?? "0", + E2E_HEAD_REF: headRef, + E2E_DEFAULT_BRANCH: "main", + E2E_COMPARE_BASE: "main", + E2E_COMPARE_HEAD: head, + E2E_LOCAL_CHECK: "1", + // Run from inside a CI job, the selection would refuse local mode (e2e/ci-selection.ts). + GITHUB_ACTIONS: "", + // A draft is complete as it is: nothing will edit it while this waits. + E2E_DESCRIPTION_WAIT_SECONDS: "0", + GITHUB_REPOSITORY: repository, + GITHUB_RUN_ID: "", + GITHUB_OUTPUT: outputs, + GITHUB_STEP_SUMMARY: path.join(directory, "summary"), + CHECK_DESCRIPTION_FILE: path.join(directory, "description.md"), + CHECK_DESCRIPTION_BASE: base, + CHECK_DESCRIPTION_MAIN: main, + CHECK_DESCRIPTION_GH: pr === undefined ? "" : ghPath, + }, + }), + ); + process.exitCode = Number(yield* child.exitCode); +}).pipe(Effect.scoped); + +NodeRuntime.runMain( + program.pipe( + Effect.catchTag("CheckFailed", (error) => + Console.error(error.message).pipe(Effect.andThen(Effect.sync(() => (process.exitCode = 2)))), + ), + Effect.provide(NodeServices.layer), + ), +); diff --git a/e2e/ci-key.ts b/e2e/ci-key.ts new file mode 100644 index 0000000000..5ee6b3ab10 --- /dev/null +++ b/e2e/ci-key.ts @@ -0,0 +1,172 @@ +/** + * What a pull request run tests, as the key a later run on the same head compares with to report + * that run's result instead of testing again (e2e/ci-repeat.ts). + * + * A run tests its merge tree with what it reads from the pull request. The key holds the merge tree + * and what that reading decided, not the description it was read from: the `select` job's outputs, + * which are everything the jobs after it are given (`selectionKey`), and the `worker-growth` + * blocks, which the size gate reads for itself (`workerGrowthKey`). So an edit to the description + * that leaves the selection as it was keeps the key, and any edit that changes what a job runs + * changes it. `pullRequestReads` lists everything a run reads from the pull request beyond its + * merge tree, with how the key covers it, and `payloadFields` the event payload's fields the + * workflows use. e2e/tests/ci-repeat.spec.ts fails for a read of these forms that neither lists: + * an API route written as a template literal (`repos/${...}/...`) or a `gh pr` command in + * e2e/ci-selection.ts; a workflow step that calls `pulls/$...` or `gh pr view` or `gh pr list`; a + * `github.event.pull_request.` in a workflow; and `GITHUB_EVENT_PATH` or + * `github.event_path` anywhere in the workflows, e2e/ or the scripts they run. It cannot see a + * route built by concatenation or held in a variable, nor a read made by a program a step starts + * that is not one of those files. + * + * node e2e/ci-key.ts worker-growth the size gate's key for a description + * node e2e/ci-key.ts record ... a record's key + */ +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { createHash } from "node:crypto"; +import { Console, Effect, FileSystem } from "effect"; + +/** A hash of `parts`, each kept whole: no part's text can stand in for a boundary. */ +const digest = (parts: ReadonlyArray) => + createHash("sha256").update(JSON.stringify(parts)).digest("hex"); + +/** + * The `select` job's outputs as one key: every `name=value` line it writes to `GITHUB_OUTPUT`, + * the same lines in any order giving the same key. + */ +export const selectionKey = (outputs: string) => + digest([ + "selection", + ...outputs + .split("\n") + .filter((line) => line !== "") + .toSorted(), + ]); + +/** + * The description's `worker-growth` blocks, as the size gate reads them + * (apps/hosted/cloud/scripts/worker-sizes.ts's `allowancesIn`; the cases check that both use this + * expression). + */ +export const workerGrowthBlock = /^```worker-growth[ \t]*\r?\n([\s\S]*?)^```/gmu; + +/** The size gate's reading of a description, as a key: each `worker-growth` block's text. */ +export const workerGrowthKey = (description: string) => + digest([ + "worker-growth", + ...[...description.matchAll(workerGrowthBlock)].map(([, block]) => block!), + ]); + +/** + * The base commit as the key holds it. A pull request into the default branch is compared with + * that commit (apps-version compares with it, and the size gate's baseline is main at it), so the + * key holds it. An upper stack layer's base commit decides nothing a run reads beyond the merge + * tree: its selection is in the key, and its size baseline (the base its stack ends on, at the + * merge base found from the head) is in the size gate's record. A stack push that moves the lower + * layer without changing the upper's tree therefore gives the upper's two runs one key. + */ +export const keyedBase = (baseRef: string, defaultBranch: string, baseSha: string) => + baseRef === defaultBranch ? baseSha : "stacked"; + +/** A record's name: what a run tested, for this merge tree and base commit. */ +export const recordKey = (tree: string, base: string, ...keys: ReadonlyArray) => + digest(["tree", tree, "base", base, ...keys]).slice(0, 40); + +/** + * Everything a pull request run reads from the pull request beyond its merge tree, and how the key + * covers it. `where` is the file that reads it and the text the read is made with. + */ +export const pullRequestReads: ReadonlyArray<{ + readonly where: string; + readonly reads: string; + readonly covered: string; +}> = [ + { + where: "e2e/ci-selection.ts: pulls/${input.number}", + reads: "the description: its e2e blocks, the skip line and the deployed-stage line", + covered: + "The blocks and skip decide the select job's outputs, which are in the key. The deployed-stage line only decides whether select succeeds; a run whose select fails records nothing, and a run is repeated only when its select succeeded.", + }, + { + where: 'e2e/ci-selection.ts: "pr", "list"', + reads: "the open pull requests based on the branch, for a skip", + covered: + "A skip with one writes skip=true, in the key; without one select fails and records nothing.", + }, + { + where: "e2e/ci-selection.ts: pulls/${pullRequest}/files", + reads: "the pull request's changed files", + covered: "They decide the outputs, in the key.", + }, + { + where: "e2e/ci-selection.ts: compare/${compareBase}...${compareHead}", + reads: "the files changed since main", + covered: "They decide the outputs, in the key.", + }, + { + where: "e2e/ci-selection.ts: contents/", + reads: + "main's package.json, patches, test plan and guarded files, compared with the merge tree", + covered: "They decide the outputs, in the key.", + }, + { + where: ".github/workflows/checks.yml: Read the pull request's description", + reads: "the description's worker-growth blocks, read by the size gate when its job runs", + covered: + "The site job records the blocks it read, and a run is repeated only when that record matches the blocks this run's select read (CI_SIZED in e2e/ci-repeat.ts).", + }, + { + where: "scripts/ci/worker-baseline.ts: pr list --head", + reads: + "the stack below the pull request, the merge base with the branch it ends on, and recorded sizes, to choose what the size gate compares with", + covered: + "The chosen baseline (branch and commit) is in the size gate's record, which select and site each compute, so a run compared with one baseline is never reported for another.", + }, + { + where: ".github/workflows/checks.yml: Record what this run tests", + reads: "the merge tree, the base commit (`keyedBase`) and the select job's two keys", + covered: "This is the key.", + }, + { + where: ".github/workflows/checks.yml: github.event.pull_request.base", + reads: + "the base branch and commit: apps-version compares with main, and the size gate finds main's sizes at the merge base", + covered: + "For a pull request into main the base commit is in the key; an upper layer's selection and size baseline are keyed instead (`keyedBase`). The head commit must be the same (e2e/ci-repeat.ts).", + }, +]; + +/** + * The event payload's fields of `pull_request` the workflows use, and how the key covers each. + */ +export const payloadFields: Readonly> = { + number: "The pull request is in the record's name.", + "head.sha": "A run is repeated only from a run on the same head (e2e/ci-repeat.ts).", + "head.ref": + "The branch whose open pull requests decide a skip; a skip with one is the output skip=true, in the key.", + "head.repo.full_name": + "Whether the pull request comes from a fork, which picks the runner: the same head, so the same repository.", + "base.sha": + "In the key for a pull request into main (`keyedBase`); an upper layer's selection and size baseline, which it could change, are keyed instead.", + "base.ref": + "Whether it is main decides whether the base commit is in the key (`keyedBase`); otherwise what it decides (the selection, the size gate's baseline) is keyed itself.", +}; + +if (process.argv[1]?.endsWith("ci-key.ts")) + NodeRuntime.runMain( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const [command, ...args] = process.argv.slice(2); + if (command === "worker-growth" && args.length === 1) + return yield* Console.log(workerGrowthKey(yield* fs.readFileString(args[0]!))); + if (command === "record" && args.length >= 4 && args.every((arg) => arg !== "")) + return yield* Console.log(recordKey(args[0]!, args[1]!, ...args.slice(2))); + if (command === "key-base" && args.length === 3 && args.every((arg) => arg !== "")) + return yield* Console.log(keyedBase(args[0]!, args[1]!, args[2]!)); + yield* Console.error( + "Usage: node e2e/ci-key.ts worker-growth \n" + + " node e2e/ci-key.ts record ... (the selection and worker-growth keys, or sized, the worker-growth key and the baseline)\n" + + " node e2e/ci-key.ts key-base ", + ); + process.exitCode = 2; + }).pipe(Effect.provide(NodeServices.layer)), + ); diff --git a/e2e/ci-repeat.ts b/e2e/ci-repeat.ts index e60c049a26..8cd7e7bca4 100644 --- a/e2e/ci-repeat.ts +++ b/e2e/ci-repeat.ts @@ -11,14 +11,19 @@ * * A run repeats another when that run of the same workflow finished on the same head, for a * `pull_request` event, and tested the same thing: its `select` job uploaded the record this run's - * did (`CI_TESTED`, an artifact named for the pull request and a hash of the merge tree and the - * description, which picks the suites). Its `checks / select` must have succeeded, so a failure + * did (`CI_TESTED`, an artifact named for the pull request and a key of the merge tree, the base + * commit for a pull request into main, the select job's outputs and the description's + * worker-growth blocks it read; see e2e/ci-key.ts). When that run's size gate decided (`checks / site` passed or failed), the gate + * must have read the same worker-growth blocks this run's select read: the site job uploads its + * own record (`CI_SIZED`). The run must be on its first attempt: a re-run's jobs may have read a + * later description than its select did. Its `checks / select` must have succeeded, so a failure * there (a missing e2e block, a missing deployed-stage line) is decided again on today's - * description. Its `checks / check` and `checks / e2e` must each have passed or failed, and a run - * that failed overall must have failed in one of them: a repeat skips the other jobs, and a - * skipped job of the newer run would cover the older run's failure of it in the pull request's - * checks. Anything else, including a re-run of this run, no record and any failure to read, tests - * in full. It writes `of`, `check` and `e2e` to `GITHUB_OUTPUT` only for a repeat; `check` and `e2e` + * description. Its `checks / check` and `checks / e2e` must both have passed, and so must the run: + * only a pass is handed on. A failure is tested again, since its cause may be a dependency rather + * than the code (the hosted emulator timing out failed one of two runs on one head, 16 Nov), and a + * repeat of it would be a red that only testing can clear. A cancelled run, or one that has not + * finished, is not repeated either. Anything else, including a re-run of this run, no record and + * any failure to read, tests in full. It writes `of`, `check` and `e2e` to `GITHUB_OUTPUT` only for a repeat; `check` and `e2e` * read the repeated run's results again when they run. */ import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; @@ -34,6 +39,8 @@ class RepeatUnknown extends Schema.TaggedError()("RepeatUnknown", const requiredChecks = { check: "checks / check", e2e: "checks / e2e" } as const; /** The job that records what a run tests and chooses its suites. */ const selectJob = "checks / select"; +/** The job whose size gate reads the description's worker-growth blocks for itself. */ +const siteJob = "checks / site"; const Run = Schema.Struct({ id: Schema.Number, @@ -43,6 +50,7 @@ const Run = Schema.Struct({ event: Schema.String, head_sha: Schema.String, workflow_id: Schema.Number, + run_attempt: Schema.Number, }); const Runs = Schema.fromJsonString(Schema.Struct({ workflow_runs: Schema.Array(Run) })); const Artifacts = Schema.fromJsonString( @@ -91,11 +99,12 @@ const repeated = Effect.gen(function* () { const attempt = Number(yield* Config.String("GITHUB_RUN_ATTEMPT")); const headSha = yield* Config.String("CI_HEAD_SHA"); const tested = yield* Config.String("CI_TESTED").pipe(Config.withDefault("")); + const sized = yield* Config.String("CI_SIZED").pipe(Config.withDefault("")); if (attempt !== 1) { yield* Console.log(`This is attempt ${attempt} of the run: a re-run tests in full.`); return Option.none(); } - if (tested === "") { + if (tested === "" || sized === "") { yield* Console.log("This run recorded nothing to compare: it tests in full."); return Option.none(); } @@ -130,24 +139,31 @@ const repeated = Effect.gen(function* () { const e2e = conclusion(requiredChecks.e2e); const select = conclusion(selectJob); const decided = (result: string) => result === "success" || result === "failure"; + const site = conclusion(siteJob); const why = - select !== "success" - ? `its select was ${select}` - : !decided(check) || !decided(e2e) - ? `its checks did not decide (check: ${check}, e2e: ${e2e})` - : run.conclusion === "failure" && check === "success" && e2e === "success" - ? "it failed in a job outside its required checks, which a repeat would skip" - : undefined; + run.run_attempt !== 1 + ? `it was run again (attempt ${run.run_attempt}), so its jobs may have read a later description than its select did` + : select !== "success" + ? `its select was ${select}` + : decided(site) && !artifacts.some((artifact) => artifact.name === sized) + ? "its size gate read other worker-growth blocks than this run's select did" + : !decided(check) || !decided(e2e) + ? `its checks did not decide (check: ${check}, e2e: ${e2e})` + : check !== "success" || e2e !== "success" + ? `its checks failed (check: ${check}, e2e: ${e2e}), and only a pass is handed on: a failure is tested again` + : run.conclusion !== "success" + ? "it failed in a job outside its required checks, which a repeat would skip" + : undefined; if (why !== undefined) { yield* Console.log( - `${run.html_url} tested the same pull request, merge tree and description, but ${why}; this run tests in full.`, + `${run.html_url} tested the same pull request, merge tree, base and selection (e2e/ci-key.ts), but ${why}; this run tests in full.`, ); return Option.none(); } return Option.some({ run, check, e2e }); } yield* Console.log( - "No finished run tested the same pull request, merge tree and description; this run tests in full.", + "No finished run tested the same pull request, merge tree, base and selection; this run tests in full.", ); return Option.none(); }); @@ -166,7 +182,7 @@ NodeRuntime.runMain( if (Option.isNone(found)) return; const { run, check, e2e } = found.value; yield* Console.log( - `This run repeats ${run.html_url}, which tested the same pull request, merge tree and description: check ${check}, e2e ${e2e}. Its check and e2e report that run's results instead of testing again.`, + `This run repeats ${run.html_url}, which tested the same pull request, merge tree, base and selection: check ${check}, e2e ${e2e}. Its check and e2e report that run's results instead of testing again.`, ); const output = yield* Config.String("GITHUB_OUTPUT").pipe(Config.option); if (Option.isSome(output)) diff --git a/e2e/ci-selection.ts b/e2e/ci-selection.ts index fc8f1b019b..64773672a4 100644 --- a/e2e/ci-selection.ts +++ b/e2e/ci-selection.ts @@ -17,7 +17,9 @@ * selected; the job is then skipped. * * The block may also say `skip` on the lower layer of a stack: another open pull request must - * build on its branch. Every check then skips, and the layer above tests the combined change. + * build on its branch. Every check then skips, and the layer above tests the combined change. The + * select job then checks that the layer above's run was built on this layer's head and compared + * Worker sizes with the base the stack ends on (scripts/ci/worker-baseline.ts `above`). * * The description is read when the job runs. `gh stack submit` opens each pull request before * its author writes the description, and opens a stack's layers one at a time, so the job waits @@ -55,10 +57,13 @@ import { } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { createHash } from "node:crypto"; -import { collectorChanged } from "./collector-inputs.ts"; +import { collectorChanged, collectorScenarios } from "./collector-inputs.ts"; import { lanesFor, releaseSources } from "./release-lanes.ts"; import { packagedEngineDependency } from "./packaged-engine.ts"; import { savePathFiles } from "./save-path.ts"; +import { fileGuards } from "./file-guards.ts"; +import { selectionKey, workerGrowthKey } from "./ci-key.ts"; +import { mcpSearchChanged, mcpSearchTitles } from "./mcp-search-guard.ts"; import { Target } from "./report-model.ts"; import { exactly, split } from "./scenario-parts.ts"; import { platformSignatures } from "./platform-signatures.ts"; @@ -173,7 +178,7 @@ const jobs = { runner: 3, pattern: - "Cloud SSO SAML accepts|Cloud's operator OAuth client signs in|Safari reports only the page's own failures|Cloud cron triggers run their jobs|Cloud cron jobs record their outcome|Cloud support dialog lists every channel|Hosted feedback enforces its API contract|Executor's catalog calls an app's own cache methods|Executor app is installed with nothing left pending|Cloud MCP routes a request inside a session|Cloud MCP sessions continue after their session object is evicted|Request and workflow attempts at one team|remote skill catalog|a skill read without a revision|abandons a GitHub skills load|reads its catalog and the tool in one cache command|a cold call's wake is reported by that call alone|another organization's admin cannot read the link|account connections cannot cross organizations|Deploys wake webhook source setup|Cloud's schedule coordinator keeps nine slow scheduled runs|a webhook source reconciling for an activation that another supersedes|A skill catalog refreshed in the background after a dashboard batch read|Cloud MCP session objects run parallel tool calls|Cloud MCP session objects cancel a statement|Cloud MCP session objects discover the other apps|Cloud MCP session objects' database pool|Cloud's database pool runs a statement whose connection|Hosted MCP resume reports an approved call's own failure|Cloud refuses to remove an organization while one of its app workflows runs|Cloud v1 session handoff|Cloud accepts MCP requests v1 forwards|Cloud validates a v1 WorkOS API key|Hosted MCP operation and bearer-check spans name|Hosted management API request spans name only|Hosted request spans record the client, the credential|Cloud's gateway names an authenticated MCP request's organization|A person deletes their own user|On Cloud a slow tool call log write|A Cloud scenario's clean-up returns once its organization's removal is accepted|The end-of-run removal check answers each look within its request limit|The end-of-run removal check asks again after a dropped request|The end-of-run removal check keeps waiting for a removal recorded done", + "Cloud SSO SAML accepts|Cloud's operator OAuth client signs in|Cloud's own Executor for Google registration refuses|Safari reports only the page's own failures|Cloud cron triggers run their jobs|Cloud cron jobs record their outcome|Cloud support dialog lists every channel|Hosted feedback enforces its API contract|Executor's catalog calls an app's own cache methods|Executor app is installed with nothing left pending|Cloud MCP routes a request inside a session|Cloud MCP sessions continue after their session object is evicted|Request and workflow attempts at one team|remote skill catalog|a skill read without a revision|abandons a GitHub skills load|reads its catalog and the tool in one cache command|a cold call's wake is reported by that call alone|another organization's admin cannot read the link|account connections cannot cross organizations|Deploys wake webhook source setup|Cloud's schedule coordinator keeps nine slow scheduled runs|a webhook source reconciling for an activation that another supersedes|A skill catalog refreshed in the background after a dashboard batch read|Cloud MCP session objects run parallel tool calls|Cloud MCP session objects cancel a statement|Cloud MCP session objects discover the other apps|Cloud MCP session objects' database pool|Cloud's database pool runs a statement whose connection|Hosted MCP resume reports an approved call's own failure|Cloud refuses to remove an organization while one of its app workflows runs|Cloud v1 session handoff|Cloud accepts MCP requests v1 forwards|Cloud validates a v1 WorkOS API key|Hosted MCP operation and bearer-check spans name|Hosted management API request spans name only|Hosted request spans record the client, the credential|Cloud's gateway names an authenticated MCP request's organization|A person deletes their own user|On Cloud a slow tool call log write|A Cloud scenario's clean-up returns once its organization's removal is accepted|The end-of-run removal check answers each look within its request limit|The end-of-run removal check asks again after a dropped request|The end-of-run removal check keeps waiting for a removal recorded done|a run's sleep whose engine call failed is called again", }, // Cloud's hosts: role hosts, the edge and sign-in on `app.`. Run in `cloud-product`, their // load kept the team installation's workflow span from arriving within its 30-second wait, so @@ -690,19 +695,6 @@ const guardedPatches = (mainManifest: string, mainPatches: string) => ); }); -/** - * Scenarios that assert a contract one file states in full. app-package.spec.ts reads back every - * protocol the host supports when it rejects an unsupported framework, so a new protocol changes - * its expected message. These run whenever the file differs from main's, so the top of a stack - * runs them when a lower layer changes it. - */ -const fileGuards: Readonly>> = { - "packages/sdk/src/implementation/app-protocols.ts": [ - "app-package.spec.ts", - "app-older-protocols.spec.ts", - ], -}; - /** * Guard spec files for every guarded file whose contents differ from main's. `mainListing` reads * one of main's directories as ` ` lines; a file missing from it counts as changed. @@ -732,8 +724,13 @@ interface Suite { readonly config: string; readonly commands: ReadonlyArray; readonly workflows: ReadonlyArray; + /** Whether a job of checks.yml that every pull request runs (`everyRunJobs`) runs it. */ + readonly everyPullRequest: boolean; } +/** The jobs of checks.yml that run on every pull request whose layer does not skip. */ +const everyRunJobs = ["static", "site", "apps-package", "tooling", "tooling-cloud"]; + const PackageScripts = Schema.Struct({ scripts: Schema.Record(Schema.String, Schema.String) }); /** * The other `e2e/*.config.ts` files, imported only when a selection needs them. Each specifier is a @@ -753,7 +750,10 @@ const suiteConfigs: Record Promise> = { "ci-supersede.config.ts": () => import("./ci-supersede.config.ts"), "ci-repeat.config.ts": () => import("./ci-repeat.config.ts"), "collector-job.config.ts": () => import("./collector-job.config.ts"), + "conflict-markers.config.ts": () => import("./conflict-markers.config.ts"), "macos-staple.config.ts": () => import("./macos-staple.config.ts"), + "npm-channel.config.ts": () => import("./npm-channel.config.ts"), + "publish-guard.config.ts": () => import("./publish-guard.config.ts"), "workflow-caches.config.ts": () => import("./workflow-caches.config.ts"), "telemetry-event-lifetime.config.ts": () => import("./telemetry-event-lifetime.config.ts"), "cli-startup-imports.config.ts": () => import("./cli-startup-imports.config.ts"), @@ -774,6 +774,8 @@ const suiteConfigs: Record Promise> = { "installer.config.ts": () => import("./installer.config.ts"), "local-bootstrap.config.ts": () => import("./local-bootstrap.config.ts"), "local-disk.config.ts": () => import("./local-disk.config.ts"), + "lost-connections-lifecycle.config.ts": () => import("./lost-connections-lifecycle.config.ts"), + "lost-connections.config.ts": () => import("./lost-connections.config.ts"), "lockfile-sync.config.ts": () => import("./lockfile-sync.config.ts"), "npm-launcher.config.ts": () => import("./npm-launcher.config.ts"), "pglite.config.ts": () => import("./pglite.config.ts"), @@ -784,6 +786,7 @@ const suiteConfigs: Record Promise> = { "prepare-cache.config.ts": () => import("./prepare-cache.config.ts"), "part-results.config.ts": () => import("./part-results.config.ts"), "process-identity.config.ts": () => import("./process-identity.config.ts"), + "cloud-postgres-volumes.config.ts": () => import("./cloud-postgres-volumes.config.ts"), "release-archives.config.ts": () => import("./release-archives.config.ts"), "release-evidence.config.ts": () => import("./release-evidence.config.ts"), "removal-evidence.config.ts": () => import("./removal-evidence.config.ts"), @@ -791,9 +794,11 @@ const suiteConfigs: Record Promise> = { "run-retention.config.ts": () => import("./run-retention.config.ts"), "standalone-lockdown.config.ts": () => import("./standalone-lockdown.config.ts"), "subrequest-depth.config.ts": () => import("./subrequest-depth.config.ts"), + "test-stage-branch.config.ts": () => import("./test-stage-branch.config.ts"), "test-stage-orphans.config.ts": () => import("./test-stage-orphans.config.ts"), "test-stage-unreadable.config.ts": () => import("./test-stage-unreadable.config.ts"), "typecheck-runner.config.ts": () => import("./typecheck-runner.config.ts"), + "webhook-sweep-waits.config.ts": () => import("./webhook-sweep-waits.config.ts"), "welcome-email.config.ts": () => import("./welcome-email.config.ts"), "worker-sizes.config.ts": () => import("./worker-sizes.config.ts"), }; @@ -828,6 +833,16 @@ const separateSuites = Effect.gen(function* () { ), ); const suites = new Map(); + const checksText = + workflows.find(({ label }) => label.endsWith("(.github/workflows/checks.yml)"))?.text ?? ""; + /** The text of one job of checks.yml, up to the next job. */ + const checksJob = (job: string) => { + const start = checksText.indexOf(`\n ${job}:\n`); + if (start === -1) return ""; + const rest = checksText.slice(start + 1); + const end = rest.slice(1).search(/\n {2}[a-z0-9-]+:\n/); + return end === -1 ? rest : rest.slice(0, end + 1); + }; const configs = (yield* fs.readDirectory(e2e)) .filter((name) => name.endsWith(".config.ts") && name !== "vitest.config.ts") .sort(); @@ -868,23 +883,87 @@ const separateSuites = Effect.gen(function* () { config, commands, workflows: workflows.filter(({ text }) => runs(text)).map(({ label }) => label), + everyPullRequest: everyRunJobs.some((job) => runs(checksJob(job))), }); } return suites; }); -/** Where a spec file these jobs never run does run, for the failure message. */ -const runsElsewhere = (file: string, suite: Suite | undefined) => { - if (suite === undefined) - return [...deployedRuns].some((title) => fileOf.get(title) === file) - ? `- ${file}: only Cloud tests on main runs its scenarios, on a deployed stage after merge. A pull request that changes the file runs those managed local Cloud can run in the ${deployedOnlyJob} job.` - : `- ${file}: these jobs exclude all of its scenarios (see the job patterns in e2e/ci-selection.ts).`; +/** + * What to do about a spec file these jobs never run, for the failure message: where it does run, + * and the edit to make. `where` names the block line it is written on. + */ +const runsElsewhere = (file: string, suite: Suite | undefined, where?: string) => { + const at = where ?? file; + // Named in the block, the file is an edit to make; changed but unnamed, it is only reported. + const nothing = + where === undefined + ? "so these jobs do not run it." + : `so naming it here runs nothing. Remove ${file} from the e2e block.`; + if (suite === undefined) { + const deployed = [...deployedRuns].filter((title) => fileOf.get(title) === file); + return deployed.length > 0 + ? `- ${at}: only the deployed stage runs ${file}'s scenarios, in Cloud tests on main after merge, ${nothing} To run its scenarios before merging, run them on a deployed stage: ${verifyCommand(deployed)}` + : `- ${at}: these jobs exclude all of ${file}'s scenarios (see the job patterns in e2e/ci-selection.ts), ${nothing}`; + } const command = suite.commands[0] ?? `bunx vitest run --config ${suite.config}`; - return suite.workflows.length === 0 - ? `- ${file} runs only by hand, with ${suite.config}: ${command}. No workflow runs it.` - : `- ${file} runs with ${suite.config} in ${suite.workflows.join(" and ")}. Locally: ${command}.`; + if (suite.workflows.length === 0) + return `- ${at}: ${file} runs only by hand, with ${suite.config} (${command}); no workflow runs it, ${nothing}`; + return suite.everyPullRequest + ? `- ${at}: ${file} runs with ${suite.config} in Checks on every pull request anyway, ${where === undefined ? "so these jobs do not run it." : `so the e2e block cannot select it. Remove ${file} from the e2e block.`} Locally: ${command}.` + : `- ${at}: ${file} runs with ${suite.config} in ${suite.workflows.join(" and ")}, not in these jobs, ${nothing} Locally: ${command}.`; }; +/** Where `token` is written in the e2e block, for a message: its line number and that line. */ +const blockLine = (block: string, token: string) => { + const lines = block.split("\n"); + const index = lines.findIndex((line) => line.split(/[\s,]+/).includes(token)); + return index === -1 + ? `\`${token}\` in the e2e block` + : `line ${index + 1} of the e2e block, \`${lines[index]!.trim()}\``; +}; + +/** The command that runs these deployed scenarios on a deployed stage. */ +const verifyCommand = (titles: ReadonlyArray) => + `bun run e2e:deployed --database planetscale --test-name '^(?:${titles.map(escape).join("|")})$'`; + +/** How many single-character edits turn `a` into `b`. */ +const editDistance = (a: string, b: string) => { + let previous = Array.from({ length: b.length + 1 }, (_, index) => index); + for (let i = 1; i <= a.length; i++) { + const current = [i]; + for (let j = 1; j <= b.length; j++) + current[j] = Math.min( + previous[j]! + 1, + current[j - 1]! + 1, + previous[j - 1]! + (a[i - 1] === b[j - 1] ? 0 : 1), + ); + previous = current; + } + return previous[b.length]!; +}; + +/** The three names in `names` nearest to `name`, nearest first. */ +const nearest = (name: string, names: Iterable) => + [...names] + .map((candidate) => ({ candidate, distance: editDistance(name, candidate) })) + .toSorted((a, b) => a.distance - b.distance || a.candidate.localeCompare(b.candidate)) + .slice(0, 3) + .map(({ candidate }) => candidate); + +/** + * The outputs, followed by what a later run on the same head compares to tell whether it would + * test the same thing (e2e/ci-key.ts): `selection`, a key of exactly these outputs, and for a pull + * request `worker-growth`, a key of the size gate's blocks in the description this run read. + */ +const keyed = (lines: string, description: string | undefined) => + [ + lines, + `selection=${selectionKey(lines)}`, + ...(description === undefined ? [] : [`worker-growth=${workerGrowthKey(description)}`]), + "", + ].join("\n"); + /** The trimmed text of each `e2e` block in a description. */ const blocksIn = (body: string) => [...body.matchAll(/^```e2e[ \t]*\r?\n([\s\S]*?)^```/gm)].map((match) => match[1]!.trim()); @@ -1045,8 +1124,10 @@ const requested = (parsed: Requested, retry: string) => */ const checkNamed = ( tokens: ReadonlyArray, + block: string, suites: ReadonlyMap, changedDeployedOnly: ReadonlySet, + retry: string, ) => Effect.gen(function* () { const named = tokens.map((token) => token.replace(/^e2e\/tests\//, "")); @@ -1055,15 +1136,24 @@ const checkNamed = ( ); if (unknown.length > 0) return yield* new SelectionFailed({ - message: `The e2e block names files that are not spec files in e2e/tests/: ${unknown.map((token) => `"${token}"`).join(", ")}. Write each name as it appears in e2e/tests/, such as groups.spec.ts. A new spec file needs its scenarios in e2e/test-plan.ts.`, + message: [ + "The e2e block names files that are not spec files in e2e/tests/:", + ...unknown.map( + (token) => + `- ${blockLine(block, token)}: no spec file has this name. The nearest names are ${nearest(token.replace(/^e2e\/tests\//, ""), [...specFiles, ...suites.keys()]).join(", ")}. Write the name exactly as it appears in e2e/tests/; a new spec file needs its scenarios in e2e/test-plan.ts first.`, + ), + `Correct each of these lines in the description's e2e block. ${retry}`, + ].join("\n"), }); const elsewhere = named.filter((file) => !jobFiles.has(file) && !changedDeployedOnly.has(file)); if (elsewhere.length > 0) return yield* new SelectionFailed({ message: [ "The e2e block names spec files that CI's E2E jobs do not run, so naming them here tests nothing:", - ...elsewhere.map((file) => runsElsewhere(file, suites.get(file))), - "Remove them from the e2e block. Name the spec files these jobs run that exercise the change, or write none.", + ...elsewhere.map((file) => + runsElsewhere(file, suites.get(file), blockLine(block, tokens[named.indexOf(file)]!)), + ), + `Name the spec files these jobs run that exercise the change, or write none. ${retry}`, ].join("\n"), }); return named; @@ -1091,7 +1181,19 @@ NodeRuntime.runMain( const output = yield* Config.String("GITHUB_OUTPUT").pipe(Config.option); const summary = yield* Config.String("GITHUB_STEP_SUMMARY").pipe(Config.option); - const retry = `CI waited ${spoken(wait)} for the description. Once it is fixed, rerun the whole workflow: gh run rerun ${runId}`; + // e2e/check-description.ts runs this program on a draft before anything is pushed: the same + // checks and messages, without the advice that only a CI run can follow. + const local = (yield* Config.String("E2E_LOCAL_CHECK").pipe(Config.withDefault(""))) !== ""; + // Local mode passes a skip with no layer above, which a workflow must never do: GitHub Actions + // sets GITHUB_ACTIONS on every run, and e2e/check-description.ts clears it for this program. + if (local && (yield* Config.String("GITHUB_ACTIONS").pipe(Config.withDefault(""))) === "true") + return yield* new SelectionFailed({ + message: + "E2E_LOCAL_CHECK is set in a GitHub Actions run. It is the mode of e2e/check-description.ts, which passes a skip that has no layer above; a workflow must never set it. Remove it from the workflow, runner or repository variables that set it.", + }); + const retry = local + ? "Edit the description and check it again" + : `CI waited ${spoken(wait)} for the description. Once it is fixed, rerun the whole workflow: gh run rerun ${runId}`; const read = pullRequest === "" ? undefined @@ -1101,9 +1203,17 @@ NodeRuntime.runMain( // A skipped layer runs nothing, so it decides before the suites are read: the layer above // reads them for the combined change. if (block === "skip") { + // Before the pull request and the layer above are open, there is nothing a skip can be + // checked against: say so, and check the layer above, which runs for both. + if (above.length === 0 && local) { + yield* Console.log( + `The e2e block says skip (${blockLine(blocksIn(read!.body)[0]!, "skip")}): this layer runs nothing, and the layer above, an open pull request whose base is ${headRef}, checks the combined change. A skip cannot be checked before both pull requests exist. Check the layer above's description now; once both are open, check this one with --pr.`, + ); + return; + } if (above.length === 0) return yield* new SelectionFailed({ - message: `The e2e block says skip, but no open pull request builds on this branch. Only the lower layers of a stack may skip: open the layer above, or select scenarios instead. ${retry}.`, + message: `The e2e block says skip (${blockLine(blocksIn(read!.body)[0]!, "skip")}), which only a lower layer of a stack may say: this run waited ${spoken(wait)} for the layer above, an open pull request whose base is this branch, ${headRef}, and there is none. Open the layer above with base ${headRef}, then run this workflow again: gh run rerun ${runId}. If this pull request is not a lower layer of a stack, replace skip with the spec files that exercise the change.`, }); const report = [ "## E2E selection", @@ -1113,7 +1223,7 @@ NodeRuntime.runMain( ].join("\n"); yield* Console.log(report); if (Option.isSome(output)) - yield* fs.writeFileString(output.value, "skip=true\n", { flag: "a" }); + yield* fs.writeFileString(output.value, keyed("skip=true", read?.body), { flag: "a" }); if (Option.isSome(summary)) yield* fs.writeFileString(summary.value, `${report}\n`, { flag: "a" }); return; @@ -1218,9 +1328,11 @@ NodeRuntime.runMain( message: [ "These scenarios are new to the deployed suite. Cloud tests on main runs them on a deployed stage after merge, and no job here can run them there:", ...added.toSorted().map((title) => `- ${fileOf.get(title)}: ${title}`), - "Run them on a deployed stage before merging, for example:", - ` bun run e2e:deployed --database planetscale --test-name '^(?:${added.map(escape).join("|")})$'`, - 'Then add a line to the description saying what ran and its result: "Verified on a deployed stage: ".', + "Run them on a deployed stage before merging:", + ` ${verifyCommand(added)}`, + `When that command passes, add this line to the description, on a line of its own, then ${local ? "check it again" : "run this workflow again"}:`, + `Verified on a deployed stage: ${verifyCommand(added)} passed`, + `${retry}.`, ].join("\n"), }); } @@ -1228,8 +1340,10 @@ NodeRuntime.runMain( const named = Array.isArray(block) ? yield* checkNamed( block, + blocksIn(read!.body)[0]!, suites, new Set([...deployedOnlyRun].map((title) => fileOf.get(title)!)), + `${retry}.`, ) : block; const changedElsewhere = changedSpecs.filter( @@ -1290,6 +1404,10 @@ NodeRuntime.runMain( Effect.map((files) => compared.some((file) => files.has(file))), Effect.mapError((error) => new SelectionFailed({ message: error.message })), )); + // The MCP search and session scenarios, from the same comparison (e2e/mcp-search-guard.ts): a + // floor under the block for the code they run through, in their own Cloud jobs. + const searchChanged = + compared === undefined || compared.length >= compareFileLimit || mcpSearchChanged(compared); // The release lanes, from the same comparison: a pull request's changes since main, so the // top of a stack runs them for every layer. A comparison GitHub cannot list in full runs both. const lanes = @@ -1327,6 +1445,8 @@ NodeRuntime.runMain( ...named, ...guarded, ...guardedByFile, + // A collector input changed: the scenarios that exercise it run too. + ...(collector ? collectorScenarios : []), ...changedSpecs.filter((file) => jobFiles.has(file)), ...[...deployedOnlyRun].map((title) => fileOf.get(title)!), ]); @@ -1347,7 +1467,10 @@ NodeRuntime.runMain( .filter( (scenario) => scenario.targets[target].status === "scheduled" && - ((base.test(scenario.title) && (files === undefined || files.has(scenario.file))) || + ((base.test(scenario.title) && + (files === undefined || + files.has(scenario.file) || + (target === "cloud" && searchChanged && mcpSearchTitles.has(scenario.title)))) || extra.has(scenario.title)), ) .map((scenario) => scenario.title); @@ -1480,12 +1603,24 @@ NodeRuntime.runMain( ...(guardedByFile.length === 0 ? [] : ["", `A guarded file differs from main, so its guards run: ${guardedByFile.join(", ")}`]), + ...(!collector || files === undefined + ? [] + : [ + "", + `A collector input changed (e2e/collector-inputs.ts), so the scenarios that exercise it run: ${collectorScenarios.join(", ")}`, + ]), ...(saveChanged && files !== undefined ? [ "", `A file a save runs or reads differs from main (e2e/save-path.ts), so these run in ${saveScenarios.job}: ${saveScenarios.files.join(", ")}`, ] : []), + ...(searchChanged && files !== undefined + ? [ + "", + `A file MCP search or sessions run through differs from main (e2e/mcp-search-guard.ts), so these Cloud scenarios run too: ${[...mcpSearchTitles].join("; ")}`, + ] + : []), ...(savePathSize === undefined ? [] : ["", `The save path (e2e/save-path.ts) has ${savePathSize} files.`]), @@ -1529,7 +1664,8 @@ NodeRuntime.runMain( "", ].join("\n"); yield* Console.log(report); - if (Option.isSome(output)) yield* fs.writeFileString(output.value, `${lines}\n`, { flag: "a" }); + if (Option.isSome(output)) + yield* fs.writeFileString(output.value, keyed(lines, read?.body), { flag: "a" }); if (Option.isSome(summary)) yield* fs.writeFileString(summary.value, `${report}\n`, { flag: "a" }); }).pipe(Effect.provide(NodeServices.layer)), diff --git a/e2e/ci-skip-check.ts b/e2e/ci-skip-check.ts new file mode 100644 index 0000000000..e815d26de9 --- /dev/null +++ b/e2e/ci-skip-check.ts @@ -0,0 +1,401 @@ +/** + * The rule for a stack layer whose e2e block says `skip`, without the reads: what one look at the + * pull requests above says, how long to wait for a better answer, and what to tell the author. + * scripts/ci/worker-baseline.ts `above` reads GitHub and drives it; e2e/tests/worker-sizes.spec.ts + * replays real pushes through the same functions. + * + * A skip layer runs nothing and passes on the strength of the layer above, so that layer's run must + * have been computed with this layer in its chain: its record (the select job's `ci-tested-…` + * artifact) says it was built on this layer's head and compared Worker sizes with the branch this + * layer's stack ends on. The E2E selection always compares with main, so the base commit covers it. + */ + +import { Cause, Effect } from "effect"; + +/** + * Observe or enforce. Observing, the step works out the decision exactly as it would enforce it, + * logs it (`decisionLine`) and passes, as a skip layer always passed: a pull request above exists, + * which the select step has already required. Enforcing, a decision to fail fails the step. + */ +export const enforce = false; + +/** + * The format of the record this workflow writes (RECORD_FORMAT in checks.yml). Format 3 adds + * `tested`, the merge commit the run checked out; format 2 records are still read, by the rule that + * fits them (`judgeRecord`). + */ +export const recordFormat = "3"; +/** The formats this check reads. Any other is never passed: it waits, and then fails. */ +const readFormats = new Set(["2", "3"]); + +/** + * How long to wait for a better answer, in seconds from the start of the step, by what was seen: + * - `stale`: the run above recorded another base commit or baseline. A push to it, or a re-run, + * may still come; only the deadline fails. + * - `noRun`: no CI run on the current head above, or none that recorded. + * - `selecting`: a run on the head above has not finished choosing what it tests. Runners can start + * the layer above's select 9 to 11 minutes after this one's (15 and 16 Nov). + * - `chained`: that run is itself a skip layer waiting on the layer above it. Its record comes a + * poll after its own check passes, so each layer below needs a little longer than the one above: + * every layer of a stack starts together, the layer second from the top waits up to `selecting` + * for the top, and each layer under it notices its upper's record within one poll (at most 60 + * s), so `chained` covers five layers under the one waiting on the top. + * + * select's timeout (checks.yml) covers the description wait (3 minutes) and `chained`, the longest. + */ +export const bounds = { stale: 240, noRun: 240, selecting: 900, chained: 1200 } as const; +export type Bound = keyof typeof bounds; + +/** Seconds before each next look: no more often than every 20 seconds, then backing off to 60. */ +export const pollAfter = (polls: number) => [20, 30, 45][polls] ?? 60; + +/** The fields of a record the check reads. */ +export interface RecordFields { + readonly format?: string | undefined; + readonly base?: string | undefined; + readonly baseline?: string | undefined; + /** A skip layer's own check: `pass` or `fail`; `none` for a layer that does not skip. */ + readonly above?: string | undefined; + /** Format 3: the merge commit the run checked out and tested (`git rev-parse HEAD`). */ + readonly tested?: string | undefined; +} + +export const parseRecord = (text: string): RecordFields => { + const field = (key: string) => text.match(new RegExp(`^${key} (\\S+)$`, "m"))?.[1]; + return { + format: field("format"), + base: field("base"), + baseline: field("baseline"), + above: field("above"), + tested: field("tested"), + }; +}; + +/** One CI run of this repository on the current head of a pull request above. */ +export interface RunSeen { + readonly id: number; + readonly url: string; + /** The newest record it uploaded, if any. */ + readonly record?: RecordFields | undefined; + /** Its select job has finished (or the run has). */ + readonly selectDone: boolean; + /** Its select job is in its own skip check, waiting on the layer above it. */ + readonly waitingAbove: boolean; + /** + * Format 3: whether the lower layer's head is an ancestor of the commit the run tested (the + * reader asks GitHub's compare API); undefined when the record names no tested commit or the + * answer could not be read. + */ + readonly contains?: boolean | undefined; +} + +/** One open pull request of this repository whose base is this layer's branch. */ +export interface UpperSeen { + readonly pull: number; + readonly head: string; + /** Its CI runs on `head` that were not cancelled, newest first. */ + readonly runs: ReadonlyArray; +} + +/** What one look saw: the pull requests above, or why they could not be read. */ +export type Observation = + | { readonly _tag: "Read"; readonly uppers: ReadonlyArray } + | { readonly _tag: "Unread"; readonly why: string }; + +export interface Lower { + readonly ref: string; + readonly head: string; + /** The branch this layer's stack ends on, from its own baseline. */ + readonly bottom: string; + readonly repository: string; + readonly runId: string; +} + +/** + * The rule that passed a run: `old`, a record from before this check (a pull request above + * exists); `base`, the run's event named this layer's head as its base (format 2 and 3); `tested`, + * the merge commit the run tested contains this layer's head (format 3). + */ +export type Rule = "old" | "base" | "tested"; + +/** What one look says. */ +export type Verdict = + | { + readonly _tag: "Pass"; + readonly pull: number; + readonly run: string; + /** The record above is in the format from before this check: judged by the old rule. */ + readonly oldFormat: boolean; + readonly rule: Rule; + } + | { + readonly _tag: "Wait"; + readonly bound: Bound; + readonly why: string; + readonly remedy?: string | undefined; + }; + +const short = (sha: string) => sha.slice(0, 9); + +/** What one run's record says: it fits, or why not. */ +const judgeRecord = ( + upper: UpperSeen, + run: RunSeen, + record: RecordFields, + lower: Lower, +): Verdict => { + const rerun = `run it again: gh run rerun ${run.id} --repo ${lower.repository}`; + // TRANSITION. An upper's run uses the workflow of its merge into this branch, so until a stack + // is rebased onto a main that writes `format`, its records are in main's format from before + // this check: neither `format` nor `baseline` (and often no `base`). Such a run compared + // Worker sizes with main, so it cannot have laundered a branch's growth; it is judged by the + // old rule, which passes, and leaves only the stale-upper weakness the old rule always had. (A + // record with a baseline and no format came only from this check's own pull requests before + // `format` was added; it is judged by the fields it has.) + // Before deleting this branch and `oldFormat`: this command, from any checkout, must list no + // pull request, that is, no open upper above a skip layer would still write main's old format: + // R=UsefulSoftwareCo/executor-next; gh pr list -R $R --state open -L 1000 --json number,baseRefName,headRefName,isCrossRepository,body -q '[.[]|select(.isCrossRepository|not)] as $p | ($p|map(select(.body|test("```e2e\\s+skip\\s+```"))|.headRefName)) as $s | $p[]|select(.baseRefName as $b|$s|index($b))|.number' | while read n; do gh api "repos/$R/contents/.github/workflows/checks.yml?ref=refs/pull/$n/merge" -q .content | base64 -d | grep -q RECORD_FORMAT || echo "#$n"; done + if (record.format === undefined && record.baseline === undefined) + return { _tag: "Pass", pull: upper.pull, run: run.url, oldFormat: true, rule: "old" }; + if (!readFormats.has(record.format ?? "2")) + return { + _tag: "Wait", + bound: "stale", + why: `#${upper.pull}'s latest run on its head, ${run.url}, wrote a record in format ${record.format}, which this check does not read`, + remedy: `bring this layer up to date with main and push it`, + }; + // A skip layer's own check that did not pass: under enforcement it would have written no record. + if (record.above !== undefined && record.above !== "pass" && record.above !== "none") + return { + _tag: "Wait", + bound: "stale", + why: `#${upper.pull} is itself a skip layer, and its own check of the layer above it fails (${run.url})`, + remedy: `fix #${upper.pull} as its run says`, + }; + // Built on this layer's head: its event named it as the base, or (format 3) the commit the run + // tested contains it. The event's base alone is not enough: a stack push can send an upper's + // `synchronize` event twice, one carrying the lower layer's previous head as its base, while the + // merge commit GitHub built for it already holds this layer's head (#4718, 16 Nov). A re-run + // keeps its event's base, but tests the same merge commit. + const builtOn: Rule | undefined = + record.base === lower.head + ? "base" + : record.tested !== undefined && run.contains === true + ? "tested" + : undefined; + if (builtOn === undefined && record.tested !== undefined && run.contains === undefined) + return { + _tag: "Wait", + bound: "stale", + why: `#${upper.pull}'s latest run on its head, ${run.url}, tested ${short(record.tested)}, and whether that contains this layer's head ${short(lower.head)} could not be read`, + }; + if (builtOn === undefined) + return { + _tag: "Wait", + bound: "stale", + why: `#${upper.pull}'s latest run on its head, ${run.url}, was built on ${record.base === undefined ? "a base it did not record" : short(record.base)}${record.tested === undefined ? "" : ` and tested ${short(record.tested)}, which does not contain`}${record.tested === undefined ? ", not" : ""} this layer's head ${short(lower.head)}`, + remedy: `bring #${upper.pull} up to date with this layer and push it (gh stack rebase, then gh stack push), which runs it on this layer's head`, + }; + const baseline = record.baseline?.replace(/@[^@]*$/, ""); + if (baseline !== lower.bottom) + return { + _tag: "Wait", + bound: "stale", + why: `#${upper.pull}'s latest run on its head, ${run.url}, compared its Worker sizes with ${baseline ?? "a baseline it did not record"}, not ${lower.bottom}, the base this stack now ends on`, + remedy: `${rerun} (a re-run reads the stack afresh)`, + }; + return { _tag: "Pass", pull: upper.pull, run: run.url, oldFormat: false, rule: builtOn }; +}; + +/** What one pull request above says, from its runs on its current head. */ +const judgeUpper = (upper: UpperSeen, lower: Lower): Verdict => { + const rerun = (run: RunSeen) => `run it again: gh run rerun ${run.id} --repo ${lower.repository}`; + // Any run on the current head whose record fits passes: it tested this layer's head against the + // right bottom, whatever another run on the same head recorded. A stack push can start two runs + // on the upper's head in the same second, one of them with the lower layer's previous head as + // its base (#4575, 16 Nov). + const judged = upper.runs.flatMap((run) => + run.record === undefined ? [] : [judgeRecord(upper, run, run.record, lower)], + ); + const fits = + judged.find((verdict) => verdict._tag === "Pass" && !verdict.oldFormat) ?? + judged.find((verdict) => verdict._tag === "Pass"); + if (fits !== undefined) return fits; + // While any run on the head is still choosing (a duplicate waiting its turn behind the other, + // a re-run), it may yet fit: wait for it, whatever the others recorded (#4575, 16 Nov: the + // fitting run was the queued duplicate, and the other had the lower layer's previous head). + const selecting = upper.runs.filter((run) => run.record === undefined && !run.selectDone); + if (selecting.length > 0) + return { + _tag: "Wait", + bound: selecting.some((run) => run.waitingAbove) ? "chained" : "selecting", + why: `#${upper.pull}'s run on its head, ${selecting[0]!.url}, has not chosen what it tests yet`, + }; + // Otherwise the newest run says why not. + const newest = upper.runs[0]; + if (newest?.record !== undefined) return judgeRecord(upper, newest, newest.record, lower); + if (newest !== undefined) + return { + _tag: "Wait", + bound: "noRun", + why: `#${upper.pull}'s latest run on its head, ${newest.url}, recorded nothing (its select failed)`, + remedy: `fix #${upper.pull}'s description or ${rerun(newest)}`, + }; + return { + _tag: "Wait", + bound: "noRun", + why: `#${upper.pull} has no CI run on its head ${short(upper.head)}`, + remedy: `push to #${upper.pull}, or run its latest CI run again`, + }; +}; + +/** + * What one look says for all the pull requests above: any one computed with this layer passes. + * Otherwise the wait is the longest any of them asks for. A look that failed keeps the previous + * look's bound: not reading is not an answer. + */ +export const judge = (observation: Observation, lower: Lower, previous?: Verdict): Verdict => { + if (observation._tag === "Unread") + return { + _tag: "Wait", + bound: previous?._tag === "Wait" ? previous.bound : "noRun", + why: `the pull requests above could not be read (${observation.why})`, + }; + if (observation.uppers.length === 0) + return { + _tag: "Wait", + bound: "noRun", + why: `no open pull request in this repository has ${lower.ref} as its base`, + remedy: `open the layer above with base ${lower.ref}`, + }; + const verdicts = observation.uppers.map((upper) => judgeUpper(upper, lower)); + const passed = + verdicts.find((verdict) => verdict._tag === "Pass" && !verdict.oldFormat) ?? + verdicts.find((verdict) => verdict._tag === "Pass"); + if (passed !== undefined) return passed; + const waits = verdicts.filter((verdict) => verdict._tag === "Wait"); + const longest = waits.toSorted((a, b) => bounds[b.bound] - bounds[a.bound])[0]!; + return { + _tag: "Wait", + bound: longest.bound, + why: waits.map((verdict) => verdict.why).join("; "), + remedy: + waits + .flatMap((verdict) => (verdict.remedy === undefined ? [] : [verdict.remedy])) + .join(", or ") || undefined, + }; +}; + +/** The decision, once a look passes or its wait is over. */ +export interface Decision { + readonly decision: "pass" | "fail"; + readonly oldFormat: boolean; + /** The rule that decided a pass; `none` for a fail. */ + readonly rule: Rule | "none"; + readonly reason: string; + /** What the author is told. */ + readonly message: string; + readonly waited: number; +} + +/** After a look `elapsed` seconds into the step: the decision, or how long until the next look. */ +export const next = ( + verdict: Verdict, + lower: Lower, + elapsed: number, + polls: number, +): + | { readonly _tag: "Done"; readonly decision: Decision } + | { readonly _tag: "Poll"; readonly after: number } => { + const waited = Math.round(elapsed); + if (verdict._tag === "Pass") + return { + _tag: "Done", + decision: { + decision: "pass", + oldFormat: verdict.oldFormat, + rule: verdict.rule, + waited, + reason: verdict.oldFormat + ? `#${verdict.pull}'s run ${verdict.run} wrote its record in the format from before this check, so the old rule applies: a pull request above exists` + : verdict.rule === "tested" + ? `#${verdict.pull}'s run ${verdict.run} tested a merge commit that contains ${short(lower.head)}, and compared its Worker sizes with ${lower.bottom}` + : `#${verdict.pull}'s run ${verdict.run} was built on ${short(lower.head)} and compared its Worker sizes with ${lower.bottom}`, + message: verdict.oldFormat + ? `The layer above runs a workflow from before the skip check. Rebasing the stack onto main turns the check on.` + : `The layer above was computed with this layer in its chain.`, + }, + }; + if (elapsed < bounds[verdict.bound]) + return { _tag: "Poll", after: Math.min(pollAfter(polls), bounds[verdict.bound] - elapsed) }; + const own = `gh run rerun ${lower.runId} --repo ${lower.repository}`; + return { + _tag: "Done", + decision: { + decision: "fail", + oldFormat: false, + rule: "none", + waited, + reason: verdict.why, + message: `The e2e block says skip: this layer passes only when the layer above has a run on its current head computed with this layer in its chain, built on ${short(lower.head)} and comparing Worker sizes with ${lower.bottom}. After ${waited} s, ${verdict.why}. ${verdict.remedy === undefined ? "Once it has" : `To fix it, ${verdict.remedy}; once that run has started`}, run this one again: ${own}`, + }, + }; +}; + +/** What the step came to: a decision, or a defect that kept it from deciding. */ +export type Outcome = + | { readonly _tag: "Decided"; readonly decision: Decision } + | { readonly _tag: "Error"; readonly why: string }; + +/** + * The check, made unable to fail the step while observing: any failure or defect inside it (a + * shape `judge` or `next` did not expect, a missing variable, a write that failed) becomes the + * `error` outcome. Enforcing, it fails the step as any defect would. + */ +export const guarded = ( + check: Effect.Effect, + enforcing: boolean, +): Effect.Effect => { + const decided = Effect.map(check, (decision): Outcome => ({ _tag: "Decided", decision })); + return enforcing + ? decided + : decided.pipe( + Effect.catchCause((cause) => + Effect.succeed({ + _tag: "Error", + why: String(Cause.squash(cause)).split("\n")[0]!, + }), + ), + ); +}; + +/** The one log line a day of decisions is counted from. */ +export const decisionLine = (outcome: Outcome, enforcing = enforce) => + `skip-check ${JSON.stringify( + outcome._tag === "Error" + ? { mode: enforcing ? "enforce" : "observe", decision: "error", reason: outcome.why } + : { + mode: enforcing ? "enforce" : "observe", + decision: `${enforcing ? "" : "would "}${outcome.decision.decision}`, + oldFormat: outcome.decision.oldFormat, + rule: outcome.decision.rule, + waited: outcome.decision.waited, + reason: outcome.decision.reason, + message: outcome.decision.message, + }, + )}`; + +/** The summary sentence for an outcome. */ +export const summarySentence = (outcome: Outcome, enforcing = enforce) => { + if (outcome._tag === "Error") + return `The skip check could not decide: ${outcome.why}.${enforcing ? "" : " It passes while observing."}`; + const { decision } = outcome; + const would = enforcing ? "" : "would "; + return decision.decision === "pass" + ? `The skip check ${would}pass${enforcing ? "es" : ""}: ${decision.reason}.` + : `The skip check ${would}fail${enforcing ? "s" : ""}: ${decision.reason}. The author ${enforcing ? "is" : "would be"} told: ${decision.message}`; +}; + +/** Whether the step fails: only a decision to fail, or a defect, and only when enforcing. */ +export const fails = (outcome: Outcome, enforcing: boolean) => + enforcing && (outcome._tag === "Error" || outcome.decision.decision === "fail"); diff --git a/e2e/cleanup-deployed.ts b/e2e/cleanup-deployed.ts index 620a3b25d2..4059deda49 100644 --- a/e2e/cleanup-deployed.ts +++ b/e2e/cleanup-deployed.ts @@ -1,4 +1,8 @@ /** A separate CI step recovers cleanup after GitHub kills a cancelled test process. */ +// First: the harness's outbound connect time-out (support/connect-timeout.ts). +import "./support/connect-timeout.ts"; +// Second: each stage request on a connection of its own (support/stage-connections.ts). +import "./support/stage-connections.ts"; import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { Console, Effect, FileSystem, Path, Schema } from "effect"; diff --git a/e2e/cli.ts b/e2e/cli.ts index 97e5b14f35..894da13ad4 100644 --- a/e2e/cli.ts +++ b/e2e/cli.ts @@ -1,4 +1,8 @@ /** Interactive CLI over the same SDK used by committed scenarios. */ +// First: the harness's outbound connect time-out (support/connect-timeout.ts). +import "./support/connect-timeout.ts"; +// Second: each stage request on a connection of its own (support/stage-connections.ts). +import "./support/stage-connections.ts"; import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { Console, Effect, FileSystem, Layer, Schedule, Schema } from "effect"; diff --git a/e2e/cloud-postgres-volumes.config.ts b/e2e/cloud-postgres-volumes.config.ts new file mode 100644 index 0000000000..4f02217e30 --- /dev/null +++ b/e2e/cloud-postgres-volumes.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from "vitest/config"; + +/** Runs the rule that decides which Cloud test database volumes a run may sweep. */ +export default defineConfig({ + test: { + include: ["e2e/tests/cloud-postgres-volumes.spec.ts"], + testTimeout: 30_000, + }, +}); diff --git a/e2e/collector-inputs.ts b/e2e/collector-inputs.ts index a0dc9ace64..f10e2bd249 100644 --- a/e2e/collector-inputs.ts +++ b/e2e/collector-inputs.ts @@ -14,6 +14,28 @@ export const collectorInputs: ReadonlyArray = [ ".github/actions/rustup/", ]; +/** + * The spec files that exercise the collector through the product and the harness: a pull request + * that changes a collector input runs them, whatever its `e2e` block names (e2e/ci-selection.ts). + * The local server, the self-host image and the e2e harness all start the collector these inputs + * build, so a crate change reaches every scenario that reads telemetry; these are the ones whose + * subject is the collector itself. + */ +export const collectorScenarios: ReadonlyArray = [ + // The product's own spans reaching the image's collector, through the host. + "hosted.spec.ts", + // Retention at the store's bound, with the collector's own figures. + "telemetry-store-bound.spec.ts", + // Exports refused and resent through a scenario's own collector. + "telemetry-backpressure.spec.ts", + // What an idle server exports, read back from a scenario's own collector. + "idle-poll-traces.spec.ts", + // The viewer the local collector serves. + "telemetry-viewer.spec.ts", + // A trace answered in part, read through the suite's own span reader. + "telemetry-partial-trace.spec.ts", +]; + /** Whether any of `files`, paths from the repository root, is a collector input. */ export const collectorChanged = (files: ReadonlyArray) => files.some((file) => diff --git a/e2e/conflict-markers.config.ts b/e2e/conflict-markers.config.ts new file mode 100644 index 0000000000..bab41b4203 --- /dev/null +++ b/e2e/conflict-markers.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from "vitest/config"; + +/** Runs the leftover conflict marker check on this checkout and on a planted conflict. */ +export default defineConfig({ + test: { include: ["e2e/tests/conflict-markers.spec.ts"], testTimeout: 60_000 }, +}); diff --git a/e2e/create-emulators.ts b/e2e/create-emulators.ts index c5e00b5272..7d0818edc9 100644 --- a/e2e/create-emulators.ts +++ b/e2e/create-emulators.ts @@ -1,3 +1,7 @@ +// First: the harness's outbound connect time-out (support/connect-timeout.ts). +import "./support/connect-timeout.ts"; +// Second: each stage request on a connection of its own (support/stage-connections.ts). +import "./support/stage-connections.ts"; import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { Console, Effect, FileSystem, Layer, Path, Redacted, Schema } from "effect"; diff --git a/e2e/deployed.ts b/e2e/deployed.ts index 69cfe791d4..d9adf960fd 100644 --- a/e2e/deployed.ts +++ b/e2e/deployed.ts @@ -1,4 +1,8 @@ /** CLI adapter: one deployment is shared by every selected Cloud scenario. */ +// First: the harness's outbound connect time-out (support/connect-timeout.ts). +import "./support/connect-timeout.ts"; +// Second: each stage request on a connection of its own (support/stage-connections.ts). +import "./support/stage-connections.ts"; import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { Effect, Layer } from "effect"; diff --git a/e2e/file-guards.ts b/e2e/file-guards.ts new file mode 100644 index 0000000000..280bd26ba7 --- /dev/null +++ b/e2e/file-guards.ts @@ -0,0 +1,45 @@ +/** + * Scenarios that assert a contract one file states in full, by that file (e2e/ci-selection.ts). + * When a guarded file's contents differ from main's, its spec files run, whatever the pull request's + * `e2e` block names. The comparison is with main, not with the layer below, so the top of a stack + * runs them when any lower layer changes the file: a long stack whose base edits one of these runs + * them on every tested layer, which is the guard doing its job, not a cost to optimise away. + * + * Add a file here when a scenario reads its text back, with the spec files that do: the cases in + * e2e/tests/ci-selection.spec.ts check that each file exists and each spec file runs on a pull + * request. + */ +export const fileGuards: Readonly>> = { + // app-package.spec.ts reads back every protocol the host supports when it rejects an unsupported + // framework, so a new protocol changes its expected message. + "packages/sdk/src/implementation/app-protocols.ts": [ + "app-package.spec.ts", + "app-older-protocols.spec.ts", + ], + // The Executor app's skill texts, which scenarios read back through the skills tool and the CLI. + // The entry skill names the deeper skills to read. + "packages/app-templates/executor/skills/executor/SKILL.md": [ + "framework-discovery.spec.ts", + "cli-tool-commands.spec.ts", + ], + "packages/app-templates/executor/skills/executor/feedback.md": ["skill-read-telemetry.spec.ts"], + // Where images and audio are explained. + "packages/app-templates/executor/skills/code-mode/SKILL.md": ["framework-discovery.spec.ts"], + // The authoring guide's heading and its table of topics. + "packages/app-templates/executor/skills/app-authoring/SKILL.md": [ + "framework-discovery.spec.ts", + "cli-tool-commands.spec.ts", + "mcp-server.spec.ts", + "local-skills.spec.ts", + ], + "packages/app-templates/executor/skills/app-authoring/ui.md": [ + "framework-discovery.spec.ts", + "cli-tool-commands.spec.ts", + "local-skills.spec.ts", + ], + // The MCP server's instructions, generated from the entry skill. + "packages/app-templates/src/implementation/executor-intro.gen.ts": [ + "cli-tool-commands.spec.ts", + "mcp-server.spec.ts", + ], +}; diff --git a/e2e/lost-connections-lifecycle.config.ts b/e2e/lost-connections-lifecycle.config.ts new file mode 100644 index 0000000000..8d2c014042 --- /dev/null +++ b/e2e/lost-connections-lifecycle.config.ts @@ -0,0 +1,22 @@ +import { defineConfig } from "vitest/config"; + +/** + * The lost-request records written by the real scenario lifecycle (`e2e/setup.ts`), against a + * stand-in stage the global setup starts: one inside a scenario, one in a layer's set-up outside + * any. The global teardown fails the run when either record was not written. + */ +export default defineConfig({ + test: { + include: ["e2e/tests/lost-connections-lifecycle.spec.ts"], + setupFiles: ["e2e/setup.ts"], + globalSetup: ["e2e/lost-connections-lifecycle.global.ts"], + fileParallelism: false, + testTimeout: 30_000, + hookTimeout: 30_000, + env: { + EXECUTOR_E2E_RUN: ".local/lost-connections-lifecycle/run", + EXECUTOR_E2E_API_KEY: "synthetic-lost-connections-key", + E2E_RECORDING_PACE_MS: "0", + }, + }, +}); diff --git a/e2e/lost-connections-lifecycle.global.ts b/e2e/lost-connections-lifecycle.global.ts new file mode 100644 index 0000000000..a1675e4633 --- /dev/null +++ b/e2e/lost-connections-lifecycle.global.ts @@ -0,0 +1,85 @@ +/** + * Global set-up for `lost-connections-lifecycle.config.ts`: a stand-in stage that resets any + * connection whose request is for `/lose` and answers every other, and a run directory whose + * `run.json` names it as a Cloud target. The teardown fails the run unless the lifecycle saved the scenario's + * lost request and the setup file saved the one made outside any scenario. + */ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { Effect, FileSystem } from "effect"; +import { createServer, type Socket } from "node:net"; +import { readLostRequests } from "./lost-requests.ts"; + +const run = ".local/lost-connections-lifecycle/run"; + +class RecordsMissing extends Error {} + +// Vitest's global set-up is a Promise API. +/* oxlint-disable executor/no-manual-effect-runtime-in-tests */ +export default function setup() { + const sockets = new Set(); + const server = createServer((socket) => { + sockets.add(socket); + socket.on("data", (chunk) => { + const text = chunk.toString("latin1"); + if (!text.includes("\r\n\r\n")) return; + if (/^[A-Z]+ \/lose /u.test(text)) socket.resetAndDestroy(); + else + socket.write( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 2\r\nconnection: keep-alive\r\n\r\n{}", + ); + }); + socket.on("close", () => sockets.delete(socket)); + socket.on("error", () => undefined); + }); + const started = Effect.callback((resume) => { + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + resume(Effect.succeed(typeof address === "object" && address !== null ? address.port : 0)); + }); + }); + return Effect.runPromise( + Effect.gen(function* () { + const port = yield* started; + const fs = yield* FileSystem.FileSystem; + yield* fs.remove(run, { recursive: true, force: true }); + yield* fs.makeDirectory(run, { recursive: true }); + yield* fs.writeFileString( + `${run}/run.json`, + JSON.stringify({ + target: "cloud", + origin: `http://127.0.0.1:${port}`, + mode: "attached", + runtime: "stand-in", + commit: "0000000", + dirty: false, + startedAt: new Date(0).toISOString(), + interactive: false, + diagnostics: "", + }), + ); + }).pipe(Effect.provide(NodeServices.layer)), + ).then( + () => () => + Effect.runPromise( + Effect.gen(function* () { + for (const socket of sockets) socket.destroy(); + server.close(); + const records = yield* readLostRequests([run]); + const inScenario = records.some( + (record) => + record.scenario?.endsWith("is saved by the scenario lifecycle") === true && + record.outcome === "lost: new connection", + ); + const outside = records.some( + (record) => + record.scenario === undefined && record.file === "lost-connections-lifecycle.spec.ts", + ); + if (!inScenario || !outside) + throw new RecordsMissing( + `The lifecycle did not save the lost requests: in a scenario ${inScenario}, outside any scenario ${outside}; ${records.length} records read.`, + ); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); +} +/* oxlint-enable executor/no-manual-effect-runtime-in-tests */ diff --git a/e2e/lost-connections.config.ts b/e2e/lost-connections.config.ts new file mode 100644 index 0000000000..b75a56c49d --- /dev/null +++ b/e2e/lost-connections.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from "vitest/config"; + +/** A stage request lost on a dropped kept-alive connection, against a local stand-in stage. */ +export default defineConfig({ + test: { include: ["e2e/tests/lost-connections.spec.ts"], testTimeout: 30_000 }, +}); diff --git a/e2e/lost-request-summary.ts b/e2e/lost-request-summary.ts new file mode 100644 index 0000000000..0890a56415 --- /dev/null +++ b/e2e/lost-request-summary.ts @@ -0,0 +1,54 @@ +/** + * Adds a job's count of stage requests lost in transport to its summary, zero included: + * `node e2e/lost-request-summary.ts ...`. The records come from + * `support/lost-connections.ts`. Only a request of which nothing was written is sent again; a job + * that passed with such requests says so, with their count and waits. When the records cannot be + * read the summary says so in words a reader cannot take for a zero. + */ +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { Config, Console, Effect, FileSystem, Option } from "effect"; +import { + lostRequestLines, + lostRequestsUncounted, + readLostRequests, + repeatTotals, +} from "./lost-requests.ts"; + +NodeRuntime.runMain( + Effect.gen(function* () { + const [status, ...directories] = process.argv.slice(2); + if (status === undefined || directories.length === 0) + return yield* Effect.die( + "Usage: node e2e/lost-request-summary.ts ...", + ); + // A summary never turns a job red: a record it cannot read is reported, not failed on. + const read = yield* readLostRequests(directories).pipe(Effect.option); + const totals = Option.isNone(read) ? undefined : repeatTotals(read.value); + const passedWithRepeats = + status === "success" && totals !== undefined && totals.requests > 0 + ? ` This job passed only after sending ${totals.requests} stage ${totals.requests === 1 ? "request" : "requests"} again (${totals.repeats} ${totals.repeats === 1 ? "repeat" : "repeats"}, ${(totals.waitedMs / 1000).toFixed(1)} s of waiting in all): nothing of them had been written to a connection.` + : ""; + const report = `## Lost stage requests\n\n${Option.isNone(read) ? lostRequestsUncounted : lostRequestLines(read.value)}${passedWithRepeats}\n`; + yield* Console.log(report); + // Where people look: an annotation on the run, not only a line in the summary. + if (passedWithRepeats !== "") + yield* Console.log(`::warning title=Stage requests sent again::${passedWithRepeats.trim()}`); + if (Option.isNone(read)) + yield* Console.log( + `::warning title=Lost stage requests not counted::${lostRequestsUncounted}`, + ); + else { + const lost = read.value.length - (totals?.requests ?? 0); + if (lost > 0) + yield* Console.log( + `::warning title=Lost stage requests::${lost} stage ${lost === 1 ? "request was" : "requests were"} lost in transport.`, + ); + } + const summary = yield* Config.String("GITHUB_STEP_SUMMARY").pipe(Config.option); + if (Option.isSome(summary)) { + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString(summary.value, `${report}\n`, { flag: "a" }); + } + }).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/e2e/lost-requests.ts b/e2e/lost-requests.ts new file mode 100644 index 0000000000..dfcc544cd8 --- /dev/null +++ b/e2e/lost-requests.ts @@ -0,0 +1,151 @@ +/** + * Stage requests lost in transport (`support/lost-connections.ts`): before any response, or after + * one began and before it was whole. Saved per scenario in + * the run's `report/lost-requests/` and counted for a job's summary, zero included: where each was + * lost (a new connection, a reused one, after a response began), with its time. Only a request of + * which nothing was written to any connection is sent again; one answered so is kept as "sent + * again: answered" with its repeats, and counted apart from the lost. + */ +import { Effect, FileSystem, Path, Schema } from "effect"; + +/** A path with its ids shortened to `:id`: a run of 16 or more id characters. */ +export const shortPath = (path: string) => path.replace(/[A-Za-z0-9_-]{16,}/gu, ":id"); + +/** Where a run keeps its records, under the run's directory. */ +export const lostRequestDirectory = "report/lost-requests"; + +export const LostRequestRecord = Schema.Struct({ + at: Schema.String, + scenario: Schema.optional(Schema.String), + file: Schema.optional(Schema.String), + method: Schema.String, + path: Schema.String, + code: Schema.optional(Schema.String), + elapsedMs: Schema.Number, + sent: Schema.Struct({ + connection: Schema.optional(Schema.Literals(["new", "reused"])), + earlierRequests: Schema.optional(Schema.Number), + connectionAgeMs: Schema.optional(Schema.Number), + idleMs: Schema.optional(Schema.Number), + written: Schema.Literals(["nothing", "headers", "whole request"]), + responseStarted: Schema.Boolean, + }), + outcome: Schema.String, + /** The failed attempts of a request sent again because nothing of it was written. */ + repeats: Schema.optional( + Schema.Array( + Schema.Struct({ + at: Schema.String, + code: Schema.optional(Schema.String), + elapsedMs: Schema.Number, + }), + ), + ), + stopped: Schema.optional(Schema.String), +}); +export type LostRequestRecord = typeof LostRequestRecord.Type; + +const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(LostRequestRecord))); + +/** + * Save lost requests in a run directory, under a name of the saver's: a scenario's id, a test + * file's records made outside any scenario, or the suite's preparation. None writes nothing. + */ +export const saveLostRequests = ( + run: string, + name: string, + records: ReadonlyArray, +) => + Effect.gen(function* () { + if (records.length === 0) return; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = path.join(run, lostRequestDirectory); + yield* fs.makeDirectory(directory, { recursive: true }); + yield* fs.writeFileString( + path.join(directory, `${name}.json`), + JSON.stringify(records, null, 2), + { mode: 0o600 }, + ); + }); + +/** Every record under the directories, at any depth, oldest first. A file that does not decode fails. */ +export const readLostRequests = (directories: ReadonlyArray) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const files: Array = []; + for (const directory of directories) + if (yield* fs.exists(directory)) + for (const file of yield* fs.readDirectory(directory, { recursive: true })) + if ( + path.basename(path.dirname(file)) === path.basename(lostRequestDirectory) && + file.endsWith(".json") + ) + files.push(path.join(directory, file)); + const records = yield* Effect.forEach(files, (file) => + fs.readFileString(file).pipe(Effect.flatMap(decode)), + ); + return records.flat().toSorted((a, b) => a.at.localeCompare(b.at)); + }); + +/** The outcome of a request answered after being sent again. */ +export const answeredAfterRepeats = "sent again: answered"; + +/** Requests answered only after being sent again: how many, their repeats, and their waits in all. */ +export const repeatTotals = (records: ReadonlyArray) => { + const answered = records.filter((record) => record.outcome === answeredAfterRepeats); + return { + requests: answered.length, + // Each request's first attempt is not a repeat. + repeats: answered.reduce((sum, record) => sum + (record.repeats?.length ?? 0), 0), + waitedMs: answered.reduce((sum, record) => sum + record.elapsedMs, 0), + }; +}; + +const seconds = (milliseconds: number) => `${(milliseconds / 1000).toFixed(1)} s`; + +/** A record's own repeats: `3 attempts over 3.2 s, not sent again: 5 repeats used`. */ +const repeatText = (record: LostRequestRecord) => + record.repeats === undefined + ? record.stopped === undefined + ? "" + : `, not sent again: ${record.stopped}` + : `, ${record.outcome === answeredAfterRepeats ? record.repeats.length + 1 : record.repeats.length} attempts over ${seconds(record.elapsedMs)}${record.stopped === undefined ? "" : `, not sent again: ${record.stopped}`}`; + +/** + * One summary paragraph: the lost requests, each outcome's count, those outside any scenario, the + * requests answered only after being sent again, and when each happened. + */ +export const lostRequestLines = (records: ReadonlyArray) => { + const lost = records.filter((record) => record.outcome !== answeredAfterRepeats); + const totals = repeatTotals(records); + const repeated = + totals.requests === 0 + ? [] + : [ + `Stage requests sent again because nothing of them was written to a connection, then answered: ${totals.requests}, after ${totals.repeats} ${totals.repeats === 1 ? "repeat" : "repeats"} and ${seconds(totals.waitedMs)} of waiting in all.`, + ]; + if (records.length === 0) return "Stage requests lost in transport: 0."; + const outcomes = new Map(); + for (const record of lost) outcomes.set(record.outcome, (outcomes.get(record.outcome) ?? 0) + 1); + const outside = lost.filter((record) => record.scenario === undefined).length; + return [ + lost.length === 0 + ? "Stage requests lost in transport: 0." + : `Stage requests lost in transport: ${lost.length} (${[...outcomes] + .map(([outcome, count]) => `${count} ${outcome}`) + .join( + ", ", + )})${outside === 0 ? "" : `; ${outside} outside any scenario (preparation, set-up or teardown)`}.`, + ...repeated, + ...records.map( + (record) => + `- ${record.at.slice(11, 19)} ${record.method} ${shortPath(record.path)} (${record.sent.connection ?? "unwritten"}${record.sent.earlierRequests === undefined || record.sent.earlierRequests === 0 ? "" : `, ${record.sent.earlierRequests} earlier requests`}${record.sent.idleMs === undefined ? "" : `, idle ${(record.sent.idleMs / 1000).toFixed(1)} s`}; ${record.outcome}${repeatText(record)})`, + ), + ].join("\n"); +}; + +/** What a job summary says when the records could not be read: never a count, never zero. */ +export const lostRequestsUncounted = + "Stage requests lost in transport: NOT COUNTED. The records could not be read, so this job's count is unknown, not zero."; diff --git a/e2e/mcp-search-guard.ts b/e2e/mcp-search-guard.ts new file mode 100644 index 0000000000..55548c5ec7 --- /dev/null +++ b/e2e/mcp-search-guard.ts @@ -0,0 +1,70 @@ +/** + * A floor under a pull request's `e2e` block for the code MCP search and MCP sessions run through + * (e2e/ci-selection.ts). Cloud's Worker is not profiled (e2e/coverage.ts), so no coverage says which + * Cloud scenarios a change to this code reaches, and the Cloud scenarios that exercise search and + * sessions under locks and load live in spec files whose names do not say so: + * cloud-database-placement.spec.ts and mcp-catalog-scale.spec.ts. When any of `mcpSearchPaths` + * differs from main, every scenario in `mcpSearchScenarios` runs in its own Cloud job, besides what + * the block names. It removes nothing, and a scenario the block already selects runs once. + * + * Add a path when MCP search or a session's calls start running through another file, and a + * scenario when one is written for this code: the cases in e2e/tests/ci-selection.spec.ts fail for + * a path no file matches, a scenario no longer scheduled on Cloud, and a Cloud MCP session scenario + * left out. + */ +import { scenarios } from "./test-plan.ts"; + +/** + * The files search and session calls run through, from the repository root. An entry ending in `/` + * is a directory and everything under it; a `*` in an entry's last segment stands for any characters + * but `/`; any other entry is one file. + */ +export const mcpSearchPaths: ReadonlyArray = [ + "packages/mcp/", + // Search rows, their store and the listings they are kept from. + "packages/sdk/src/implementation/tool-search*.ts", + "packages/sdk/src/implementation/listings.ts", + // The SDK's tool reads, which search and execute make for every app they reach. + "packages/sdk/src/implementation/tools.ts", + // The hosted MCP server and the searchable tools it reads for each app. + "apps/hosted/server/src/implementation/mcp*.ts", + "apps/hosted/server/src/implementation/tools.ts", +]; + +/** The Cloud scenarios that run when one of `mcpSearchPaths` changes, by their test-plan keys. */ +export const mcpSearchScenarios: ReadonlyArray = [ + // cloud-database-placement.spec.ts: session objects' calls against held locks and connections. + "cloudMcpDiscoveryListings", + "cloudMcpParallelCalls", + "cloudMcpAbandonedStatement", + "cloudMcpObjectConnections", + "cloudMcpObjectConnectRetry", + "cloudMcpObjectPool", + // mcp-catalog-scale.spec.ts: slow and stalled listings and aged search rows on Cloud. + "mcpBackgroundListingCloud", + "mcpStoppedListingCloud", + "mcpStoppedRenewalCloud", + // The session objects' own spec files. + "cloudMcpSessionTiming", + "cloudMcpSessionRouting", + "cloudMcpSessionRestore", + "cloudMcpStreamLifetime", + "cloudMcpCallBatch", +]; + +/** The titles of `mcpSearchScenarios`. */ +export const mcpSearchTitles: ReadonlySet = new Set( + mcpSearchScenarios.map((key) => scenarios[key].title), +); + +const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +/** Whether `file`, a path from the repository root, is matched by the path entry `entry`. */ +export const matchesPath = (entry: string, file: string) => + entry.endsWith("/") + ? file.startsWith(entry) + : new RegExp(`^${entry.split("*").map(escape).join("[^/]*")}$`).test(file); + +/** Whether any of `files` is one search and session calls run through. */ +export const mcpSearchChanged = (files: ReadonlyArray) => + files.some((file) => mcpSearchPaths.some((entry) => matchesPath(entry, file))); diff --git a/e2e/npm-channel.config.ts b/e2e/npm-channel.config.ts new file mode 100644 index 0000000000..0031917051 --- /dev/null +++ b/e2e/npm-channel.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from "vitest/config"; + +/** Runs the release's wait for npm's dist-tags against a local registry that replays npm. */ +export default defineConfig({ + test: { include: ["e2e/tests/npm-channel.spec.ts"], testTimeout: 60_000 }, +}); diff --git a/e2e/packaged-engine.ts b/e2e/packaged-engine.ts index d7215f317a..159c862613 100644 --- a/e2e/packaged-engine.ts +++ b/e2e/packaged-engine.ts @@ -43,6 +43,10 @@ export const packagedEngineInputs = [ // scenarios exercise. No source lane runs them. "apps/hosted/self-host/native/**", "apps/hosted/self-host/entrypoint.sh", + // The telemetry collector that supervisor starts beside workerd and stops with it on a container + // stop, built in the Dockerfile's collector stage from the crate by this script. + "packages/telemetry/collector/**", + "packages/telemetry/scripts/build-collector.ts", // How the lane chooses them. "e2e/packaged-engine.ts", ] as const; diff --git a/e2e/prepare-cache.config.ts b/e2e/prepare-cache.config.ts index 960123c90d..e5326698de 100644 --- a/e2e/prepare-cache.config.ts +++ b/e2e/prepare-cache.config.ts @@ -1,6 +1,12 @@ import { defineConfig } from "vitest/config"; -/** Runs `scripts/e2e-prepare.ts` as a process in a fake repository with fake build steps. */ +/** + * Runs `scripts/e2e-prepare.ts` and `packages/telemetry/scripts/build-collector.ts` as processes in + * fake repositories with fake build steps and tools. + */ export default defineConfig({ - test: { include: ["e2e/tests/prepare-cache.spec.ts"], testTimeout: 60_000 }, + test: { + include: ["e2e/tests/prepare-cache.spec.ts", "e2e/tests/collector-build.spec.ts"], + testTimeout: 60_000, + }, }); diff --git a/e2e/publish-guard.config.ts b/e2e/publish-guard.config.ts new file mode 100644 index 0000000000..f756bb9272 --- /dev/null +++ b/e2e/publish-guard.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from "vitest/config"; + +/** Runs the publish job's channel guard and failure report against a local stub of each public surface. */ +export default defineConfig({ + test: { include: ["e2e/tests/publish-guard.spec.ts"], testTimeout: 90_000 }, +}); diff --git a/e2e/release-lanes.ts b/e2e/release-lanes.ts index 78b00d47ba..e4bfa1c4ea 100644 --- a/e2e/release-lanes.ts +++ b/e2e/release-lanes.ts @@ -54,19 +54,15 @@ const packagingPaths = [ "apps/docs/public/install.*", ".github/workflows/bun-runtime.yml", "scripts/pack-test-apps.ts", -] as const; - -/** - * Only the matrix after merge reads its own workflow, and the actions its collector jobs build the - * Rust telemetry collector with. The pull request lanes start running those actions at the cutover, - * when packaging starts using the collector; until then a change to them starts only the check. - */ -const matrixOnlyPaths = [ - ".github/workflows/release-artifacts.yml", + // The run's collector job builds or restores the Rust telemetry collector through these; the + // release-local jobs wait for it and fetch its build before they package it. ".github/actions/collector/**", ".github/actions/rustup/**", ] as const; +/** Only the matrix after merge reads its own workflow. */ +const matrixOnlyPaths = [".github/workflows/release-artifacts.yml"] as const; + /** * Workspace packages the suites start at run time rather than import: e2e/sdk/fixtures.ts runs * the scenario server in `apps/hosted/testing` for the fixtures the scenarios use. diff --git a/e2e/run.ts b/e2e/run.ts index 81fa74f9f1..0111320ed6 100644 --- a/e2e/run.ts +++ b/e2e/run.ts @@ -1,4 +1,8 @@ /** CLI adapter for the shared suite lifecycle. */ +// First: the harness's outbound connect time-out (support/connect-timeout.ts). +import "./support/connect-timeout.ts"; +// Second: each stage request on a connection of its own (support/stage-connections.ts). +import "./support/stage-connections.ts"; import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { Effect, Layer, Option, Schema } from "effect"; diff --git a/e2e/sdk/branch-deleted.ts b/e2e/sdk/branch-deleted.ts index e7fed25a4e..73c8f666a6 100644 --- a/e2e/sdk/branch-deleted.ts +++ b/e2e/sdk/branch-deleted.ts @@ -72,7 +72,7 @@ export const confirmBranchDeleted = ( ), Effect.catchTag("Unconfirmed", ({ reason }) => Effect.gen(function* () { - const message = `PlanetScale branch ${branch} was not confirmed deleted after its teardown (${reason}). If it is still there, it keeps billing until someone with permission deletes it.`; + const message = `PlanetScale branch ${branch} is not confirmed gone after its teardown (${reason}). The teardown's own line above says whether it sent a delete and what PlanetScale answered. If the branch is still there, it keeps billing until someone with permission deletes it.`; const ci = yield* Config.String("GITHUB_ACTIONS").pipe(Config.option); if (Option.isSome(ci)) yield* Console.error(`::error title=Leftover PlanetScale branch::${message}`); diff --git a/e2e/sdk/prepare-scenarios.ts b/e2e/sdk/prepare-scenarios.ts index 587e47e66f..de9d1462e2 100644 --- a/e2e/sdk/prepare-scenarios.ts +++ b/e2e/sdk/prepare-scenarios.ts @@ -22,6 +22,11 @@ import { OriginUnreachable, PreparedScenarios, ProvisioningFailure } from "./con import { startScenario } from "./scenario.ts"; import { awaitCompiler } from "./compiler-readiness.ts"; import { FixtureFailed, fixtureRequest, type FixtureControl } from "./fixtures.ts"; +import { saveLostRequests } from "../lost-requests.ts"; +import { configureLostConnections, takeAllLostRequests } from "../support/lost-connections.ts"; +import { targetHosts } from "../support/role-hosts.ts"; +import { drainWebhookQueue, fixtureQueueCount } from "./webhook-queue.ts"; +import type { SpanQuery } from "../support/contracts.ts"; class DomainNotReady extends Schema.TaggedError()("DomainNotReady", { reason: Schema.Literals(["http", "transport", "timeout"]), @@ -262,13 +267,17 @@ const waitForDomain = ( const infrastructureDeadline = "5 minutes"; /** The suite owns provisioned actors until individual cleanup or environment teardown, including failed preparation. */ -export const prepareCloudScenarios = (input: { +const prepareCloudScenariosUnrecorded = (input: { readonly target: typeof Target.Service; readonly scenarios: readonly { readonly title: string; readonly appOrigin?: true }[]; readonly appUiBaseUrl: string; /** The attached stage's fixture control, which reads the product's report of each address. */ readonly fixtures: typeof FixtureControl.Type; readonly workers: number; + /** Delivered webhook sweep passes on this stage since a time (epoch milliseconds). */ + readonly sweepPasses: ( + since: number, + ) => Effect.Effect; }) => Effect.gen(function* () { const started = yield* Clock.currentTimeMillis; @@ -393,9 +402,29 @@ export const prepareCloudScenarios = (input: { // setup failure; it never becomes a skip or a passing scenario. Effect.catchTag("TimeoutError", () => Effect.void), ); + // Provisioning queues every organization's webhook sources at once. Scenarios start once the + // sweep has served that queue, not behind it. + const drain = yield* drainWebhookQueue({ + count: fixtureQueueCount(input.fixtures), + passes: input.sweepPasses, + record: (figures) => + Effect.flatMap(FileSystem.FileSystem, (fs) => + fs.writeFileString( + `${input.target.directory}/webhook-queue.json`, + JSON.stringify(figures, null, 2), + ), + ).pipe( + Effect.andThen( + Console.log( + `Webhook source queue: provisioning finished in ${Math.round(figures.provisioningMs / 1000)}s${figures.provisioningOutstanding === undefined ? "" : ` (${figures.provisioningOutstanding} jobs still running at its bound)`}, ${figures.queued} queued, ${figures.drainMs === undefined ? `not drained within ${Math.round(figures.boundMs / 1000)}s` : `drained in ${Math.round(figures.drainMs / 1000)}s`} (bound ${Math.round(figures.boundMs / 1000)}s, ${figures.passes} passes, ${figures.passesWithRoom} with room${figures.failedReads === undefined ? "" : `, ${figures.failedReads} failed reads`}).`, + ), + ), + Effect.orDie, + ), + }); const finished = yield* Clock.currentTimeMillis; yield* Console.log( - `Scenario infrastructure: ${Math.round((finished - started) / 1000)}s (${prepared.length} organizations, ${(yield* Ref.get(ready)).size}/${domains.length} HTTPS domains verified).`, + `Scenario infrastructure: ${Math.round((finished - started) / 1000)}s (${prepared.length} organizations, ${(yield* Ref.get(ready)).size}/${domains.length} HTTPS domains verified, ${drain.queued} webhook sources drained in ${Math.round((drain.drainMs ?? 0) / 1000)}s${drain.provisioningOutstanding === undefined ? "" : `; ${drain.provisioningOutstanding} provisioning jobs were still running when the queue was counted, so their sources may join after it and a webhook scenario may wait behind them`}).`, ); const readyIds = yield* Ref.get(ready); const finalObservations = yield* Ref.get(observations); @@ -431,3 +460,33 @@ export const prepareCloudScenarios = (input: { ]), ); }); + +/** + * Prepare the run's Cloud organizations with the stage's requests watched as a scenario's are + * (support/lost-connections.ts): the suite process sends them, `POST /api/onboarding/prepare` + * among them, outside any Vitest worker. Requests lost in transport are saved in the run's + * `report/lost-requests/` as the preparation's, whatever the preparation's outcome. + */ +export const prepareCloudScenarios = ( + input: Parameters[0], +) => + Effect.suspend(() => { + // The wrapper itself is installed by the suite's entry point, before anything could read + // `fetch` (support/stage-connections.ts); this names the stage and the records' owner. + configureLostConnections({ + origins: Object.values(targetHosts(input.target)), + scenario: () => undefined, + file: () => "preparation", + }); + return prepareCloudScenariosUnrecorded(input); + }).pipe( + Effect.ensuring( + Effect.suspend(() => + saveLostRequests( + input.target.directory, + `preparation-${Date.now()}`, + takeAllLostRequests(), + ), + ).pipe(Effect.ignore), + ), + ); diff --git a/e2e/sdk/run-retention.ts b/e2e/sdk/run-retention.ts index 29c99e2cdd..d53429f308 100644 --- a/e2e/sdk/run-retention.ts +++ b/e2e/sdk/run-retention.ts @@ -4,9 +4,15 @@ * removes older run folders in its own workspace: * * - any run started more than `keepMs` ago; - * - passed runs past the newest `keepPassedRuns`. A run with a failed scenario, or whose outcome - * cannot be read (it crashed, was stopped or timed out), is kept for its failure evidence; - * - then, while the rest exceed `defaultKeepBytes`, passed runs oldest first, then the others oldest first. + * - passed runs past the newest `keepPassedRuns` that ended more than `recentMs` ago. A run with a + * failed scenario, or whose outcome cannot be read (it crashed, was stopped or timed out), is kept + * for its failure evidence; + * - then, while the rest exceed `defaultKeepBytes`, passed runs oldest first, then the others oldest + * first. Only this cap, which protects the disk, takes a run that ended within `recentMs`. + * + * A series of quick runs, one scenario each, read together afterwards is an ordinary way to work: + * on the MacBook Pro sixteen passed runs over 13 minutes, read 30-45 minutes after they ended, + * lost all but the last six to the count before `recentMs` existed. * * A run passed only when its `evidence.json` says `"outcome": "passed"`, which `runSuite` writes * from the same checks that fail the run. Runs saved before that field, and runs whose outcome @@ -24,6 +30,20 @@ * or a `.local/e2e` that is a link, is left alone and never followed; removal never follows links * inside a run folder either. `runSuite` does not prune in CI: one job runs several suites in a * fresh checkout, then uploads and collects coverage from every run folder. + * + * Runs it keeps are shrunk: their bulk is removed and the rest stays. Bulk is each target's + * (`local`, `self-host`, `cloud`) `coverage/` and `data/` folders and every `data/` folder under + * its `scenarios//`: the product databases, Cloud's diagnostics SQLite and the raw coverage + * reports, most of a run's bytes. On the shared host no file in them was read more than an hour + * after its run ended, while `server.log`, the reports, evidence files, traces, videos and + * screenshots were; those stay. A passed run is shrunk once a newer passed run exists and it ended + * more than `recentMs` ago; any other run 24 hours after it ended (its `evidence.json` was written, + * or its folder last changed). Nothing under a target's `report/` is ever bulk. + * A shrunk run holds `shrunk.json`, naming what was removed and when; with no bulk left, a later + * start passes it by. + * + * A `keep-all-runs` file in `.local/e2e` turns removal and shrinking off for the workspace, for + * someone investigating; the host's evidence sweep honours it too. */ import { Cause, Console, Effect, FileSystem, Option, Path, Schema } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; @@ -39,6 +59,19 @@ import { alive } from "../host-slot.ts"; const keepPassedRuns = 5; const keepMs = 3 * 24 * 60 * 60_000; const defaultKeepBytes = 3 * 1024 ** 3; +/** + * A run that ended this recently is neither removed for the count nor shrunk. Later reads of a + * run's files on the shared host came within an hour of its end (none of its bulk after one), and + * the series above was read 45 minutes after its first run ended; two hours covers both with room. + */ +const recentMs = 2 * 60 * 60_000; +/** A run other than a passed one keeps its bulk this long after it ended. */ +const shrinkAfterMs = 24 * 60 * 60_000; +/** In `.local/e2e`: keep every run whole. */ +export const keepAllMarker = "keep-all-runs"; +/** In a shrunk run folder: what was removed and when. */ +export const shrunkMarker = "shrunk.json"; +const targets = ["local", "self-host", "cloud"] as const; /** The outcome `runSuite` records in a run's `evidence.json`. */ const Outcome = Schema.fromJsonString( @@ -89,6 +122,8 @@ export const pruneRuns = ({ const real = yield* fs.realPath(root); if (real !== path.join(yield* fs.realPath(workspace), ".local", "e2e")) return yield* Console.error(`e2e: kept old runs: ${root} is or is under a symbolic link.`); + if (yield* fs.exists(path.join(root, keepAllMarker))) + return yield* Console.log(`e2e: kept every run: ${path.join(root, keepAllMarker)} is set.`); /** True while a live process holds a marker, or when a marker cannot be read. */ const inProgress = (directory: string) => @@ -110,15 +145,15 @@ export const pruneRuns = ({ * Bytes allocated under `file`, without following links and counting each inode once. A file * that cannot be read, or vanishes while another run writes, counts as nothing. */ - const seen = new Set(); - const sizeOf = (file: string): Effect.Effect => + const counted = new Set(); + const sizeOf = (file: string, seen = counted): Effect.Effect => Effect.gen(function* () { if (Option.isSome(yield* Effect.option(fs.readLink(file)))) return 0; const info = yield* fs.stat(file); if (info.type === "Directory") { let total = 0; for (const name of yield* fs.readDirectory(file)) - total += yield* sizeOf(path.join(file, name)); + total += yield* sizeOf(path.join(file, name), seen); return total; } const inode = `${info.dev}:${Option.getOrElse(info.ino, () => file)}`; @@ -131,6 +166,50 @@ export const pruneRuns = ({ }); }).pipe(Effect.orElseSucceed(() => 0)); + /** A real directory, not a link to one. */ + const isDirectory = (file: string) => + Effect.gen(function* () { + if (Option.isSome(yield* Effect.option(fs.readLink(file)))) return false; + return (yield* fs.stat(file)).type === "Directory"; + }).pipe(Effect.orElseSucceed(() => false)); + + /** The run's bulk folders, relative to it: see the comment at the top. */ + const bulkOf = (directory: string) => + Effect.gen(function* () { + const found: Array = []; + const dataUnder = (relative: string): Effect.Effect => + Effect.gen(function* () { + for (const name of yield* fs.readDirectory(path.join(directory, relative))) { + const child = path.join(relative, name); + if (!(yield* isDirectory(path.join(directory, child)))) continue; + if (name === "data") found.push(child); + else yield* dataUnder(child); + } + }).pipe(Effect.orElseSucceed(() => undefined)); + for (const target of targets) { + if (!(yield* isDirectory(path.join(directory, target)))) continue; + for (const name of ["coverage", "data"]) + if (yield* isDirectory(path.join(directory, target, name))) + found.push(path.join(target, name)); + const scenarios = path.join(target, "scenarios"); + if (yield* isDirectory(path.join(directory, scenarios))) yield* dataUnder(scenarios); + } + return found; + }); + + /** When the run ended: its `evidence.json` was written, or else its folder last changed. */ + const endedAt = (directory: string) => + fs.stat(path.join(directory, "evidence.json")).pipe( + Effect.catch(() => fs.stat(directory)), + Effect.map((info) => + Option.getOrElse( + Option.map(info.mtime, (date) => date.getTime()), + () => now, + ), + ), + Effect.orElseSucceed(() => now), + ); + /** Only an outcome `runSuite` recorded as passed. */ const passed = (directory: string) => fs.readFileString(path.join(directory, "evidence.json")).pipe( @@ -149,7 +228,12 @@ export const pruneRuns = ({ const now = Date.now(); const names = (yield* fs.readDirectory(root)).filter((name) => runName.test(name)).sort(); - type Run = { directory: string; passed: boolean; size: number }; + type Run = { + directory: string; + passed: boolean; + size: number; + ended: number; + }; const aged: Array = []; // Finished runs from the last 3 days, newest first. const runs: Array = []; @@ -166,9 +250,39 @@ export const pruneRuns = ({ if (name === current || (yield* inProgress(directory))) continue; if (now - started > keepMs) aged.push(directory); else - runs.push({ directory, passed: yield* passed(directory), size: yield* sizeOf(directory) }); + runs.push({ + directory, + passed: yield* passed(directory), + size: yield* sizeOf(directory), + ended: yield* endedAt(directory), + }); } - const removed = new Set(runs.filter((run) => run.passed).slice(keepPassedRuns)); + // Shrink before sizing against the cap, so the cap counts what stays. + const newestPassed = runs.find((run) => run.passed); + const shrinking = new Map; bytes: number; reason: string }>(); + for (const run of runs) { + const reason = run.passed + ? run === newestPassed || now - run.ended <= recentMs + ? undefined + : "passed, a newer passed run exists, and it ended more than 2 hours ago" + : now - run.ended > shrinkAfterMs + ? "not passed, and ended more than 24 hours ago" + : undefined; + if (reason === undefined) continue; + const bulk = yield* bulkOf(run.directory); + if (bulk.length === 0) continue; + const seen = new Set(); + let bytes = 0; + for (const relative of bulk) bytes += yield* sizeOf(path.join(run.directory, relative), seen); + shrinking.set(run, { bulk, bytes, reason }); + run.size = Math.max(0, run.size - bytes); + } + const removed = new Set( + runs + .filter((run) => run.passed) + .slice(keepPassedRuns) + .filter((run) => now - run.ended > recentMs), + ); let bytes = runs.filter((run) => !removed.has(run)).reduce((sum, run) => sum + run.size, 0); const oldestFirst = runs.filter((run) => !removed.has(run)).reverse(); for (const run of [ @@ -180,7 +294,8 @@ export const pruneRuns = ({ bytes -= run.size; } const doomed = [...aged, ...runs.filter((run) => removed.has(run)).map((run) => run.directory)]; - if (doomed.length === 0) return; + const shrinks = [...shrinking].filter(([run]) => !removed.has(run)); + if (doomed.length === 0 && shrinks.length === 0) return; const before = yield* free; let count = 0; @@ -194,14 +309,47 @@ export const pruneRuns = ({ failed.push(`${path.basename(directory)}: ${result.failure.message}`); else count++; } + let shrunk = 0; + for (const [run, plan] of shrinks) { + if (yield* inProgress(run.directory)) continue; + const done: Array = []; + for (const relative of plan.bulk) { + const target = path.join(run.directory, relative); + // Checked again just before: a real directory inside this run folder, not a link. + const resolved = yield* Effect.option(fs.realPath(target)); + if ( + Option.isNone(resolved) || + resolved.value !== path.join(real, path.relative(root, target)) + ) + continue; + const result = yield* Effect.result(fs.remove(target, { recursive: true, force: true })); + if (result._tag === "Failure") + failed.push(`${path.basename(run.directory)}/${relative}: ${result.failure.message}`); + else done.push(relative); + } + if (done.length === 0) continue; + const marker = yield* Effect.result( + fs.writeFileString( + path.join(run.directory, shrunkMarker), + `${JSON.stringify({ shrunkAt: new Date().toISOString(), reason: plan.reason, removed: done, note: "Removed by e2e/sdk/run-retention.ts: product databases, diagnostics and raw coverage. Logs, reports and evidence files remain." }, null, 2)}\n`, + ), + ); + if (marker._tag === "Failure") + failed.push(`${path.basename(run.directory)}/${shrunkMarker}: ${marker.failure.message}`); + shrunk++; + } const after = yield* free; const space = Option.isSome(before) && Option.isSome(after) ? ` (${Math.max(0, Math.round((after.value - before.value) / 1024 ** 2))} MB freed)` : ""; + if (shrunk > 0) + yield* Console.log( + `e2e: shrank ${shrunk} older ${shrunk === 1 ? "run" : "runs"} in ${root}: removed product databases, diagnostics and raw coverage (passed runs once a newer one passed, others a day after they ended); logs, reports and evidence stay. Each says so in ${shrunkMarker}. Keep every run whole with: touch ${path.join(root, keepAllMarker)}`, + ); if (count > 0) yield* Console.log( - `e2e: removed ${count} old ${count === 1 ? "run" : "runs"}${space} from ${root}. It keeps runs from the last 3 days: the newest ${keepPassedRuns} passed, every other one, up to ${keepBytes >= 1024 ** 3 ? `${keepBytes / 1024 ** 3} GB` : `${Math.round(keepBytes / 1024 ** 2)} MB`}.`, + `e2e: removed ${count} old ${count === 1 ? "run" : "runs"}${space} from ${root}. It keeps runs from the last 3 days: the newest ${keepPassedRuns} passed and any from the last 2 hours, every other one, up to ${keepBytes >= 1024 ** 3 ? `${keepBytes / 1024 ** 3} GB` : `${Math.round(keepBytes / 1024 ** 2)} MB`}.`, ); if (failed.length > 0) yield* Console.error( diff --git a/e2e/sdk/scenario.ts b/e2e/sdk/scenario.ts index 97067f799c..20f95fb9bf 100644 --- a/e2e/sdk/scenario.ts +++ b/e2e/sdk/scenario.ts @@ -6,6 +6,8 @@ import { Target } from "../support/platform.ts"; import { startManagedServer } from "../support/managed-server.ts"; import { OriginUnreachable, ProvisioningFailure, ScenarioId } from "./contracts.ts"; import { platformSignatures } from "../platform-signatures.ts"; +import { addStageOrigins } from "../support/lost-connections.ts"; +import { targetHosts } from "../support/role-hosts.ts"; /** A prerequisite failed within its own deadline, before scenario work could start. */ export class ScenarioPreparationFailed extends Schema.TaggedError()( @@ -96,9 +98,12 @@ export const startScenario = ( const target = yield* prepareScenario(base, label, id); if (base.metadata.target === "cloud") return target; const server = yield* startManagedServer(target, "product", environment); - return Target.of({ + const started = Target.of({ ...target, controlOrigin: server.controlOrigin, metadata: { ...target.metadata, origin: server.origin }, }); + // Its own server is the stage from its first request on, provisioning included. + addStageOrigins(Object.values(targetHosts(started))); + return started; }); diff --git a/e2e/sdk/suite.ts b/e2e/sdk/suite.ts index 09ad914266..388bef01aa 100644 --- a/e2e/sdk/suite.ts +++ b/e2e/sdk/suite.ts @@ -2,6 +2,7 @@ import { Cause, Config, + ConfigProvider, Console, Effect, Exit, @@ -13,12 +14,7 @@ import { } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { randomBytes } from "node:crypto"; -import { - patternForTarget, - scenariosForSuite, - type CloudMode, - type TestPlan, -} from "../test-plan.ts"; +import { scenariosForSuite, type CloudMode, type TestPlan } from "../test-plan.ts"; import { readEvidence, combineEvidenceReports, reportRuntimeCrashes } from "../evidence-results.ts"; import { type EvidenceReport, type RunMetadata } from "../report-model.ts"; import { startCloudEnvironment } from "../support/cloud-environment.ts"; @@ -28,6 +24,8 @@ import { type FixtureControl, fixtureControlEnvironment } from "./fixtures.ts"; import { RecordingPaceMs, Target } from "../support/platform.ts"; import { roleHost } from "../support/role-hosts.ts"; import { prepareCloudScenarios } from "./prepare-scenarios.ts"; +import { attachedStage, axiomSpans } from "../support/axiom.ts"; +import { sweepSpan } from "../support/webhook-sweep.ts"; import { removalSettlementFile, settleRemovals } from "./removal-settlement.ts"; import { removalStallsFile } from "./removal-evidence.ts"; import { holdRun, pruneRuns } from "./run-retention.ts"; @@ -312,6 +310,9 @@ export const runSuite = ({ }), }) : undefined; + // Spans of an attached stage are read from Axiom by its stage name; an origin + // that does not name it fails here, once. + const stage = attachment === undefined ? undefined : yield* attachedStage(origin); const preparedScenarios = attachment?.appUiBaseUrl === undefined ? {} @@ -328,6 +329,25 @@ export const runSuite = ({ appUiBaseUrl: attachment.appUiBaseUrl, fixtures: attachment.fixtures, workers: targetWorkers, + sweepPasses: (since) => + axiomSpans(stage).pipe( + Effect.flatMap((spans) => + spans.search(sweepSpan, {}, undefined, since), + ), + Effect.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown( + attachment.axiom === undefined + ? {} + : { + E2E_AXIOM_TOKEN: reveal(attachment.axiom.token), + E2E_AXIOM_DATASET: attachment.axiom.dataset, + E2E_AXIOM_ORG_ID: attachment.axiom.organization, + }, + ), + ), + ), + ), scenarios: plan .filter( (scenario: typeof TestPlan.Type) => @@ -347,19 +367,7 @@ export const runSuite = ({ const exit = yield* processes.exitCode( ChildProcess.make( "node", - [ - "node_modules/vitest/vitest.mjs", - "run", - "--config", - "e2e/vitest.config.ts", - "--testNamePattern", - patternForTarget( - target, - selected === "hosted" ? "hosted" : "all", - name, - cloudMode, - ), - ], + ["node_modules/vitest/vitest.mjs", "run", "--config", "e2e/vitest.config.ts"], { extendEnv: false, env: { @@ -396,7 +404,6 @@ export const runSuite = ({ ...(process.env.E2E_COVERAGE === undefined ? {} : { E2E_COVERAGE: process.env.E2E_COVERAGE }), - E2E_PREPARED_SCENARIOS: JSON.stringify(preparedScenarios), E2E_TEST_NAME: name, E2E_WORKERS: String(interactive || observeUI ? 1 : targetWorkers), E2E_TARGET: target, diff --git a/e2e/sdk/webhook-queue.ts b/e2e/sdk/webhook-queue.ts new file mode 100644 index 0000000000..a7b150da26 --- /dev/null +++ b/e2e/sdk/webhook-queue.ts @@ -0,0 +1,205 @@ +/** + * The start-up webhook source queue of a stage. Provisioning every scenario's organization at once + * queues their webhook sources together; a scenario that starts behind that queue waits for the + * whole of it before the sweep reaches its own source. Preparation waits here until the queue it + * caused has drained, so scenarios start against a sweep that is keeping up. + */ +import { Clock, Effect, Schema } from "effect"; +import type { SpanQuery } from "../support/contracts.ts"; +import { + type Delivered, + hasRoom, + spanEnd, + spanStart, + sweepLimit, + sweepTaken, +} from "../support/webhook-sweep.ts"; +import { fixtureRequest, type FixtureControl } from "./fixtures.ts"; + +/** + * The sweep's measured rate on a deployed stage, in sources a second: its passes of 8 served the + * start-up queue of a deployed run at 2.33 a second (the run's `sdk.webhooks.reconcileSource` + * spans). The bound below follows from it; a slower sweep fails the bound and says so. + */ +export const measuredRate = 2.33; +/** The most a pass takes: the SDK's `defaultScheduleWorkerOptions.concurrency`. */ +export const passLimit = 8; +/** The figures the drain's bound and reads use; a test gives its own. */ +export interface DrainPace { + /** The sweep's rate, in sources a second. */ + readonly rate: number; + /** Time for a finished pass's span to be delivered and readable. */ + readonly deliveryMs: number; + /** How often the passes and the queue are read. */ + readonly readSpacingMs: number; + /** How long the stage's team and member provisioning may take to finish queuing its sources. */ + readonly provisioningMs: number; + /** + * The longest an idle sweeper waits before its next pass. When the count catches the pass that + * takes the last sources already running, no later pass comes until the next wake. + */ + readonly idleWakeMs: number; +} +export const deployedPace: DrainPace = { + rate: measuredRate, + deliveryMs: 30_000, + readSpacingMs: 5_000, + // Preparation's infrastructure deadline, which provisioning is part of. + provisioningMs: 300_000, + // An idle Cloud coordinator is woken by the `schedule-wake` job on the `* * * * *` Cron Trigger + // (apps/hosted/cloud/src/main.ts): every 60 s. On 16 November an idle test stage's passes were at + // most 62.2 s apart (2,816 passes), so 5 s more covers the trigger's late starts. + idleWakeMs: 65_000, +}; + +const Queue = Schema.Struct({ + pending: Schema.Int, + provisioning: Schema.Int, + at: Schema.Number, +}); + +/** + * How long the queue may take: every queued source served at the measured rate, then one more + * pass (the one that has room) taken and finished, at the latest on the sweeper's next idle wake, + * then its span delivered. + */ +export const drainBound = (queued: number, pace: DrainPace = deployedPace) => + Math.ceil(((queued + 2 * passLimit) / pace.rate) * 1000) + pace.idleWakeMs + pace.deliveryMs; + +/** What the drain records in the run's evidence (preparation.json). */ +export interface QueueDrain { + /** How long the stage's team and member provisioning took to finish once the drain began. */ + readonly provisioningMs: number; + /** Provisioning jobs still to finish at its bound, whose sources may join after the count. */ + readonly provisioningOutstanding?: number; + /** Sources pending and unleased when provisioning had finished queuing them. */ + readonly queued: number; + readonly boundMs: number; + /** From the count to the end of the first pass with room after it; absent if none came. */ + readonly drainMs?: number; + /** Passes that started after the count, and how many of them had room. */ + readonly passes: number; + readonly passesWithRoom: number; + /** The pass that showed the queue drained: its start, what it took and its limit. */ + readonly drainedBy?: { + readonly startTime: string; + readonly traceId: string; + readonly taken: number; + readonly limit: number; + }; + /** Sources still pending and unleased when the bound was reached. */ + readonly pendingAtBound?: number; + /** Reads of the passes that failed, each read again at the next spacing. */ + readonly failedReads?: number; +} + +export class QueueNotDrained extends Schema.TaggedError()("QueueNotDrained", { + message: Schema.String, +}) {} + +/** + * Wait until the stage's team and member provisioning has finished, since it links each + * organization's default app accounts and so queues their sources after actor setup answers; + * then count the queue, and wait until a sweep pass that started after the count has had room. + * Passes take pending sources first in, first out, so such a pass took the last source queued + * before the count. Fails at `drainBound(queued)` with the figures. Provisioning that has not + * finished by its bound is recorded, and the queue is counted then. + */ +/** The stage's queue as the fixture control counts it (`/webhook-queue`). */ +export const fixtureQueueCount = (fixtures: typeof FixtureControl.Type) => + fixtureRequest(fixtures, "/webhook-queue", {}).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Queue)), + ); + +export const drainWebhookQueue = (input: { + /** The stage's queue: pending sources, provisioning jobs still to finish, the database's time. */ + readonly count: Effect.Effect; + /** Delivered sweep passes on this stage since `since` (epoch milliseconds). */ + readonly passes: (since: number) => Effect.Effect; + readonly record: (drain: QueueDrain) => Effect.Effect; + readonly pace?: DrainPace; +}) => + Effect.gen(function* () { + const pace = input.pace ?? deployedPace; + const read = input.count; + const provisioningStarted = yield* Clock.currentTimeMillis; + let queue = yield* read; + while ( + queue.provisioning > 0 && + (yield* Clock.currentTimeMillis) - provisioningStarted < pace.provisioningMs + ) { + yield* Effect.sleep(pace.readSpacingMs); + queue = yield* read; + } + const started = yield* Clock.currentTimeMillis; + const provisioning = { + provisioningMs: started - provisioningStarted, + ...(queue.provisioning > 0 ? { provisioningOutstanding: queue.provisioning } : {}), + }; + const boundMs = drainBound(queue.pending, pace); + // Nothing queued: nothing to drain, and no pass need come. + if (queue.pending === 0) { + const result: QueueDrain = { + ...provisioning, + queued: 0, + boundMs, + drainMs: 0, + passes: 0, + passesWithRoom: 0, + }; + yield* input.record(result); + return result; + } + let after: readonly Delivered[] = []; + let failedReads = 0; + for (;;) { + after = (yield* input.passes(queue.at - 60_000).pipe( + Effect.map((found) => found.data), + // A read that fails is counted and read again; only the bound ends the wait. + Effect.orElseSucceed(() => { + failedReads += 1; + return after; + }), + )) + .filter(({ span }) => spanStart(span) > queue.at) + .toSorted((a, b) => spanStart(a.span) - spanStart(b.span)); + const drained = after.find(({ span }) => hasRoom(span)); + const summary = { + ...provisioning, + queued: queue.pending, + boundMs, + passes: after.length, + passesWithRoom: after.filter(({ span }) => hasRoom(span)).length, + ...(failedReads > 0 ? { failedReads } : {}), + }; + if (drained !== undefined) { + const result: QueueDrain = { + ...summary, + drainMs: spanEnd(drained.span) - queue.at, + drainedBy: { + startTime: drained.span.startTime, + traceId: drained.traceId, + taken: sweepTaken(drained.span), + limit: sweepLimit(drained.span), + }, + }; + yield* input.record(result); + return result; + } + if ((yield* Clock.currentTimeMillis) - started >= boundMs) { + const left = yield* read.pipe( + Effect.map(({ pending }) => pending), + Effect.option, + ); + const result: QueueDrain = { + ...summary, + ...(left._tag === "Some" ? { pendingAtBound: left.value } : {}), + }; + yield* input.record(result); + return yield* new QueueNotDrained({ + message: `The webhook source queue did not drain within its bound: ${JSON.stringify(result)}; no sweep pass after the count had room (${pace.rate} sources a second expected).`, + }); + } + yield* Effect.sleep(pace.readSpacingMs); + } + }); diff --git a/e2e/setup.ts b/e2e/setup.ts index 8dc073e9a8..8413e541ad 100644 --- a/e2e/setup.ts +++ b/e2e/setup.ts @@ -1,3 +1,18 @@ -import { installScenarioLifecycle } from "./support/lifecycle.ts"; +// First: the harness's outbound connect time-out (support/connect-timeout.ts). +import "./support/connect-timeout.ts"; +// Second: each stage request on a connection of its own (support/stage-connections.ts). +import "./support/stage-connections.ts"; +import { expect } from "vitest"; +import { configureLostConnections } from "./support/lost-connections.ts"; +import { installLostRequestSaving, installScenarioLifecycle } from "./support/lifecycle.ts"; + +// Stage requests lost on a reused connection before any response (support/lost-connections.ts). +// Each scenario's lifecycle names its stage's origins; records name their scenario and test file. +configureLostConnections({ + origins: [], + scenario: () => expect.getState().currentTestName, + file: () => expect.getState().testPath?.split("/").at(-1), +}); +installLostRequestSaving(); installScenarioLifecycle(); diff --git a/e2e/skip-check-replay.json b/e2e/skip-check-replay.json new file mode 100644 index 0000000000..ca7ef2e4a4 --- /dev/null +++ b/e2e/skip-check-replay.json @@ -0,0 +1,2253 @@ +[ + { + "lower": 4364, + "run": 43799666868, + "attempt": 2, + "ref": "fix-2491-app-watchdog", + "head": "8a7e19daab54059f0800ced48ed1043ac20801b1", + "bottom": "main", + "uppers": [ + { + "pull": 4365, + "head": "8f1f155a8840794149683aca1066b476ae997bb5" + } + ], + "runs": [ + { + "id": 43799666989, + "pull": 4365, + "head": "8f1f155a8840794149683aca1066b476ae997bb5", + "phases": [ + { + "at": -359, + "selectDone": false + }, + { + "at": -266, + "selectDone": true + }, + { + "at": -31, + "selectDone": false + }, + { + "at": 4, + "selectDone": true, + "record": "pull request 4365\nmerge tree 7128865d2b8d3ddbab0517153e04725139beff31\nbase 8a7e19daab54059f0800ced48ed1043ac20801b1\nselection 5c35bd593af495fe9c7b0d98bed84fa8fd8c516411da45e1996d6ee251a1504c\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + }, + { + "id": 43799666368, + "pull": 4365, + "head": "8f1f155a8840794149683aca1066b476ae997bb5", + "phases": [ + { + "at": -359, + "selectDone": false + }, + { + "at": -318, + "selectDone": true + }, + { + "at": -29, + "selectDone": false + }, + { + "at": 768, + "selectDone": true, + "record": "pull request 4365\nmerge tree 7128865d2b8d3ddbab0517153e04725139beff31\nbase 8a7e19daab54059f0800ced48ed1043ac20801b1\nselection 5c35bd593af495fe9c7b0d98bed84fa8fd8c516411da45e1996d6ee251a1504c\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + } + ] + }, + { + "lower": 4574, + "run": 43797130175, + "attempt": 1, + "ref": "ci/worker-baseline", + "head": "74b91945cea1c71cfa0eb5058dc8d28ea30d500d", + "bottom": "main", + "uppers": [ + { + "pull": 4575, + "head": "588d03fd09a27864d87c99dfe5bb16d601798faa" + } + ], + "runs": [ + { + "id": 43797129049, + "pull": 4575, + "head": "588d03fd09a27864d87c99dfe5bb16d601798faa", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 11, + "selectDone": true, + "record": "pull request 4575\nmerge tree 91c4b6f4ba044882c06dfc7b54ab120cd4501409\nbase 74b91945cea1c71cfa0eb5058dc8d28ea30d500d\nselection 37f62186dfaffccc8d3c2965b2d9846bf4475edf2232461e7269cc7863706740\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\nbaseline main@9cbea0adc2607de7d6fbabcaa35171d7171b0de8\n" + } + ] + }, + { + "id": 43797129620, + "pull": 4575, + "head": "588d03fd09a27864d87c99dfe5bb16d601798faa", + "phases": [ + { + "at": -25, + "selectDone": false + }, + { + "at": 455, + "selectDone": true, + "record": "pull request 4575\nmerge tree 91c4b6f4ba044882c06dfc7b54ab120cd4501409\nbase 4a434124a26fd438712d84b4ff66603260f55fdb\nselection 37f62186dfaffccc8d3c2965b2d9846bf4475edf2232461e7269cc7863706740\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\nbaseline main@9cbea0adc2607de7d6fbabcaa35171d7171b0de8\n" + } + ], + "cancelledAt": 566 + }, + { + "id": 43798043633, + "pull": 4575, + "head": "404ff786b7aa74a55be40271078081438d564e18", + "phases": [ + { + "at": 512, + "selectDone": false + }, + { + "at": 546, + "selectDone": true, + "record": "pull request 4575\nmerge tree ece6515ac490ababd45f253b3ebac8f7ba1bc578\nbase 74b91945cea1c71cfa0eb5058dc8d28ea30d500d\nselection 37f62186dfaffccc8d3c2965b2d9846bf4475edf2232461e7269cc7863706740\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\nbaseline main@9cbea0adc2607de7d6fbabcaa35171d7171b0de8\n" + } + ] + } + ] + }, + { + "lower": 4535, + "run": 43796364777, + "attempt": 1, + "ref": "durable/cron-rules", + "head": "47c2a7b04175762e611ac1b7686c1bea64ddc9c0", + "bottom": "durable/cron-scheduler-tests", + "uppers": [ + { + "pull": 4536, + "head": "73820f8ccd094b04716cad7a0108164ea7238ba9" + } + ], + "runs": [ + { + "id": 43796364168, + "pull": 4536, + "head": "73820f8ccd094b04716cad7a0108164ea7238ba9", + "phases": [ + { + "at": -57, + "selectDone": false + }, + { + "at": -23, + "selectDone": true, + "record": "pull request 4536\nmerge tree 1a4590a550158a5dcf867471e6cf80695ec31bb1\ndescription sha256 1bdc79149185ec9f759c77c315ed99526c91b32576ddf146565afe5b44a9e415\n" + } + ] + }, + { + "id": 43796365288, + "pull": 4536, + "head": "73820f8ccd094b04716cad7a0108164ea7238ba9", + "phases": [ + { + "at": -56, + "selectDone": false + }, + { + "at": 338, + "selectDone": true, + "record": "pull request 4536\nmerge tree 1a4590a550158a5dcf867471e6cf80695ec31bb1\ndescription sha256 1bdc79149185ec9f759c77c315ed99526c91b32576ddf146565afe5b44a9e415\n" + } + ] + } + ] + }, + { + "lower": 4557, + "run": 43796364217, + "attempt": 1, + "ref": "durable/cron-ops", + "head": "249c119e20fa0f675f9b675eaa658f918c78562d", + "bottom": "durable/cron-rules-tests", + "uppers": [ + { + "pull": 4558, + "head": "47ab6f3e4fa67334f175c53945740be9d494cad2" + } + ], + "runs": [ + { + "id": 43796363742, + "pull": 4558, + "head": "47ab6f3e4fa67334f175c53945740be9d494cad2", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 6, + "selectDone": true, + "record": "pull request 4558\nmerge tree 76d1ac9be1098171b1e78ecc762075c9805e6837\ndescription sha256 cad92e29bd70a0b6cf6f968f398f95fea47d331a046c9636e6eb8f9e399ed4ea\n" + } + ] + }, + { + "id": 43796365156, + "pull": 4558, + "head": "47ab6f3e4fa67334f175c53945740be9d494cad2", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 1292, + "selectDone": true, + "record": "pull request 4558\nmerge tree 76d1ac9be1098171b1e78ecc762075c9805e6837\ndescription sha256 ad61884995e7c9a5c2ad0bef01e5a85fd62f7c8f3b249e29dd250622b66f9d41\n" + } + ] + } + ] + }, + { + "lower": 4535, + "run": 43796363943, + "attempt": 1, + "ref": "durable/cron-rules", + "head": "47c2a7b04175762e611ac1b7686c1bea64ddc9c0", + "bottom": "durable/cron-scheduler-tests", + "uppers": [ + { + "pull": 4536, + "head": "73820f8ccd094b04716cad7a0108164ea7238ba9" + } + ], + "runs": [ + { + "id": 43796364168, + "pull": 4536, + "head": "73820f8ccd094b04716cad7a0108164ea7238ba9", + "phases": [ + { + "at": -28, + "selectDone": false + }, + { + "at": 6, + "selectDone": true, + "record": "pull request 4536\nmerge tree 1a4590a550158a5dcf867471e6cf80695ec31bb1\ndescription sha256 1bdc79149185ec9f759c77c315ed99526c91b32576ddf146565afe5b44a9e415\n" + } + ] + }, + { + "id": 43796365288, + "pull": 4536, + "head": "73820f8ccd094b04716cad7a0108164ea7238ba9", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 367, + "selectDone": true, + "record": "pull request 4536\nmerge tree 1a4590a550158a5dcf867471e6cf80695ec31bb1\ndescription sha256 1bdc79149185ec9f759c77c315ed99526c91b32576ddf146565afe5b44a9e415\n" + } + ] + } + ] + }, + { + "lower": 4526, + "run": 43796363272, + "attempt": 1, + "ref": "durable/cron-scheduler", + "head": "831ca2b08aa2214a2e8bac2582895c71af5821cf", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4527, + "head": "a703bf6e16c726ba169b05f7f58fe8acee155d9d" + } + ], + "runs": [ + { + "id": 43796363445, + "pull": 4527, + "head": "a703bf6e16c726ba169b05f7f58fe8acee155d9d", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 9, + "selectDone": true, + "record": "pull request 4527\nmerge tree 0448c485f28c58e11c0db27651385939a5ea6ec6\ndescription sha256 41a2dbc07c23e6d72de28d624bb77089549116d9ef6c12505bd07d06d24cd209\n" + } + ] + }, + { + "id": 43796364203, + "pull": 4527, + "head": "a703bf6e16c726ba169b05f7f58fe8acee155d9d", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 419, + "selectDone": true, + "record": "pull request 4527\nmerge tree 0448c485f28c58e11c0db27651385939a5ea6ec6\ndescription sha256 41a2dbc07c23e6d72de28d624bb77089549116d9ef6c12505bd07d06d24cd209\n" + } + ] + } + ] + }, + { + "lower": 4586, + "run": 43795625424, + "attempt": 1, + "ref": "scratch/skip-above-base", + "head": "e9d068195d6939de0f6e1939b97501a318820095", + "bottom": "main", + "uppers": [ + { + "pull": 4585, + "head": "5d641c749becc45df98407478fe0d0ba0f7851fe" + } + ], + "runs": [ + { + "id": 43795138622, + "pull": 4585, + "head": "5d641c749becc45df98407478fe0d0ba0f7851fe", + "phases": [ + { + "at": -330, + "selectDone": false + }, + { + "at": -297, + "selectDone": true, + "record": "pull request 4585\nmerge tree 92630e7577db10cf4aac5722b60b984021f08053\nbase e9d068195d6939de0f6e1939b97501a318820095\nselection 37f62186dfaffccc8d3c2965b2d9846bf4475edf2232461e7269cc7863706740\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\nbaseline scratch/skip-above-base@e9d068195d6939de0f6e1939b97501a318820095\n" + }, + { + "at": 307, + "selectDone": false, + "record": "pull request 4585\nmerge tree 92630e7577db10cf4aac5722b60b984021f08053\nbase e9d068195d6939de0f6e1939b97501a318820095\nselection 37f62186dfaffccc8d3c2965b2d9846bf4475edf2232461e7269cc7863706740\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\nbaseline scratch/skip-above-base@e9d068195d6939de0f6e1939b97501a318820095\n" + }, + { + "at": 364, + "selectDone": true, + "record": "pull request 4585\nmerge tree 92630e7577db10cf4aac5722b60b984021f08053\nbase e9d068195d6939de0f6e1939b97501a318820095\nselection 37f62186dfaffccc8d3c2965b2d9846bf4475edf2232461e7269cc7863706740\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\nbaseline main@9cbea0adc2607de7d6fbabcaa35171d7171b0de8\n" + } + ] + } + ] + }, + { + "lower": 4568, + "run": 43794794693, + "attempt": 1, + "ref": "rhys/runtime-connect-repeat", + "head": "8bb1fc78714af3bbbd9e26a8b65ce62e6777f5ae", + "bottom": "main", + "uppers": [ + { + "pull": 4569, + "head": "ece1d20860069e21b60da9a39eafd0665ef0cb56" + } + ], + "runs": [ + { + "id": 43794795386, + "pull": 4569, + "head": "ece1d20860069e21b60da9a39eafd0665ef0cb56", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 8, + "selectDone": true, + "record": "pull request 4569\nmerge tree 24ca9046a9c842a6311a51feb3554b577ff7dc76\nbase 8bb1fc78714af3bbbd9e26a8b65ce62e6777f5ae\nselection 26b39ca02f1dc2b385ac7d1643aa11226aa354450273c7104cb938cc9ff1f42a\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + }, + { + "id": 43794795889, + "pull": 4569, + "head": "ece1d20860069e21b60da9a39eafd0665ef0cb56", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 432, + "selectDone": true, + "record": "pull request 4569\nmerge tree 24ca9046a9c842a6311a51feb3554b577ff7dc76\nbase 7fa6f1f36ed65946f11e87980e56d1713f8d65e8\nselection 26b39ca02f1dc2b385ac7d1643aa11226aa354450273c7104cb938cc9ff1f42a\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + } + ] + }, + { + "lower": 4582, + "run": 43793923058, + "attempt": 1, + "ref": "eapps-pages-live", + "head": "dc0011c202a7291a374a3c49fb47e9c0771c52e7", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4583, + "head": "64b040778576b9040508f36bfbb891dcb1434fab" + } + ], + "runs": [ + { + "id": 43793924084, + "pull": 4583, + "head": "64b040778576b9040508f36bfbb891dcb1434fab", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 376, + "selectDone": true, + "record": "pull request 4583\nmerge tree 8bebf6457ba4db3405ddc878982eade48fe9d1ab\ndescription sha256 aa605f66f09fb9ec4d39be7739178cb15d72ff43514ed312db69e20e28c926cd\n" + } + ] + }, + { + "id": 43793923880, + "pull": 4583, + "head": "64b040778576b9040508f36bfbb891dcb1434fab", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 7, + "selectDone": true, + "record": "pull request 4583\nmerge tree 8bebf6457ba4db3405ddc878982eade48fe9d1ab\ndescription sha256 aa605f66f09fb9ec4d39be7739178cb15d72ff43514ed312db69e20e28c926cd\n" + } + ] + } + ] + }, + { + "lower": 4364, + "run": 43793324814, + "attempt": 1, + "ref": "fix-2491-app-watchdog", + "head": "c0e8d93ab61273343367fa1304e472f7d12d5b48", + "bottom": "main", + "uppers": [ + { + "pull": 4365, + "head": "a602b49a55abc7bf588c02d110f4afc2c8cbc3ce" + } + ], + "runs": [ + { + "id": 43793324465, + "pull": 4365, + "head": "a602b49a55abc7bf588c02d110f4afc2c8cbc3ce", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 5, + "selectDone": true, + "record": "pull request 4365\nmerge tree 1a09b2e8b0932ff88d8d60cf1458dcb251901b18\nbase c0e8d93ab61273343367fa1304e472f7d12d5b48\nselection 5bfbe6ffec087605c3497f6d2a58d87974dce5f500b3f66f7c3ad600bfb69bc3\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + }, + { + "id": 43793326342, + "pull": 4365, + "head": "a602b49a55abc7bf588c02d110f4afc2c8cbc3ce", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 787, + "selectDone": true, + "record": "pull request 4365\nmerge tree 1a09b2e8b0932ff88d8d60cf1458dcb251901b18\nbase c0e8d93ab61273343367fa1304e472f7d12d5b48\nselection 5bfbe6ffec087605c3497f6d2a58d87974dce5f500b3f66f7c3ad600bfb69bc3\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + } + ] + }, + { + "lower": 4557, + "run": 43793230202, + "attempt": 1, + "ref": "durable/cron-ops", + "head": "daf1863a4349f30b35d0bc5552f2e3922f6eccca", + "bottom": "durable/cron-rules-tests", + "uppers": [ + { + "pull": 4558, + "head": "727803bd1d93303eea2cc67b8b13131d0b0e03b7" + } + ], + "runs": [ + { + "id": 43793228840, + "pull": 4558, + "head": "727803bd1d93303eea2cc67b8b13131d0b0e03b7", + "phases": [ + { + "at": -59, + "selectDone": false + }, + { + "at": -25, + "selectDone": true, + "record": "pull request 4558\nmerge tree 7d9267be4b4cd9cd440685e4a6540d3e8d1246f2\ndescription sha256 cad92e29bd70a0b6cf6f968f398f95fea47d331a046c9636e6eb8f9e399ed4ea\n" + } + ] + }, + { + "id": 43793229125, + "pull": 4558, + "head": "727803bd1d93303eea2cc67b8b13131d0b0e03b7", + "phases": [ + { + "at": -59, + "selectDone": false + }, + { + "at": 1251, + "selectDone": true, + "record": "pull request 4558\nmerge tree 7d9267be4b4cd9cd440685e4a6540d3e8d1246f2\ndescription sha256 cad92e29bd70a0b6cf6f968f398f95fea47d331a046c9636e6eb8f9e399ed4ea\n" + } + ] + } + ] + }, + { + "lower": 4557, + "run": 43793229529, + "attempt": 1, + "ref": "durable/cron-ops", + "head": "daf1863a4349f30b35d0bc5552f2e3922f6eccca", + "bottom": "durable/cron-rules-tests", + "uppers": [ + { + "pull": 4558, + "head": "727803bd1d93303eea2cc67b8b13131d0b0e03b7" + } + ], + "runs": [ + { + "id": 43793228840, + "pull": 4558, + "head": "727803bd1d93303eea2cc67b8b13131d0b0e03b7", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 7, + "selectDone": true, + "record": "pull request 4558\nmerge tree 7d9267be4b4cd9cd440685e4a6540d3e8d1246f2\ndescription sha256 cad92e29bd70a0b6cf6f968f398f95fea47d331a046c9636e6eb8f9e399ed4ea\n" + } + ] + }, + { + "id": 43793229125, + "pull": 4558, + "head": "727803bd1d93303eea2cc67b8b13131d0b0e03b7", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 1283, + "selectDone": true, + "record": "pull request 4558\nmerge tree 7d9267be4b4cd9cd440685e4a6540d3e8d1246f2\ndescription sha256 cad92e29bd70a0b6cf6f968f398f95fea47d331a046c9636e6eb8f9e399ed4ea\n" + } + ] + } + ] + }, + { + "lower": 4535, + "run": 43793228984, + "attempt": 1, + "ref": "durable/cron-rules", + "head": "36ea0ac9bfa96900180cd42f3be1a9029e736b73", + "bottom": "durable/cron-scheduler-tests", + "uppers": [ + { + "pull": 4536, + "head": "80b3759b859a8486c61b073e7b3082d19039758a" + } + ], + "runs": [ + { + "id": 43793228281, + "pull": 4536, + "head": "80b3759b859a8486c61b073e7b3082d19039758a", + "phases": [ + { + "at": -56, + "selectDone": false + }, + { + "at": -23, + "selectDone": true, + "record": "pull request 4536\nmerge tree b7d394a9c5618516ab5e702d60ad982012619c5c\ndescription sha256 1bdc79149185ec9f759c77c315ed99526c91b32576ddf146565afe5b44a9e415\n" + } + ] + }, + { + "id": 43793228635, + "pull": 4536, + "head": "80b3759b859a8486c61b073e7b3082d19039758a", + "phases": [ + { + "at": -55, + "selectDone": false + }, + { + "at": 335, + "selectDone": true, + "record": "pull request 4536\nmerge tree b7d394a9c5618516ab5e702d60ad982012619c5c\ndescription sha256 1bdc79149185ec9f759c77c315ed99526c91b32576ddf146565afe5b44a9e415\n" + } + ] + } + ] + }, + { + "lower": 4535, + "run": 43793228808, + "attempt": 1, + "ref": "durable/cron-rules", + "head": "36ea0ac9bfa96900180cd42f3be1a9029e736b73", + "bottom": "durable/cron-scheduler-tests", + "uppers": [ + { + "pull": 4536, + "head": "80b3759b859a8486c61b073e7b3082d19039758a" + } + ], + "runs": [ + { + "id": 43793228281, + "pull": 4536, + "head": "80b3759b859a8486c61b073e7b3082d19039758a", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 6, + "selectDone": true, + "record": "pull request 4536\nmerge tree b7d394a9c5618516ab5e702d60ad982012619c5c\ndescription sha256 1bdc79149185ec9f759c77c315ed99526c91b32576ddf146565afe5b44a9e415\n" + } + ] + }, + { + "id": 43793228635, + "pull": 4536, + "head": "80b3759b859a8486c61b073e7b3082d19039758a", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 364, + "selectDone": true, + "record": "pull request 4536\nmerge tree b7d394a9c5618516ab5e702d60ad982012619c5c\ndescription sha256 1bdc79149185ec9f759c77c315ed99526c91b32576ddf146565afe5b44a9e415\n" + } + ] + } + ] + }, + { + "lower": 4437, + "run": 43790115574, + "attempt": 1, + "ref": "app-scale/5-rejected-accounts", + "head": "452e96027efba5aee4e60df70bdab91cb47ba82d", + "bottom": "app-scale/4-search-discovery", + "uppers": [ + { + "pull": 4438, + "head": "7129756670588599355df60af52e6803a170cb38" + } + ], + "runs": [ + { + "id": 43790114389, + "pull": 4438, + "head": "7129756670588599355df60af52e6803a170cb38", + "phases": [ + { + "at": -58, + "selectDone": false + }, + { + "at": -25, + "selectDone": true, + "record": "pull request 4438\nmerge tree 1b686ae1d56349bcbce66f4fa3ba4b79c833ff91\ndescription sha256 639b6fc9b15cbce9e6e4461850caaf947b4d11516093e0ef05374b0bbc0f671c\n" + } + ] + }, + { + "id": 43790115244, + "pull": 4438, + "head": "7129756670588599355df60af52e6803a170cb38", + "phases": [ + { + "at": -58, + "selectDone": false + }, + { + "at": 333, + "selectDone": true, + "record": "pull request 4438\nmerge tree 1b686ae1d56349bcbce66f4fa3ba4b79c833ff91\ndescription sha256 cbe283bdb8a9a6f035b80cf4bea0a02d69ca8f9796c160536817093e92cc963c\n" + } + ] + }, + { + "id": 43792016137, + "pull": 4438, + "head": "630d0fc390d1b7600b83c1056fbf89242ee30cdb", + "phases": [ + { + "at": 1062, + "selectDone": false + }, + { + "at": 1094, + "selectDone": true, + "record": "pull request 4438\nmerge tree 799a8cb56d416cd83e82ee3fc71e9ca7009c46c9\ndescription sha256 343b4f78b812df86b793350e50c199e97641a1d991cd705b7366b0fa921b3cb6\n" + } + ] + } + ] + }, + { + "lower": 4437, + "run": 43790115295, + "attempt": 1, + "ref": "app-scale/5-rejected-accounts", + "head": "452e96027efba5aee4e60df70bdab91cb47ba82d", + "bottom": "app-scale/4-search-discovery", + "uppers": [ + { + "pull": 4438, + "head": "7129756670588599355df60af52e6803a170cb38" + } + ], + "runs": [ + { + "id": 43790114389, + "pull": 4438, + "head": "7129756670588599355df60af52e6803a170cb38", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 6, + "selectDone": true, + "record": "pull request 4438\nmerge tree 1b686ae1d56349bcbce66f4fa3ba4b79c833ff91\ndescription sha256 639b6fc9b15cbce9e6e4461850caaf947b4d11516093e0ef05374b0bbc0f671c\n" + } + ] + }, + { + "id": 43790115244, + "pull": 4438, + "head": "7129756670588599355df60af52e6803a170cb38", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 364, + "selectDone": true, + "record": "pull request 4438\nmerge tree 1b686ae1d56349bcbce66f4fa3ba4b79c833ff91\ndescription sha256 cbe283bdb8a9a6f035b80cf4bea0a02d69ca8f9796c160536817093e92cc963c\n" + } + ] + }, + { + "id": 43792016137, + "pull": 4438, + "head": "630d0fc390d1b7600b83c1056fbf89242ee30cdb", + "phases": [ + { + "at": 1093, + "selectDone": false + }, + { + "at": 1125, + "selectDone": true, + "record": "pull request 4438\nmerge tree 799a8cb56d416cd83e82ee3fc71e9ca7009c46c9\ndescription sha256 343b4f78b812df86b793350e50c199e97641a1d991cd705b7366b0fa921b3cb6\n" + } + ] + } + ] + }, + { + "lower": 4436, + "run": 43790114596, + "attempt": 1, + "ref": "app-scale/4-search-discovery", + "head": "c300a2ea790816dfcc93264cf5183656a19cb324", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4437, + "head": "452e96027efba5aee4e60df70bdab91cb47ba82d" + } + ], + "runs": [ + { + "id": 43790115574, + "pull": 4437, + "head": "452e96027efba5aee4e60df70bdab91cb47ba82d", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 34, + "selectDone": true, + "record": "pull request 4437\nmerge tree 4d5aac311ff86d5bb0f651ffb0d5f41f5c1fb074\ndescription sha256 fdda1ccd74ee28417e96a8350babbe2785bd06c1c5399f154537733069ccf6bf\n" + } + ] + }, + { + "id": 43790115295, + "pull": 4437, + "head": "452e96027efba5aee4e60df70bdab91cb47ba82d", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 4, + "selectDone": true, + "record": "pull request 4437\nmerge tree 4d5aac311ff86d5bb0f651ffb0d5f41f5c1fb074\ndescription sha256 4588350f9492220c842e07cab54124a837501e8ebc775c13be02172bfc2e4cbc\n" + } + ] + } + ] + }, + { + "lower": 4574, + "run": 43788608204, + "attempt": 1, + "ref": "ci/worker-baseline", + "head": "4a434124a26fd438712d84b4ff66603260f55fdb", + "bottom": "main", + "uppers": [ + { + "pull": 4575, + "head": "09424c0a6cc6a128ea4cf4a4d5523085454349e3" + } + ], + "runs": [ + { + "id": 43788608976, + "pull": 4575, + "head": "09424c0a6cc6a128ea4cf4a4d5523085454349e3", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 9, + "selectDone": true, + "record": "pull request 4575\nmerge tree 0e8765f9aef3e24f883278faf081335c462f94a0\nbase 4a434124a26fd438712d84b4ff66603260f55fdb\nselection 3428b1adb8b635248c9113de2433206d47843fb0136340af636d8e0568551f66\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + }, + { + "id": 43788609651, + "pull": 4575, + "head": "09424c0a6cc6a128ea4cf4a4d5523085454349e3", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 427, + "selectDone": true, + "record": "pull request 4575\nmerge tree 0e8765f9aef3e24f883278faf081335c462f94a0\nbase 4a434124a26fd438712d84b4ff66603260f55fdb\nselection 3428b1adb8b635248c9113de2433206d47843fb0136340af636d8e0568551f66\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + }, + { + "id": 43789429081, + "pull": 4575, + "head": "09424c0a6cc6a128ea4cf4a4d5523085454349e3", + "phases": [ + { + "at": 502, + "selectDone": false + }, + { + "at": 538, + "selectDone": true, + "record": "pull request 4575\nmerge tree 0e8765f9aef3e24f883278faf081335c462f94a0\nbase 9cbea0adc2607de7d6fbabcaa35171d7171b0de8\nselection 3428b1adb8b635248c9113de2433206d47843fb0136340af636d8e0568551f66\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + }, + { + "id": 43790207238, + "pull": 4575, + "head": "09424c0a6cc6a128ea4cf4a4d5523085454349e3", + "phases": [ + { + "at": 970, + "selectDone": false + }, + { + "at": 1006, + "selectDone": true, + "record": "pull request 4575\nmerge tree 0e8765f9aef3e24f883278faf081335c462f94a0\nbase 4a434124a26fd438712d84b4ff66603260f55fdb\nselection 3428b1adb8b635248c9113de2433206d47843fb0136340af636d8e0568551f66\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + } + ] + }, + { + "lower": 4574, + "run": 43786864732, + "attempt": 1, + "ref": "ci/worker-baseline", + "head": "db47b4bf867350ca834773bffcd3c61f69328373", + "bottom": "main", + "uppers": [ + { + "pull": 4575, + "head": "ddbfad88f5f6fe9deedf3cef2ff3eee8cd26bb8c" + } + ], + "runs": [ + { + "id": 43786867470, + "pull": 4575, + "head": "ddbfad88f5f6fe9deedf3cef2ff3eee8cd26bb8c", + "phases": [ + { + "at": -25, + "selectDone": false + }, + { + "at": 10, + "selectDone": true, + "record": "pull request 4575\nmerge tree 1d4bbbe099369bc0f4a17f88851fee4168cae5fe\nbase db47b4bf867350ca834773bffcd3c61f69328373\nselection 3428b1adb8b635248c9113de2433206d47843fb0136340af636d8e0568551f66\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + }, + { + "id": 43788608976, + "pull": 4575, + "head": "09424c0a6cc6a128ea4cf4a4d5523085454349e3", + "phases": [ + { + "at": 1058, + "selectDone": false + }, + { + "at": 1093, + "selectDone": true, + "record": "pull request 4575\nmerge tree 0e8765f9aef3e24f883278faf081335c462f94a0\nbase 4a434124a26fd438712d84b4ff66603260f55fdb\nselection 3428b1adb8b635248c9113de2433206d47843fb0136340af636d8e0568551f66\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + }, + { + "id": 43788609651, + "pull": 4575, + "head": "09424c0a6cc6a128ea4cf4a4d5523085454349e3", + "phases": [ + { + "at": 1058, + "selectDone": false + }, + { + "at": 1511, + "selectDone": true, + "record": "pull request 4575\nmerge tree 0e8765f9aef3e24f883278faf081335c462f94a0\nbase 4a434124a26fd438712d84b4ff66603260f55fdb\nselection 3428b1adb8b635248c9113de2433206d47843fb0136340af636d8e0568551f66\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + }, + { + "id": 43789429081, + "pull": 4575, + "head": "09424c0a6cc6a128ea4cf4a4d5523085454349e3", + "phases": [ + { + "at": 1586, + "selectDone": false + }, + { + "at": 1622, + "selectDone": true, + "record": "pull request 4575\nmerge tree 0e8765f9aef3e24f883278faf081335c462f94a0\nbase 9cbea0adc2607de7d6fbabcaa35171d7171b0de8\nselection 3428b1adb8b635248c9113de2433206d47843fb0136340af636d8e0568551f66\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + } + ] + }, + { + "lower": 4562, + "run": 43783348587, + "attempt": 1, + "ref": "eapps-pages-page-session", + "head": "4ba8b180c72905409b218149099001133c0d5b47", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4563, + "head": "ef0de40af8a8e23b1e075879ebcda305709ca812" + } + ], + "runs": [ + { + "id": 43785160079, + "pull": 4563, + "head": "aab12205e15bf0102fe19277c59814f5330534e5", + "phases": [ + { + "at": 1071, + "selectDone": false + }, + { + "at": 1115, + "selectDone": true, + "record": "pull request 4563\nmerge tree 411ac3ae3d7ec131513869995a515d0a0a2fcc5c\ndescription sha256 0f33cfb3229643d0ca9ea48d92b913d7622bbbec49e3bd0df26564966e9d0a31\n" + } + ] + }, + { + "id": 43783349471, + "pull": 4563, + "head": "ef0de40af8a8e23b1e075879ebcda305709ca812", + "phases": [ + { + "at": -28, + "selectDone": false + }, + { + "at": 396, + "selectDone": true, + "record": "pull request 4563\nmerge tree b87db6e0516520cb8bca5906904addaac19d5e52\ndescription sha256 3fcd4e854b9ade5d4242ee647009fe19472c9eb8e971f436c23b9ad154496020\n" + } + ] + }, + { + "id": 43783349171, + "pull": 4563, + "head": "ef0de40af8a8e23b1e075879ebcda305709ca812", + "phases": [ + { + "at": -28, + "selectDone": false + }, + { + "at": 5, + "selectDone": true, + "record": "pull request 4563\nmerge tree b87db6e0516520cb8bca5906904addaac19d5e52\ndescription sha256 3fcd4e854b9ade5d4242ee647009fe19472c9eb8e971f436c23b9ad154496020\n" + } + ] + } + ] + }, + { + "lower": 4568, + "run": 43781460811, + "attempt": 1, + "ref": "rhys/runtime-connect-repeat", + "head": "7fa6f1f36ed65946f11e87980e56d1713f8d65e8", + "bottom": "main", + "uppers": [ + { + "pull": 4569, + "head": "54029152a2fce3e19dfe2f5c0a82c0c95e38cd98" + } + ], + "runs": [ + { + "id": 43781467443, + "pull": 4569, + "head": "54029152a2fce3e19dfe2f5c0a82c0c95e38cd98", + "phases": [ + { + "at": -23, + "selectDone": false + }, + { + "at": 12, + "selectDone": true, + "record": "pull request 4569\nmerge tree b71976708a0c919117bee64063a5f73ed90c69d0\nbase 7fa6f1f36ed65946f11e87980e56d1713f8d65e8\nselection 26b39ca02f1dc2b385ac7d1643aa11226aa354450273c7104cb938cc9ff1f42a\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + } + ] + }, + { + "lower": 4565, + "run": 43779277043, + "attempt": 1, + "ref": "app-links-inventory", + "head": "1328d39d2b8e5ffe3c633423bdc45e6e521f6e38", + "bottom": "main", + "uppers": [ + { + "pull": 4566, + "head": "ab6586c9551fc38ebf4b48ad1eed5aedd71993f6" + } + ], + "runs": [ + { + "id": 43779281582, + "pull": 4566, + "head": "ab6586c9551fc38ebf4b48ad1eed5aedd71993f6", + "phases": [ + { + "at": -25, + "selectDone": false + }, + { + "at": 1, + "selectDone": true + }, + { + "at": 506, + "selectDone": false + }, + { + "at": 539, + "selectDone": true, + "record": "pull request 4566\nmerge tree 5e54349dde88ede68daec2379a16c37aaf5a7ad0\nbase 1328d39d2b8e5ffe3c633423bdc45e6e521f6e38\nselection c12d68450c86ea46f14f741c55c4dc39c6ea95411933c56c983b8b9c14d8c36d\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + } + ] + }, + { + "lower": 4562, + "run": 43771920578, + "attempt": 1, + "ref": "eapps-pages-page-session", + "head": "29269c351f23c8af456e26f6406975fe44849f76", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4563, + "head": "0699a074d25ba5b8fbc7993dcfd256039762b629" + } + ], + "runs": [ + { + "id": 43771925484, + "pull": 4563, + "head": "0699a074d25ba5b8fbc7993dcfd256039762b629", + "phases": [ + { + "at": -23, + "selectDone": false + }, + { + "at": 9, + "selectDone": true, + "record": "pull request 4563\nmerge tree 83e8d3e2ed1529b72ec227a03af681a3cc1b264d\ndescription sha256 075590e161ae1057ed8d40dcc75442c6bbba9c385fb595c635cb5bb951b1dce6\n" + }, + { + "at": 580, + "selectDone": false, + "record": "pull request 4563\nmerge tree 83e8d3e2ed1529b72ec227a03af681a3cc1b264d\ndescription sha256 075590e161ae1057ed8d40dcc75442c6bbba9c385fb595c635cb5bb951b1dce6\n" + }, + { + "at": 613, + "selectDone": true, + "record": "pull request 4563\nmerge tree 83e8d3e2ed1529b72ec227a03af681a3cc1b264d\ndescription sha256 075590e161ae1057ed8d40dcc75442c6bbba9c385fb595c635cb5bb951b1dce6\n" + } + ] + } + ] + }, + { + "lower": 4548, + "run": 43771847447, + "attempt": 1, + "ref": "eapps-pages-page", + "head": "65ea2660652432a959b635454d4105bfaf05b3f9", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4549, + "head": "abeb51ca3e96b67e9ea790091ae94e4db16bbbf5" + } + ], + "runs": [ + { + "id": 43773925008, + "pull": 4549, + "head": "dae54b308c893385ab6467395665265514f55f7e", + "phases": [ + { + "at": 1283, + "selectDone": false + }, + { + "at": 1314, + "selectDone": true, + "record": "pull request 4549\nmerge tree cd63612bd20a31f3c73224112c9d69dd6458522b\ndescription sha256 b67215f7fe11f6b3e1d19c0dda377bec3f92b9a5d7c6f5f1c1e67b40428cf492\n" + } + ] + }, + { + "id": 43773926004, + "pull": 4549, + "head": "dae54b308c893385ab6467395665265514f55f7e", + "phases": [ + { + "at": 1283, + "selectDone": false + }, + { + "at": 1665, + "selectDone": true, + "record": "pull request 4549\nmerge tree cd63612bd20a31f3c73224112c9d69dd6458522b\ndescription sha256 b67215f7fe11f6b3e1d19c0dda377bec3f92b9a5d7c6f5f1c1e67b40428cf492\n" + } + ] + }, + { + "id": 43771845440, + "pull": 4549, + "head": "abeb51ca3e96b67e9ea790091ae94e4db16bbbf5", + "phases": [ + { + "at": -57, + "selectDone": false + }, + { + "at": -24, + "selectDone": true, + "record": "pull request 4549\nmerge tree cd63612bd20a31f3c73224112c9d69dd6458522b\ndescription sha256 b67215f7fe11f6b3e1d19c0dda377bec3f92b9a5d7c6f5f1c1e67b40428cf492\n" + } + ] + }, + { + "id": 43771845608, + "pull": 4549, + "head": "abeb51ca3e96b67e9ea790091ae94e4db16bbbf5", + "phases": [ + { + "at": -57, + "selectDone": false + }, + { + "at": 330, + "selectDone": true, + "record": "pull request 4549\nmerge tree cd63612bd20a31f3c73224112c9d69dd6458522b\ndescription sha256 b67215f7fe11f6b3e1d19c0dda377bec3f92b9a5d7c6f5f1c1e67b40428cf492\n" + } + ] + } + ] + }, + { + "lower": 4548, + "run": 43771846978, + "attempt": 1, + "ref": "eapps-pages-page", + "head": "65ea2660652432a959b635454d4105bfaf05b3f9", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4549, + "head": "abeb51ca3e96b67e9ea790091ae94e4db16bbbf5" + } + ], + "runs": [ + { + "id": 43773925008, + "pull": 4549, + "head": "dae54b308c893385ab6467395665265514f55f7e", + "phases": [ + { + "at": 1313, + "selectDone": false + }, + { + "at": 1344, + "selectDone": true, + "record": "pull request 4549\nmerge tree cd63612bd20a31f3c73224112c9d69dd6458522b\ndescription sha256 b67215f7fe11f6b3e1d19c0dda377bec3f92b9a5d7c6f5f1c1e67b40428cf492\n" + } + ] + }, + { + "id": 43773926004, + "pull": 4549, + "head": "dae54b308c893385ab6467395665265514f55f7e", + "phases": [ + { + "at": 1313, + "selectDone": false + }, + { + "at": 1695, + "selectDone": true, + "record": "pull request 4549\nmerge tree cd63612bd20a31f3c73224112c9d69dd6458522b\ndescription sha256 b67215f7fe11f6b3e1d19c0dda377bec3f92b9a5d7c6f5f1c1e67b40428cf492\n" + } + ] + }, + { + "id": 43771845440, + "pull": 4549, + "head": "abeb51ca3e96b67e9ea790091ae94e4db16bbbf5", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 6, + "selectDone": true, + "record": "pull request 4549\nmerge tree cd63612bd20a31f3c73224112c9d69dd6458522b\ndescription sha256 b67215f7fe11f6b3e1d19c0dda377bec3f92b9a5d7c6f5f1c1e67b40428cf492\n" + } + ] + }, + { + "id": 43771845608, + "pull": 4549, + "head": "abeb51ca3e96b67e9ea790091ae94e4db16bbbf5", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 360, + "selectDone": true, + "record": "pull request 4549\nmerge tree cd63612bd20a31f3c73224112c9d69dd6458522b\ndescription sha256 b67215f7fe11f6b3e1d19c0dda377bec3f92b9a5d7c6f5f1c1e67b40428cf492\n" + } + ] + } + ] + }, + { + "lower": 4544, + "run": 43771846802, + "attempt": 3, + "ref": "eapps-pages-supplied-tests", + "head": "8d93ac9b95553e2a000e17fca9798302e065140f", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4548, + "head": "65ea2660652432a959b635454d4105bfaf05b3f9" + } + ], + "runs": [ + { + "id": 43773906718, + "pull": 4548, + "head": "10d8fe46f7c18dc19ba50b5334112da57e34be0e", + "phases": [ + { + "at": 181, + "selectDone": false + }, + { + "at": 404, + "selectDone": true + } + ] + }, + { + "id": 43773907143, + "pull": 4548, + "head": "10d8fe46f7c18dc19ba50b5334112da57e34be0e", + "phases": [ + { + "at": 181, + "selectDone": false + }, + { + "at": 635, + "selectDone": true + } + ] + }, + { + "id": 43771846978, + "pull": 4548, + "head": "65ea2660652432a959b635454d4105bfaf05b3f9", + "phases": [ + { + "at": -1146, + "selectDone": false + }, + { + "at": -1117, + "selectDone": true, + "record": "pull request 4548\nmerge tree 28ab4500d833cce7f7e74dca6eac810d5ff84a4b\ndescription sha256 d9b82e98631eb1c65a4ddc27ecf8761d3b132b7a792e6829bca4f1534dfee1cf\n" + } + ] + }, + { + "id": 43771847447, + "pull": 4548, + "head": "65ea2660652432a959b635454d4105bfaf05b3f9", + "phases": [ + { + "at": -1146, + "selectDone": false + }, + { + "at": -1086, + "selectDone": true, + "record": "pull request 4548\nmerge tree 28ab4500d833cce7f7e74dca6eac810d5ff84a4b\ndescription sha256 d9b82e98631eb1c65a4ddc27ecf8761d3b132b7a792e6829bca4f1534dfee1cf\n" + } + ] + } + ] + }, + { + "lower": 4544, + "run": 43771846274, + "attempt": 2, + "ref": "eapps-pages-supplied-tests", + "head": "8d93ac9b95553e2a000e17fca9798302e065140f", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4548, + "head": "65ea2660652432a959b635454d4105bfaf05b3f9" + } + ], + "runs": [ + { + "id": 43773906718, + "pull": 4548, + "head": "10d8fe46f7c18dc19ba50b5334112da57e34be0e", + "phases": [ + { + "at": 648, + "selectDone": false + }, + { + "at": 871, + "selectDone": true + } + ] + }, + { + "id": 43773907143, + "pull": 4548, + "head": "10d8fe46f7c18dc19ba50b5334112da57e34be0e", + "phases": [ + { + "at": 648, + "selectDone": false + }, + { + "at": 1102, + "selectDone": true + } + ] + }, + { + "id": 43771846978, + "pull": 4548, + "head": "65ea2660652432a959b635454d4105bfaf05b3f9", + "phases": [ + { + "at": -679, + "selectDone": false + }, + { + "at": -650, + "selectDone": true, + "record": "pull request 4548\nmerge tree 28ab4500d833cce7f7e74dca6eac810d5ff84a4b\ndescription sha256 d9b82e98631eb1c65a4ddc27ecf8761d3b132b7a792e6829bca4f1534dfee1cf\n" + } + ] + }, + { + "id": 43771847447, + "pull": 4548, + "head": "65ea2660652432a959b635454d4105bfaf05b3f9", + "phases": [ + { + "at": -679, + "selectDone": false + }, + { + "at": -619, + "selectDone": true, + "record": "pull request 4548\nmerge tree 28ab4500d833cce7f7e74dca6eac810d5ff84a4b\ndescription sha256 d9b82e98631eb1c65a4ddc27ecf8761d3b132b7a792e6829bca4f1534dfee1cf\n" + } + ] + } + ] + }, + { + "lower": 4551, + "run": 43771846053, + "attempt": 1, + "ref": "eapps-pages-guide", + "head": "f148ecd6cbdf314fa26ee6a5d9819a8ae952340f", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4552, + "head": "433f11ce6f056cec7432c2cc56889ea0d5fab6bd" + } + ], + "runs": [ + { + "id": 43773957274, + "pull": 4552, + "head": "8bac4152b92066961b76ff939971213a4f44a9d7", + "phases": [ + { + "at": 1307, + "selectDone": false + }, + { + "at": 1684, + "selectDone": true, + "record": "pull request 4552\nmerge tree 87532b7370f9434c0382769bb1fe854b350c31ef\ndescription sha256 b16dfb0dc4c2fe37e781cbc32e8ba50de932d7a801830dd11b89758040d791b5\n" + } + ] + }, + { + "id": 43773956226, + "pull": 4552, + "head": "8bac4152b92066961b76ff939971213a4f44a9d7", + "phases": [ + { + "at": 1306, + "selectDone": false + }, + { + "at": 1336, + "selectDone": true, + "record": "pull request 4552\nmerge tree 87532b7370f9434c0382769bb1fe854b350c31ef\ndescription sha256 b16dfb0dc4c2fe37e781cbc32e8ba50de932d7a801830dd11b89758040d791b5\n" + } + ] + }, + { + "id": 43771845606, + "pull": 4552, + "head": "433f11ce6f056cec7432c2cc56889ea0d5fab6bd", + "phases": [ + { + "at": -55, + "selectDone": false + }, + { + "at": -23, + "selectDone": true, + "record": "pull request 4552\nmerge tree 87532b7370f9434c0382769bb1fe854b350c31ef\ndescription sha256 9aff0525bc8c18d8314ca79051c8047b4f613d70de8b691ff950173e4d3ab92f\n" + } + ] + }, + { + "id": 43771846143, + "pull": 4552, + "head": "433f11ce6f056cec7432c2cc56889ea0d5fab6bd", + "phases": [ + { + "at": -55, + "selectDone": false + }, + { + "at": 345, + "selectDone": true, + "record": "pull request 4552\nmerge tree 87532b7370f9434c0382769bb1fe854b350c31ef\ndescription sha256 b16dfb0dc4c2fe37e781cbc32e8ba50de932d7a801830dd11b89758040d791b5\n" + } + ] + } + ] + }, + { + "lower": 4543, + "run": 43771845869, + "attempt": 1, + "ref": "eapps-pages-supplied", + "head": "1dbb3e2b30928890ab7ef5c1161215a72e5c7aee", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4544, + "head": "8d93ac9b95553e2a000e17fca9798302e065140f" + } + ], + "runs": [ + { + "id": 43773889970, + "pull": 4544, + "head": "d9b24e38aa00e738fa7426e706dd7ce315763c78", + "phases": [ + { + "at": 1291, + "selectDone": false + }, + { + "at": 1744, + "selectDone": true + } + ] + }, + { + "id": 43773888274, + "pull": 4544, + "head": "d9b24e38aa00e738fa7426e706dd7ce315763c78", + "phases": [ + { + "at": 1290, + "selectDone": false + }, + { + "at": 1510, + "selectDone": true + } + ] + }, + { + "id": 43771846802, + "pull": 4544, + "head": "8d93ac9b95553e2a000e17fca9798302e065140f", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 366, + "selectDone": true, + "record": "pull request 4544\nmerge tree d798f07f911f343a6578c191b36c95e667e0a9f0\ndescription sha256 bbcf569f8e650f49c5b0652f540a67b4ae1e249fb3697e96de7a8744071a2266\n" + }, + { + "at": 1093, + "selectDone": false, + "record": "pull request 4544\nmerge tree d798f07f911f343a6578c191b36c95e667e0a9f0\ndescription sha256 bbcf569f8e650f49c5b0652f540a67b4ae1e249fb3697e96de7a8744071a2266\n" + }, + { + "at": 1121, + "selectDone": true, + "record": "pull request 4544\nmerge tree d798f07f911f343a6578c191b36c95e667e0a9f0\ndescription sha256 bbcf569f8e650f49c5b0652f540a67b4ae1e249fb3697e96de7a8744071a2266\n" + } + ] + }, + { + "id": 43771846274, + "pull": 4544, + "head": "8d93ac9b95553e2a000e17fca9798302e065140f", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 7, + "selectDone": true, + "record": "pull request 4544\nmerge tree d798f07f911f343a6578c191b36c95e667e0a9f0\ndescription sha256 bbcf569f8e650f49c5b0652f540a67b4ae1e249fb3697e96de7a8744071a2266\n" + }, + { + "at": 626, + "selectDone": false, + "record": "pull request 4544\nmerge tree d798f07f911f343a6578c191b36c95e667e0a9f0\ndescription sha256 bbcf569f8e650f49c5b0652f540a67b4ae1e249fb3697e96de7a8744071a2266\n" + }, + { + "at": 654, + "selectDone": true, + "record": "pull request 4544\nmerge tree d798f07f911f343a6578c191b36c95e667e0a9f0\ndescription sha256 bbcf569f8e650f49c5b0652f540a67b4ae1e249fb3697e96de7a8744071a2266\n" + } + ] + } + ] + }, + { + "lower": 4551, + "run": 43771845232, + "attempt": 1, + "ref": "eapps-pages-guide", + "head": "f148ecd6cbdf314fa26ee6a5d9819a8ae952340f", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4552, + "head": "433f11ce6f056cec7432c2cc56889ea0d5fab6bd" + } + ], + "runs": [ + { + "id": 43773957274, + "pull": 4552, + "head": "8bac4152b92066961b76ff939971213a4f44a9d7", + "phases": [ + { + "at": 1336, + "selectDone": false + }, + { + "at": 1713, + "selectDone": true, + "record": "pull request 4552\nmerge tree 87532b7370f9434c0382769bb1fe854b350c31ef\ndescription sha256 b16dfb0dc4c2fe37e781cbc32e8ba50de932d7a801830dd11b89758040d791b5\n" + } + ] + }, + { + "id": 43773956226, + "pull": 4552, + "head": "8bac4152b92066961b76ff939971213a4f44a9d7", + "phases": [ + { + "at": 1335, + "selectDone": false + }, + { + "at": 1365, + "selectDone": true, + "record": "pull request 4552\nmerge tree 87532b7370f9434c0382769bb1fe854b350c31ef\ndescription sha256 b16dfb0dc4c2fe37e781cbc32e8ba50de932d7a801830dd11b89758040d791b5\n" + } + ] + }, + { + "id": 43771845606, + "pull": 4552, + "head": "433f11ce6f056cec7432c2cc56889ea0d5fab6bd", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 6, + "selectDone": true, + "record": "pull request 4552\nmerge tree 87532b7370f9434c0382769bb1fe854b350c31ef\ndescription sha256 9aff0525bc8c18d8314ca79051c8047b4f613d70de8b691ff950173e4d3ab92f\n" + } + ] + }, + { + "id": 43771846143, + "pull": 4552, + "head": "433f11ce6f056cec7432c2cc56889ea0d5fab6bd", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 374, + "selectDone": true, + "record": "pull request 4552\nmerge tree 87532b7370f9434c0382769bb1fe854b350c31ef\ndescription sha256 b16dfb0dc4c2fe37e781cbc32e8ba50de932d7a801830dd11b89758040d791b5\n" + } + ] + } + ] + }, + { + "lower": 4559, + "run": 43770386335, + "attempt": 1, + "ref": "repo-lazy-create", + "head": "5bd82330e775c38f879f7c8a5efe95e890f69362", + "bottom": "main", + "uppers": [ + { + "pull": 4560, + "head": "a56247a40df6e886dcf9e099df7c7d0cd2b575a9" + } + ], + "runs": [ + { + "id": 43770390767, + "pull": 4560, + "head": "a56247a40df6e886dcf9e099df7c7d0cd2b575a9", + "phases": [ + { + "at": -24, + "selectDone": false + }, + { + "at": 23, + "selectDone": true, + "record": "pull request 4560\nmerge tree 5c4944512295962cb4b84db8f6ee6db2f0c96cfb\nbase 5bd82330e775c38f879f7c8a5efe95e890f69362\nselection 1fbf4ea5f7d416fb2fe3f49947d2c945981d261f5a48e5fc106d2bfa245f22e3\nworker-growth 5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb\n" + } + ] + } + ] + }, + { + "lower": 4436, + "run": 43768234280, + "attempt": 1, + "ref": "app-scale/4-search-discovery", + "head": "8edfb7cbab42aab0bdc7b115ffcc0243d581f19f", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4437, + "head": "80866dd0f4cc157dac43b746828e6fdab9b63629" + } + ], + "runs": [ + { + "id": 43768232548, + "pull": 4437, + "head": "80866dd0f4cc157dac43b746828e6fdab9b63629", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 3, + "selectDone": true, + "record": "pull request 4437\nmerge tree a9e3828b81a4370cb9516750119461301532c180\ndescription sha256 4588350f9492220c842e07cab54124a837501e8ebc775c13be02172bfc2e4cbc\n" + } + ] + }, + { + "id": 43768233846, + "pull": 4437, + "head": "80866dd0f4cc157dac43b746828e6fdab9b63629", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 33, + "selectDone": true, + "record": "pull request 4437\nmerge tree a9e3828b81a4370cb9516750119461301532c180\ndescription sha256 4588350f9492220c842e07cab54124a837501e8ebc775c13be02172bfc2e4cbc\n" + } + ] + } + ] + }, + { + "lower": 4437, + "run": 43768233846, + "attempt": 1, + "ref": "app-scale/5-rejected-accounts", + "head": "80866dd0f4cc157dac43b746828e6fdab9b63629", + "bottom": "app-scale/4-search-discovery", + "uppers": [ + { + "pull": 4438, + "head": "3cfef7b353cfcd174ed8e7431931d151d6fc2d1c" + } + ], + "runs": [ + { + "id": 43768232720, + "pull": 4438, + "head": "3cfef7b353cfcd174ed8e7431931d151d6fc2d1c", + "phases": [ + { + "at": -56, + "selectDone": false + }, + { + "at": -26, + "selectDone": true, + "record": "pull request 4438\nmerge tree 6adecef33fec8630adc6dfcc9d8f31129930196c\ndescription sha256 d2cbadefda39a12f6bcd4614985e959fe9a56b79dc40bd8ed70b2356accf9a17\n" + } + ] + }, + { + "id": 43768233035, + "pull": 4438, + "head": "3cfef7b353cfcd174ed8e7431931d151d6fc2d1c", + "phases": [ + { + "at": -55, + "selectDone": false + }, + { + "at": 5, + "selectDone": true, + "record": "pull request 4438\nmerge tree 6adecef33fec8630adc6dfcc9d8f31129930196c\ndescription sha256 d2cbadefda39a12f6bcd4614985e959fe9a56b79dc40bd8ed70b2356accf9a17\n" + } + ] + } + ] + }, + { + "lower": 4438, + "run": 43768233035, + "attempt": 1, + "ref": "app-scale/6-listing-counts", + "head": "3cfef7b353cfcd174ed8e7431931d151d6fc2d1c", + "bottom": "app-scale/5-rejected-accounts", + "uppers": [ + { + "pull": 4439, + "head": "2715af9e0c3e4175001faf3b2fddad0cfdda424f" + } + ], + "runs": [ + { + "id": 43768232499, + "pull": 4439, + "head": "2715af9e0c3e4175001faf3b2fddad0cfdda424f", + "phases": [ + { + "at": -57, + "selectDone": false + }, + { + "at": -24, + "selectDone": true, + "record": "pull request 4439\nmerge tree e73346d4427bbd45264599b7745557563634ae14\ndescription sha256 208abda877bb3c0561046e53473075467ac819bdd18df5f4df1d9749bbe84a46\n" + } + ] + }, + { + "id": 43768233024, + "pull": 4439, + "head": "2715af9e0c3e4175001faf3b2fddad0cfdda424f", + "phases": [ + { + "at": -56, + "selectDone": false + }, + { + "at": 343, + "selectDone": true, + "record": "pull request 4439\nmerge tree e73346d4427bbd45264599b7745557563634ae14\ndescription sha256 208abda877bb3c0561046e53473075467ac819bdd18df5f4df1d9749bbe84a46\n" + } + ] + } + ] + }, + { + "lower": 4438, + "run": 43768232720, + "attempt": 1, + "ref": "app-scale/6-listing-counts", + "head": "3cfef7b353cfcd174ed8e7431931d151d6fc2d1c", + "bottom": "app-scale/5-rejected-accounts", + "uppers": [ + { + "pull": 4439, + "head": "2715af9e0c3e4175001faf3b2fddad0cfdda424f" + } + ], + "runs": [ + { + "id": 43768232499, + "pull": 4439, + "head": "2715af9e0c3e4175001faf3b2fddad0cfdda424f", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 6, + "selectDone": true, + "record": "pull request 4439\nmerge tree e73346d4427bbd45264599b7745557563634ae14\ndescription sha256 208abda877bb3c0561046e53473075467ac819bdd18df5f4df1d9749bbe84a46\n" + } + ] + }, + { + "id": 43768233024, + "pull": 4439, + "head": "2715af9e0c3e4175001faf3b2fddad0cfdda424f", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 373, + "selectDone": true, + "record": "pull request 4439\nmerge tree e73346d4427bbd45264599b7745557563634ae14\ndescription sha256 208abda877bb3c0561046e53473075467ac819bdd18df5f4df1d9749bbe84a46\n" + } + ] + } + ] + }, + { + "lower": 4437, + "run": 43768232548, + "attempt": 1, + "ref": "app-scale/5-rejected-accounts", + "head": "80866dd0f4cc157dac43b746828e6fdab9b63629", + "bottom": "app-scale/4-search-discovery", + "uppers": [ + { + "pull": 4438, + "head": "3cfef7b353cfcd174ed8e7431931d151d6fc2d1c" + } + ], + "runs": [ + { + "id": 43768232720, + "pull": 4438, + "head": "3cfef7b353cfcd174ed8e7431931d151d6fc2d1c", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 3, + "selectDone": true, + "record": "pull request 4438\nmerge tree 6adecef33fec8630adc6dfcc9d8f31129930196c\ndescription sha256 d2cbadefda39a12f6bcd4614985e959fe9a56b79dc40bd8ed70b2356accf9a17\n" + } + ] + }, + { + "id": 43768233035, + "pull": 4438, + "head": "3cfef7b353cfcd174ed8e7431931d151d6fc2d1c", + "phases": [ + { + "at": -26, + "selectDone": false + }, + { + "at": 34, + "selectDone": true, + "record": "pull request 4438\nmerge tree 6adecef33fec8630adc6dfcc9d8f31129930196c\ndescription sha256 d2cbadefda39a12f6bcd4614985e959fe9a56b79dc40bd8ed70b2356accf9a17\n" + } + ] + } + ] + }, + { + "lower": 4557, + "run": 43764101615, + "attempt": 1, + "ref": "durable/cron-ops", + "head": "1c24039a356380bab2b2843c0aff1bff9367e5a9", + "bottom": "durable/cron-rules-tests", + "uppers": [ + { + "pull": 4558, + "head": "9ebca85b6ca971c6a1d84d5e0236da8aa3ab5d31" + } + ], + "runs": [ + { + "id": 43764104967, + "pull": 4558, + "head": "9ebca85b6ca971c6a1d84d5e0236da8aa3ab5d31", + "phases": [ + { + "at": -24, + "selectDone": false + }, + { + "at": 9, + "selectDone": true, + "record": "pull request 4558\nmerge tree a13dd4c01ea77833867d4254a500e2e9b3d064da\ndescription sha256 6ec47cd13aa6d8608cd15c9bec8b8af1f9459c9e320da2135b3f4a66c9dd6b24\n" + } + ] + }, + { + "id": 43765747857, + "pull": 4558, + "head": "8eec3b720fd05d57b5317248710a102678418523", + "phases": [ + { + "at": 1046, + "selectDone": false + }, + { + "at": 1079, + "selectDone": true, + "record": "pull request 4558\nmerge tree 3ba500fd61c3187197fac7214e8b677f52c63c47\ndescription sha256 cad92e29bd70a0b6cf6f968f398f95fea47d331a046c9636e6eb8f9e399ed4ea\n" + } + ] + } + ] + }, + { + "lower": 4526, + "run": 43764086735, + "attempt": 1, + "ref": "durable/cron-scheduler", + "head": "4bc48ca94f25bb8d4c2d1b49696e36299e45678d", + "bottom": "effect-apps", + "uppers": [ + { + "pull": 4527, + "head": "766d0bd190ac0a672201635da3af034ee8584ee4" + } + ], + "runs": [ + { + "id": 43764086251, + "pull": 4527, + "head": "766d0bd190ac0a672201635da3af034ee8584ee4", + "phases": [ + { + "at": -28, + "selectDone": false + }, + { + "at": 5, + "selectDone": true, + "record": "pull request 4527\nmerge tree 46745a42800b9d60221098965c3b6e6c84311789\ndescription sha256 6563e8be33d64486e892fdaf86a34884c87053a5af56f878a71ae3652088b3f8\n" + } + ] + }, + { + "id": 43764087596, + "pull": 4527, + "head": "766d0bd190ac0a672201635da3af034ee8584ee4", + "phases": [ + { + "at": -27, + "selectDone": false + }, + { + "at": 367, + "selectDone": true, + "record": "pull request 4527\nmerge tree 46745a42800b9d60221098965c3b6e6c84311789\ndescription sha256 41a2dbc07c23e6d72de28d624bb77089549116d9ef6c12505bd07d06d24cd209\n" + } + ] + } + ] + }, + { + "lower": 4535, + "run": 43764086626, + "attempt": 1, + "ref": "durable/cron-rules", + "head": "b9b2e3e26d359962440d05db53dc241a52fb7da1", + "bottom": "durable/cron-scheduler-tests", + "uppers": [ + { + "pull": 4536, + "head": "f16d345d49a2023111d8754ef14b873bdb563be1" + } + ], + "runs": [ + { + "id": 43764086232, + "pull": 4536, + "head": "f16d345d49a2023111d8754ef14b873bdb563be1", + "phases": [ + { + "at": -57, + "selectDone": false + }, + { + "at": 338, + "selectDone": true, + "record": "pull request 4536\nmerge tree 31cf157aa63df38de5a12c3ecebc3333f688b3da\ndescription sha256 1bdc79149185ec9f759c77c315ed99526c91b32576ddf146565afe5b44a9e415\n" + } + ] + }, + { + "id": 43764085653, + "pull": 4536, + "head": "f16d345d49a2023111d8754ef14b873bdb563be1", + "phases": [ + { + "at": -57, + "selectDone": false + }, + { + "at": -24, + "selectDone": true, + "record": "pull request 4536\nmerge tree 31cf157aa63df38de5a12c3ecebc3333f688b3da\ndescription sha256 81ecb891c666bb9110d1df62d3623ca80e1282949c24df92d4b6704d537c341e\n" + } + ] + } + ] + }, + { + "lower": 4535, + "run": 43764086047, + "attempt": 1, + "ref": "durable/cron-rules", + "head": "b9b2e3e26d359962440d05db53dc241a52fb7da1", + "bottom": "durable/cron-scheduler-tests", + "uppers": [ + { + "pull": 4536, + "head": "f16d345d49a2023111d8754ef14b873bdb563be1" + } + ], + "runs": [ + { + "id": 43764086232, + "pull": 4536, + "head": "f16d345d49a2023111d8754ef14b873bdb563be1", + "phases": [ + { + "at": -28, + "selectDone": false + }, + { + "at": 367, + "selectDone": true, + "record": "pull request 4536\nmerge tree 31cf157aa63df38de5a12c3ecebc3333f688b3da\ndescription sha256 1bdc79149185ec9f759c77c315ed99526c91b32576ddf146565afe5b44a9e415\n" + } + ] + }, + { + "id": 43764085653, + "pull": 4536, + "head": "f16d345d49a2023111d8754ef14b873bdb563be1", + "phases": [ + { + "at": -28, + "selectDone": false + }, + { + "at": 5, + "selectDone": true, + "record": "pull request 4536\nmerge tree 31cf157aa63df38de5a12c3ecebc3333f688b3da\ndescription sha256 81ecb891c666bb9110d1df62d3623ca80e1282949c24df92d4b6704d537c341e\n" + } + ] + } + ] + } +] diff --git a/e2e/support/app-package.ts b/e2e/support/app-package.ts index 07073c84e4..e222ff675c 100644 --- a/e2e/support/app-package.ts +++ b/e2e/support/app-package.ts @@ -1,6 +1,6 @@ /** Serve real npm archives through a scoped local registry boundary; no npm publication is needed. */ -import { createServer } from "node:http"; -import { Effect, FileSystem, Path, Schema } from "effect"; +import { createServer, request as httpRequest } from "node:http"; +import { Config, Effect, FileSystem, Path, Schema } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { lockedArchive } from "./npm-registry.ts"; @@ -33,6 +33,9 @@ class PackageFixtureFailed extends Schema.TaggedError()( /** A published archive as `e2e:prepare` downloaded it, refused unless it matches its integrity. */ const publishedArchive = (version: Release = "0.0.1-beta.0") => lockedArchive("apps", version); +/** The reason phrase of a missing archive's 404, which a build failure must not repeat. */ +export const missingArchiveReason = "Archive withheld by its host"; + /** Serve archives on loopback for the scope, counting each request path. */ const loopbackArchives = Effect.gen(function* () { const archives = new Map(); @@ -45,9 +48,12 @@ const loopbackArchives = Effect.gen(function* () { const route = request.url ?? ""; requests.set(route, (requests.get(route) ?? 0) + 1); const bytes = archives.get(route); - response.writeHead(bytes === undefined ? 404 : 200, { - "Content-Type": "application/octet-stream", - }); + if (bytes === undefined) { + response.writeHead(404, missingArchiveReason, { "Content-Type": "text/plain" }); + response.end(`${route} is not served here`); + return; + } + response.writeHead(200, { "Content-Type": "application/octet-stream" }); response.end(bytes); }); server.once("error", reject); @@ -162,3 +168,156 @@ export const publishedRelease = (version: Release) => archives.set(route, yield* publishedArchive(version)); return { url: `${base}${route}`, route, requests }; }); + +/** Text a registry sends that no build failure may repeat. */ +export const upstreamMarker = "UPSTREAM-REGISTRY-TEXT"; + +/** + * A private registry on loopback, standing in for an operator's own, beside archives on a second + * loopback origin. Packages: + * + * - `registry-fixture`: its tarball is on the registry's origin. + * - `offsite-fixture`: its metadata names a tarball on the other origin. + * - `redirected-fixture`: its tarball URL is on the registry's origin and redirects to the other. + * - `nested-fixture`: on the registry's origin, depending on a tarball URL on the other origin. + * - `malformed-fixture`, `versionless-fixture` and `manifest-fixture`: metadata that is not JSON, + * that names no tarball, and that lists a dependency by a name and range of the registry's own. + * + * Every other request goes to the suite's registry, addressed as this registry, so its archive + * links (`apps` among them) point back here. + */ +export const privateRegistryFixture = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const suite = new URL(yield* Config.String("E2E_NPM_REGISTRY")); + const directory = yield* fs.makeTempDirectoryScoped(); + const offsite = yield* loopbackArchives; + const pack = (name: string, content: string, dependencies: Record = {}) => + Effect.gen(function* () { + const source = yield* fs.makeTempDirectoryScoped(); + yield* fs.writeFileString( + path.join(source, "package.json"), + JSON.stringify({ name, version: "1.0.0", type: "module", main: "index.js", dependencies }), + ); + yield* fs.writeFileString(path.join(source, "index.js"), content); + return yield* packDirectory(source, directory); + }); + const offsiteUrl = `${offsite.base}/offsite-fixture.tgz`; + offsite.archives.set( + "/offsite-fixture.tgz", + yield* pack("offsite-fixture", 'export const value = "offsite-package";'), + ); + const archives = new Map([ + [ + "/registry-fixture/-/registry-fixture-1.0.0.tgz", + yield* pack("registry-fixture", 'export const value = "registry-package";'), + ], + [ + "/nested-fixture/-/nested-fixture-1.0.0.tgz", + yield* pack("nested-fixture", 'export { value } from "offsite-fixture";', { + "offsite-fixture": offsiteUrl, + }), + ], + [ + "/manifest-fixture/-/manifest-fixture-1.0.0.tgz", + yield* pack("manifest-fixture", 'export const value = "manifest-package";'), + ], + ]); + const version = ( + name: string, + tarball: string | undefined, + dependencies?: Record, + ) => ({ + name, + "dist-tags": { latest: "1.0.0" }, + versions: { + "1.0.0": { + name, + version: tarball === undefined ? upstreamMarker : "1.0.0", + ...(dependencies === undefined ? {} : { dependencies }), + ...(tarball === undefined ? {} : { dist: { tarball } }), + }, + }, + }); + const requests = new Map(); + const server = yield* Effect.acquireRelease( + Effect.tryPromise({ + try: () => + new Promise>((resolve, reject) => { + const server = createServer((request, response) => { + const route = request.url ?? "/"; + requests.set(route, (requests.get(route) ?? 0) + 1); + const base = `http://${request.headers.host ?? ""}`; + const metadata: Record = { + "/registry-fixture": version( + "registry-fixture", + `${base}/registry-fixture/-/registry-fixture-1.0.0.tgz`, + ), + "/offsite-fixture": version("offsite-fixture", offsiteUrl), + "/redirected-fixture": version( + "redirected-fixture", + `${base}/redirected-fixture.tgz`, + ), + "/nested-fixture": version( + "nested-fixture", + `${base}/nested-fixture/-/nested-fixture-1.0.0.tgz`, + { "offsite-fixture": offsiteUrl }, + ), + "/versionless-fixture": version("versionless-fixture", undefined), + "/manifest-fixture": version( + "manifest-fixture", + `${base}/manifest-fixture/-/manifest-fixture-1.0.0.tgz`, + { [`${upstreamMarker.toLowerCase()}-name`]: `${upstreamMarker}-range` }, + ), + }; + const archive = archives.get(route); + if (archive !== undefined) { + response.writeHead(200, { "Content-Type": "application/octet-stream" }); + response.end(archive); + } else if (route === "/redirected-fixture.tgz") { + response.writeHead(302, { Location: offsiteUrl }); + response.end(); + } else if (route === "/malformed-fixture") { + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(`{"versions": ${upstreamMarker}`); + } else if (metadata[route] !== undefined) { + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify(metadata[route])); + } else { + const forwarded = httpRequest( + { + host: suite.hostname, + port: suite.port, + path: route, + headers: { host: request.headers.host ?? "", accept: "application/json" }, + }, + (incoming) => { + response.writeHead(incoming.statusCode ?? 502, incoming.headers); + incoming.pipe(response); + }, + ); + forwarded.once("error", () => { + response.writeHead(502); + response.end(); + }); + forwarded.end(); + } + }); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve(server)); + }), + catch: () => new PackageFixtureFailed({ reason: "private registry listener" }), + }), + (server) => Effect.promise(() => new Promise((resolve) => server.close(() => resolve()))), + ); + const address = server.address(); + if (address === null || typeof address === "string") + return yield* new PackageFixtureFailed({ reason: "private registry address" }); + return { + url: `http://127.0.0.1:${address.port}`, + /** Requests to the registry's origin, by path. */ + requests: Effect.sync(() => Object.fromEntries(requests)), + /** Requests to the other origin, by path. */ + offsite: offsite.requests, + }; +}); diff --git a/e2e/support/axiom.ts b/e2e/support/axiom.ts index e372140783..5984cdd110 100644 --- a/e2e/support/axiom.ts +++ b/e2e/support/axiom.ts @@ -114,118 +114,154 @@ const attributeColumn = (name: string) => ? `['attributes.${name}']` : `['attributes.custom']['${name}']`; +/** The longest one Axiom query may take. */ +export const axiomQueryMs = 10_000; + +/** The stage name its spans carry, `test-`, from the stage's origin `https://.`. */ +export const stageEnvironment = (origin: string) => { + const host = URL.parse(origin)?.hostname ?? ""; + const slug = host.split(".")[0] ?? ""; + return /^[a-z0-9-]+$/u.test(slug) ? `test-${slug}` : undefined; +}; + /** - * Queries remain limited to the current run's time window and to filters built from validated - * terms: one trace, or one operation whose attributes match exactly, in one trace or any. + * The stage name of an attached stage, needed by every span read; an origin that does not name its + * stage is a configuration error, raised once when the run starts rather than in each scenario. */ -export const axiomSpans = Effect.gen(function* () { - const client = yield* HttpClient.HttpClient; - const start = yield* Clock.currentTimeMillis; - const query = (filter: string) => - Effect.gen(function* () { - const token = yield* secretConfig("E2E_AXIOM_TOKEN"); - const organization = yield* Config.option(Config.NonEmptyString("E2E_AXIOM_ORG_ID")); - const dataset = yield* Config.String("E2E_AXIOM_DATASET").pipe( - Effect.flatMap( - Schema.decodeUnknownEffect(Schema.String.check(Schema.isPattern(/^[a-zA-Z0-9_-]+$/u))), - ), - ); - const now = yield* Clock.currentTimeMillis; - const request = yield* HttpClientRequest.post( - "https://api.axiom.co/v1/datasets/_apl?format=tabular", - ).pipe( - HttpClientRequest.setHeader("authorization", `Bearer ${reveal(token)}`), - HttpClientRequest.setHeaders( - Option.isSome(organization) ? { "x-axiom-org-id": organization.value } : {}, +export const attachedStage = (origin: string) => { + const stage = stageEnvironment(origin); + return stage === undefined + ? Effect.die( + new Error( + `The attached stage's origin ${origin} does not name its stage as https://., so its spans cannot be read from Axiom.`, ), - HttpClientRequest.bodyJson({ - apl: `['${dataset}'] | where ${filter} | project traceId=trace_id, spanId=span_id, parentSpanId=parent_span_id, operationName=name, serviceName=['service.name'], startTime=_time, durationMs=duration/1ms, status=['status.code'], tags=['attributes.custom'], build=['resource.custom']['executor.build.id'], links, events=column_ifexists('events', dynamic(null)), statusMessage=column_ifexists('status.message', ''), standard=${standard} | take 5000`, - startTime: new Date(start - 60_000).toISOString(), - endTime: new Date(now + 60_000).toISOString(), - }), - ); - const response = yield* client.pipe(HttpClient.filterStatusOk).execute(request); - const payload = yield* response.json.pipe( - Effect.flatMap(Schema.decodeUnknownEffect(Tabular)), - ); - if (payload.status.isPartial) - return yield* Effect.fail(new Error("Axiom returned partial trace data")); - const rows = []; - for (const table of payload.tables) { - if (table.fields.length !== table.columns.length) - return yield* Effect.fail(new Error("Axiom returned inconsistent columns")); - const count = table.columns[0]?.length ?? 0; - if (count === 5000 || table.columns.some((column) => column.length !== count)) - return yield* Effect.fail(new Error("Axiom span result is incomplete")); - for (let row = 0; row < count; row++) { - const value = Object.fromEntries( - table.fields.map((field, index) => [field.name, table.columns[index]?.[row]]), - ); - rows.push(yield* Schema.decodeUnknownEffect(Row)(value)); + ) + : Effect.succeed(stage); +}; + +/** + * Queries remain limited to the current run's time window, to this run's own stage, and to + * filters built from validated terms: one trace, or one operation whose attributes match exactly, + * in one trace or any. Every test stage shares one dataset, so a search that named no stage would + * read other runs' spans. + */ +export const axiomSpans = (environment: string | undefined) => + Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + const start = yield* Clock.currentTimeMillis; + const query = (filter: string, since?: number) => + Effect.gen(function* () { + const stage = yield* Schema.decodeUnknownEffect(Term)(environment ?? "").pipe( + Effect.mapError(() => new Error("Axiom reads need this run's stage")), + ); + const token = yield* secretConfig("E2E_AXIOM_TOKEN"); + const organization = yield* Config.option(Config.NonEmptyString("E2E_AXIOM_ORG_ID")); + const dataset = yield* Config.String("E2E_AXIOM_DATASET").pipe( + Effect.flatMap( + Schema.decodeUnknownEffect(Schema.String.check(Schema.isPattern(/^[a-zA-Z0-9_-]+$/u))), + ), + ); + const now = yield* Clock.currentTimeMillis; + const request = yield* HttpClientRequest.post( + "https://api.axiom.co/v1/datasets/_apl?format=tabular", + ).pipe( + HttpClientRequest.setHeader("authorization", `Bearer ${reveal(token)}`), + HttpClientRequest.setHeaders( + Option.isSome(organization) ? { "x-axiom-org-id": organization.value } : {}, + ), + HttpClientRequest.bodyJson({ + apl: `['${dataset}'] | where ['resource.deployment.environment.name'] == '${stage}' and ${filter} | project traceId=trace_id, spanId=span_id, parentSpanId=parent_span_id, operationName=name, serviceName=['service.name'], startTime=_time, durationMs=duration/1ms, status=['status.code'], tags=['attributes.custom'], build=['resource.custom']['executor.build.id'], links, events=column_ifexists('events', dynamic(null)), statusMessage=column_ifexists('status.message', ''), standard=${standard} | take 5000`, + startTime: new Date(since ?? start - 60_000).toISOString(), + endTime: new Date(now + 60_000).toISOString(), + }), + ); + const response = yield* client.pipe(HttpClient.filterStatusOk).execute(request); + const payload = yield* response.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Tabular)), + ); + if (payload.status.isPartial) + return yield* Effect.fail(new Error("Axiom returned partial trace data")); + const rows = []; + for (const table of payload.tables) { + if (table.fields.length !== table.columns.length) + return yield* Effect.fail(new Error("Axiom returned inconsistent columns")); + const count = table.columns[0]?.length ?? 0; + if (count === 5000 || table.columns.some((column) => column.length !== count)) + return yield* Effect.fail(new Error("Axiom span result is incomplete")); + for (let row = 0; row < count; row++) { + const value = Object.fromEntries( + table.fields.map((field, index) => [field.name, table.columns[index]?.[row]]), + ); + rows.push(yield* Schema.decodeUnknownEffect(Row)(value)); + } } - } - return yield* Schema.decodeUnknownEffect(SpanQuery)({ - data: rows.map((row) => ({ - traceId: row.traceId, - span: { - spanId: row.spanId, - parentSpanId: row.parentSpanId === "" ? null : row.parentSpanId, - operationName: row.operationName, - serviceName: row.serviceName, - startTime: row.startTime, - durationMs: row.durationMs, - status: row.status === "Error" || row.status === "ERROR" ? "error" : "ok", - links: (row.links ?? []).map((link) => ({ - traceId: link.trace_id, - spanId: link.span_id, - })), - events: (row.events ?? []).map((event) => ({ - name: event.name, - attributes: text(event.attributes ?? {}), - })), - ...(row.statusMessage === null || row.statusMessage === "" - ? {} - : { statusMessage: row.statusMessage }), - tags: { - ...text({ ...row.tags, ...row.standard }), - ...(row.build === null ? {} : { "executor.build.id": row.build }), + return yield* Schema.decodeUnknownEffect(SpanQuery)({ + data: rows.map((row) => ({ + traceId: row.traceId, + span: { + spanId: row.spanId, + parentSpanId: row.parentSpanId === "" ? null : row.parentSpanId, + operationName: row.operationName, + serviceName: row.serviceName, + startTime: row.startTime, + durationMs: row.durationMs, + status: row.status === "Error" || row.status === "ERROR" ? "error" : "ok", + links: (row.links ?? []).map((link) => ({ + traceId: link.trace_id, + spanId: link.span_id, + })), + events: (row.events ?? []).map((event) => ({ + name: event.name, + attributes: text(event.attributes ?? {}), + })), + ...(row.statusMessage === null || row.statusMessage === "" + ? {} + : { statusMessage: row.statusMessage }), + tags: { + ...text({ ...row.tags, ...row.standard }), + ...(row.build === null ? {} : { "executor.build.id": row.build }), + }, }, - }, - })), - }); - }).pipe(Effect.scoped, Effect.timeout("10 seconds")); - return { - trace: (traceId: string) => - Schema.decodeUnknownEffect(Schema.String.check(Schema.isPattern(/^[a-f0-9]{32}$/u)))( - traceId, - ).pipe(Effect.flatMap((id) => query(`trace_id == '${id}'`))), - /** One operation whose attributes match exactly, in one trace when `traceId` names one. */ - search: ( - operation: string, - attributes: Readonly>, - traceId?: string | undefined, - ) => - Effect.forEach([operation, ...Object.entries(attributes).flat()], (term) => - Schema.decodeUnknownEffect(Term)(term), - ).pipe( - Effect.andThen( - traceId === undefined - ? Effect.succeed(undefined) - : Schema.decodeUnknownEffect(Schema.String.check(Schema.isPattern(/^[a-f0-9]{32}$/u)))( - traceId, - ), - ), - Effect.flatMap((trace) => - query( - [ - ...(trace === undefined ? [] : [`trace_id == '${trace}'`]), - `name == '${operation}'`, - ...Object.entries(attributes).map( - ([name, value]) => `tostring(${attributeColumn(name)}) == '${value}'`, - ), - ].join(" and "), + })), + }); + }).pipe(Effect.scoped, Effect.timeout(axiomQueryMs)); + return { + trace: (traceId: string) => + Schema.decodeUnknownEffect(Schema.String.check(Schema.isPattern(/^[a-f0-9]{32}$/u)))( + traceId, + ).pipe(Effect.flatMap((id) => query(`trace_id == '${id}'`))), + /** + * One operation whose attributes match exactly, in one trace when `traceId` names one, from + * `since` (epoch milliseconds) when given, else over the run's window. + */ + search: ( + operation: string, + attributes: Readonly>, + traceId?: string | undefined, + since?: number, + ) => + Effect.forEach([operation, ...Object.entries(attributes).flat()], (term) => + Schema.decodeUnknownEffect(Term)(term), + ).pipe( + Effect.andThen( + traceId === undefined + ? Effect.succeed(undefined) + : Schema.decodeUnknownEffect( + Schema.String.check(Schema.isPattern(/^[a-f0-9]{32}$/u)), + )(traceId), + ), + Effect.flatMap((trace) => + query( + [ + ...(trace === undefined ? [] : [`trace_id == '${trace}'`]), + `name == '${operation}'`, + ...Object.entries(attributes).map( + ([name, value]) => `tostring(${attributeColumn(name)}) == '${value}'`, + ), + ].join(" and "), + since, + ), ), ), - ), - }; -}); + }; + }); diff --git a/e2e/support/browser.ts b/e2e/support/browser.ts index e74ff58b44..53948bff34 100644 --- a/e2e/support/browser.ts +++ b/e2e/support/browser.ts @@ -5,6 +5,7 @@ import { type Browser as NativeBrowser, type BrowserType, type Page, + type Request, } from "playwright"; import { Cause, @@ -92,6 +93,31 @@ export class BrowserDriver extends Context.Service static readonly webkitCaptureLayer = Layer.effect(BrowserDriver, captureBrowser(webkit)); } /** A case-scoped browser with captured steps and a single isolated context. */ +/** + * One request a scenario's browser sent, in `page-requests.json`: its URL without query or fragment + * (they carry codes and state), when it started, when its response arrived and when it ended. A + * redirect's hops are separate rows; a request still open when the scenario ended has no `endedAt`. + */ +interface PageRequest { + readonly method: string; + readonly url: string; + readonly kind: string; + readonly startedAt: string; + status?: number; + respondedAt?: string; + endedAt?: string; + error?: string; +} + +/** The most rows `page-requests.json` keeps; a scenario's pages send a few hundred. */ +const pageRequestLimit = 5000; + +/** A Chromium network error code, or `NETWORK_ERROR` for anything else. */ +const networkError = (request: Request) => { + const reason = request.failure()?.errorText; + return reason !== undefined && /^net::ERR_[A-Z_]+$/.test(reason) ? reason : "NETWORK_ERROR"; +}; + export class Browser extends Context.Service< Browser, { @@ -149,6 +175,44 @@ export class Browser extends Context.Service< if (pageErrors.length < 200) pageErrors.push({ kind, page: safeText(from?.url() ?? "unknown"), text: safeText(text) }); }; + // Every request of every page, kept even when an OAuth scenario discards the network trace, + // so a page that waited shows which request it waited on and for how long. + const pageRequests: PageRequest[] = []; + const openRequests = new Map(); + let requestsClosed = false; + const now = () => new Date(clock.currentTimeMillisUnsafe()).toISOString(); + context.on("request", (request) => { + const url = new URL(request.url()); + if (requestsClosed || pageRequests.length >= pageRequestLimit) return; + if (url.protocol !== "http:" && url.protocol !== "https:") return; + const row: PageRequest = { + method: request.method(), + url: `${url.origin}${url.pathname}`, + kind: request.resourceType(), + startedAt: now(), + }; + pageRequests.push(row); + openRequests.set(request, row); + }); + context.on("response", (response) => { + const row = openRequests.get(response.request()); + if (requestsClosed || row === undefined) return; + row.status = response.status(); + row.respondedAt = now(); + }); + context.on("requestfinished", (request) => { + const row = openRequests.get(request); + if (requestsClosed || row === undefined) return; + row.endedAt = now(); + openRequests.delete(request); + }); + context.on("requestfailed", (request) => { + const row = openRequests.get(request); + if (requestsClosed || row === undefined) return; + row.endedAt = now(); + row.error = networkError(request); + openRequests.delete(request); + }); context.on("weberror", (error) => pageError("page", error.page(), String(error.error()))); context.on("console", (message) => { if (message.type() === "error") pageError("console", message.page(), message.text()); @@ -223,6 +287,8 @@ export class Browser extends Context.Service< const video = page?.video(); // Coverage is evidence about the suite, not the scenario; it never fails one. yield* saveCoverage.pipe(Effect.ignore); + // Closing fails every open request; those stay open in the record. + requestsClosed = true; yield* driver("close browser context", () => context.close()).pipe(Effect.orDie); yield* driver("collect sanitized resource failures", () => Promise.all(failureReads), @@ -237,6 +303,7 @@ export class Browser extends Context.Service< } yield* evidence.json("navigation.json", navigations); yield* evidence.json("failed-resources.json", failedResources); + yield* evidence.json("page-requests.json", pageRequests); if (pageErrors.length > 0) yield* evidence.json("page-errors.json", pageErrors); for (const id of traceIds) yield* evidence.browserTrace(id); }), @@ -312,14 +379,10 @@ export class Browser extends Context.Service< }); current.on("requestfailed", (request) => { const url = new URL(request.url()); - const reason = request.failure()?.errorText; failedResources.push({ url: `${url.origin}${url.pathname}`, kind: request.resourceType(), - error: - reason !== undefined && /^net::ERR_[A-Z_]+$/.test(reason) - ? reason - : "NETWORK_ERROR", + error: networkError(request), }); }); return { page: current, focus, capture }; diff --git a/e2e/support/case.ts b/e2e/support/case.ts index b9d2f0419a..7cad2ed553 100644 --- a/e2e/support/case.ts +++ b/e2e/support/case.ts @@ -11,10 +11,16 @@ import { RecordingFocus } from "./recording-focus.ts"; import { Terminal } from "./terminal.ts"; import { processCoverageLayer } from "./process-coverage.ts"; import { holdSecret } from "./failure-text.ts"; +import { ScenarioDeadline } from "./scenario-deadline.ts"; /** Build per-case layers using the shared runtime from Effect Vitest's layer helper. */ export const withCase = (context: TestContext, program: Effect.Effect) => Effect.gen(function* () { + // Vitest times the scenario from when it calls the body, just before this; its time-out is + // the scenario's (`e2e/timeouts.ts`, or its own). A wait that must report what it saw ends + // before it, so the time is taken before any per-case layer is built. + const timeout = context.task.timeout; + const deadline = timeout > 0 ? Date.now() + timeout : undefined; const lifetime = scenarioLifetime(context); const target = lifetime.target; // The case's API key is masked in every failure text, like each secret the harness reveals. @@ -45,7 +51,11 @@ export const withCase = (context: TestContext, program: Effect.Effect Context.get(services, Evidence).failed(cause)), ), ); - return yield* program.pipe(Effect.provideContext(services), Scope.provide(tests)); + return yield* program.pipe( + Effect.provideContext(services), + Effect.provideService(ScenarioDeadline, deadline), + Scope.provide(tests), + ); }).pipe(Effect.onExit((exit) => Effect.sync(() => lifetime.completed(exit)))); }); diff --git a/e2e/support/cloud-environment.ts b/e2e/support/cloud-environment.ts index d60fd10a15..36580e9fbf 100644 --- a/e2e/support/cloud-environment.ts +++ b/e2e/support/cloud-environment.ts @@ -1,3 +1,4 @@ +import { engineCallFaults } from "../test-plan.ts"; import { Clock, Console, @@ -34,12 +35,13 @@ import { import { startFixtureControl, fixtureRequest } from "../sdk/fixtures.ts"; import { roleHost } from "./role-hosts.ts"; import { freePort } from "./ports.ts"; -import { OperatorOAuthFixture, operatorOAuthFile } from "./operator-oauth.ts"; +import { OperatorOAuthFixture, operatorOAuthFile, syntheticCloudGoogle } from "./operator-oauth.ts"; import { reveal, secret } from "./failure-text.ts"; /** * The operator settings for `fixture`: one test client, whose tokens go only to its own host, and - * its credentials. Emulated Cloud adds the definition to its reviewed clients. + * its credentials, which emulated Cloud adds to its reviewed clients; and synthetic credentials for + * the reviewed Executor for Google client. */ const operatorOAuthClients = (fixture: typeof OperatorOAuthFixture.Type) => { const origin = `http://127.0.0.1:${fixture.port}`; @@ -60,6 +62,9 @@ const operatorOAuthClients = (fixture: typeof OperatorOAuthFixture.Type) => { ]), EXECUTOR_OAUTH_SYNTHETIC_CLOUD_MAIL_CLIENT_ID: fixture.clientId, EXECUTOR_OAUTH_SYNTHETIC_CLOUD_MAIL_CLIENT_SECRET: fixture.clientSecret, + // The reviewed Executor for Google client, as Cloud's own registration. + EXECUTOR_OAUTH_GOOGLE_CLIENT_ID: syntheticCloudGoogle.clientId, + EXECUTOR_OAUTH_GOOGLE_CLIENT_SECRET: syntheticCloudGoogle.clientSecret, }; }; @@ -78,9 +83,10 @@ class CloudStartFailed extends Schema.TaggedError()("CloudStar const readinessDeadline = "10 minutes"; /** - * Motels behind this Cloud's collector. Six scenario workers provisioning organizations at once - * export about 1,500 spans a second, as much as one Motel stores; four keep up with several times - * that and still leave a 16-vCPU runner most of its cores. + * Collectors behind this Cloud's collector router. Six scenario workers provisioning organizations + * at once export about 1,500 spans a second, as much as one Motel stored; four Motels kept up with + * several times that. The Rust collector stores more than one Motel did; the count is unchanged + * until one has been measured in this job (`serveShardedOtlpCollector`). */ const collectorShards = 4; @@ -253,6 +259,8 @@ export const startCloudEnvironment = (input: { CLOUD_DEV_DATABASE_PASSWORD: databasePassword, CLOUD_DEV_EXTERNAL_DATABASE: "true", EXECUTOR_EMULATORS: JSON.stringify(reveal(fixture).services), + // Fails only engine calls to steps the workflow engine calls scenario's app names. + EXECUTOR_TEST_WORKFLOW_ENGINE_FAULTS: JSON.stringify(engineCallFaults), V1_MCP_FORWARD_SECRET: v1ForwardSecret, V1_SESSION_HANDOFF_SECRET: v1HandoffSecret, // Serve the role hosts as production does, at `app.`, `mcp.` and `api.` of the origin's host. diff --git a/e2e/support/cloud-postgres.ts b/e2e/support/cloud-postgres.ts index 2495ed0afd..51f1f8e46b 100644 --- a/e2e/support/cloud-postgres.ts +++ b/e2e/support/cloud-postgres.ts @@ -7,8 +7,24 @@ * then removes whatever it made. A run killed outright cannot clean up, so each container carries * the process that owns it and the process namespace that process ID belongs to, and the next run * in that namespace removes those whose owner has exited. + * + * The database's files live in a named volume of the run's own. A container that vanishes by any + * other route than this run's teardown (a hand-run `docker rm -f`, a prune, a daemon restart, a + * reboot) would otherwise leave an anonymous volume nothing names. The volume carries the boot, + * the process namespace, and the owning process's ID with the start time the operating system + * keeps for it, so a later run can tell for certain that its owner is gone (`volumeVerdict`). */ -import { Clock, Console, Effect, FileSystem, Schedule, Schema, Stream } from "effect"; +import { + Clock, + Console, + Duration, + Effect, + FileSystem, + Option, + Schedule, + Schema, + Stream, +} from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; const role = "executor.e2e.role=cloud-postgres"; @@ -17,6 +33,18 @@ const pidLabel = "executor.e2e.pid"; /** A container younger than this is never removed by another run, whatever its owner's state. */ const staleAfterMs = 5 * 60_000; +/** The labels of a run's database volume. Only volumes with the role label are ever listed. */ +export const volumeLabels = { + role: "executor.e2e.role", + boot: "executor.e2e.boot", + namespace: "executor.e2e.owner-namespace", + pid: "executor.e2e.owner-pid", + started: "executor.e2e.owner-started", +} as const; +const volumeRole = "cloud-postgres-data"; +/** How long the sweep of abandoned volumes may take before the run goes on without it. */ +const sweepTimeout = Duration.seconds(30); + class CloudPostgresFailed extends Schema.TaggedError()("CloudPostgresFailed", { operation: Schema.String, }) { @@ -132,6 +160,220 @@ const removeAbandoned = (dockerEnv: DockerEnv, namespace: string) => ); }); +/** + * This run as its database volume records it, or undefined where a process's start time cannot + * be read. Linux: the boot ID, the pid and time namespaces (start times count clock ticks from + * boot, as the reader's time namespace sees them), and field 22 of `/proc/self/stat`. macOS: the + * boot session UUID, one process namespace per boot, and the second `ps` says the process + * started, in UTC. Elsewhere volumes carry no owner and are never swept. + */ +export interface VolumeOwner { + readonly boot: string; + readonly namespace: string; + readonly pid: number; + readonly started: string; +} + +const uuid = /^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$/i; + +/** + * When process `pid` started, as `VolumeOwner.started` records it. None when no process has the + * pid, or only one that has exited; undefined when the operating system did not say. + */ +export const processStarted = (pid: number) => + Effect.gen(function* () { + if (!running(pid)) return Option.none(); + if (process.platform === "linux") { + const fs = yield* FileSystem.FileSystem; + const stat = yield* fs.readFileString(`/proc/${pid}/stat`); + // The command name, field 2, is in parentheses and may hold spaces; field 3 follows it. + const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" "); + const [state, started] = [fields[0], fields[19]]; + if (state === undefined || started === undefined || !/^\d+$/.test(started)) return undefined; + return state === "Z" || state === "X" ? Option.none() : Option.some(started); + } + if (process.platform === "darwin") { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = (yield* processes.string( + ChildProcess.make("/bin/ps", ["-o", "stat=", "-o", "lstart=", "-p", String(pid)], { + env: { LC_ALL: "C", TZ: "UTC0" }, + }), + )).trim(); + const match = /^(\S+)\s+(\S.*)$/.exec(output); + if (match === null) return undefined; + const [, state = "", started = ""] = match; + return state.startsWith("Z") ? Option.none() : Option.some(started); + } + return undefined; + }).pipe( + // A process that exits while it is read leaves nothing to read. + Effect.orElseSucceed(() => (running(pid) ? undefined : Option.none())), + ); + +const volumeOwner = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const started = yield* processStarted(process.pid); + if (started === undefined || Option.isNone(started)) return undefined; + if (process.platform === "linux") { + const boot = (yield* fs.readFileString("/proc/sys/kernel/random/boot_id")).trim(); + const pids = yield* fs.readLink("/proc/self/ns/pid"); + const time = yield* fs + .readLink("/proc/self/ns/time") + .pipe(Effect.orElseSucceed(() => "time:none")); + if (!uuid.test(boot) || !/^pid:\[\d+\]$/.test(pids)) return undefined; + return { + boot, + namespace: `linux/${boot}/${pids}/${time}`, + pid: process.pid, + started: started.value, + }; + } + if (process.platform === "darwin") { + const boot = (yield* processes.string( + ChildProcess.make("sysctl", ["-n", "kern.bootsessionuuid"]), + )).trim(); + if (!uuid.test(boot)) return undefined; + return { boot, namespace: `darwin/${boot}`, pid: process.pid, started: started.value }; + } + return undefined; +}).pipe(Effect.orElseSucceed((): VolumeOwner | undefined => undefined)); + +/** A run's database volume as the sweep sees it. */ +export interface SweptVolume { + readonly name: string; + readonly labels: Readonly>; + readonly createdAt: number; + /** How many containers, running or stopped, mount it. */ + readonly mountedBy: number; +} + +export type VolumeVerdict = + | { readonly remove: true; readonly why: "another boot" | "owner exited" | "owner's pid reused" } + | { + readonly remove: false; + readonly why: + | "not ours" + | "incomplete labels" + | "mounted" + | "young" + | "another process namespace" + | "owner runs" + | "owner unknown"; + }; + +/** + * Whether the volume's run is certainly gone. Anything short of certain keeps it: leaking a volume + * is the safe failure, taking a live run's database is not. `started` reads a process's start + * time as `processStarted` does. + */ +export const volumeVerdict = ( + volume: SweptVolume, + here: VolumeOwner, + now: number, + started: (pid: number) => Effect.Effect | undefined, never, R>, +) => + Effect.gen(function* (): Effect.fn.Return { + const labels = volume.labels; + if (labels[volumeLabels.role] !== volumeRole) return { remove: false, why: "not ours" }; + const boot = labels[volumeLabels.boot] ?? ""; + const namespace = labels[volumeLabels.namespace] ?? ""; + const pid = labels[volumeLabels.pid] ?? ""; + const recorded = labels[volumeLabels.started] ?? ""; + if (!uuid.test(boot) || namespace === "" || !/^[1-9]\d*$/.test(pid) || recorded === "") + return { remove: false, why: "incomplete labels" }; + if (volume.mountedBy !== 0) return { remove: false, why: "mounted" }; + // Covers the moment between a run creating its volume and its container. + if (!(now - volume.createdAt > staleAfterMs)) return { remove: false, why: "young" }; + if (boot !== here.boot) return { remove: true, why: "another boot" }; + // A process ID names a process only in the namespace that issued it. + if (namespace !== here.namespace) return { remove: false, why: "another process namespace" }; + const current = yield* started(Number(pid)); + if (current === undefined) return { remove: false, why: "owner unknown" }; + if (Option.isNone(current)) return { remove: true, why: "owner exited" }; + return current.value === recorded + ? { remove: false, why: "owner runs" } + : { remove: true, why: "owner's pid reused" }; + }); + +const Volumes = Schema.fromJsonString( + Schema.Array( + Schema.Struct({ + Name: Schema.String, + CreatedAt: Schema.String, + Labels: Schema.NullOr(Schema.Record(Schema.String, Schema.String)), + }), + ), +); + +/** + * Remove the database volumes, listed by their role label and nothing else, whose run is + * certainly gone. Never fails the run: what it could not do is reported and the run goes on. + */ +const removeAbandonedVolumes = (dockerEnv: DockerEnv, here: VolumeOwner) => + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const docker = (args: ReadonlyArray) => + ChildProcess.make("docker", args, { env: dockerEnv, extendEnv: false }); + const names = (yield* processes.string( + docker(["volume", "ls", "--quiet", "--filter", `label=${volumeLabels.role}=${volumeRole}`]), + )) + .split("\n") + .filter((name) => name.length > 0); + if (names.length === 0) return; + const volumes = yield* processes + .string(docker(["volume", "inspect", ...names])) + .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Volumes))); + const now = yield* Clock.currentTimeMillis; + for (const volume of volumes) { + const mountedBy = (yield* processes.string( + docker(["ps", "--all", "--quiet", "--filter", `volume=${volume.Name}`]), + )) + .split("\n") + .filter((id) => id.length > 0).length; + const verdict = yield* volumeVerdict( + { + name: volume.Name, + labels: volume.Labels ?? {}, + createdAt: Date.parse(volume.CreatedAt), + mountedBy, + }, + here, + now, + processStarted, + ); + if (!verdict.remove) continue; + // Not forced: Docker refuses a volume that a container created since the check mounts. + const [code, errors] = yield* Effect.scoped( + Effect.gen(function* () { + const rm = yield* processes.spawn( + ChildProcess.make("docker", ["volume", "rm", volume.Name], { + env: dockerEnv, + extendEnv: false, + stdout: "ignore", + stderr: "pipe", + }), + ); + return yield* Effect.all([rm.exitCode, Stream.mkString(Stream.decodeText(rm.stderr))], { + concurrency: 2, + }); + }), + ); + yield* code === 0 + ? Console.error( + `Removed abandoned Cloud test database volume ${volume.Name} (${verdict.why})`, + ) + : Console.error( + `Cannot remove abandoned Cloud test database volume ${volume.Name}: ${errors.trim()}`, + ); + } + }).pipe( + Effect.timeout(sweepTimeout), + Effect.catchCause((cause) => + Console.error(`Skipped the sweep of abandoned Cloud test database volumes: ${String(cause)}`), + ), + ); + /** Create, start and await a Postgres container that is removed when the caller's scope closes. */ export const startCloudPostgres = (input: { readonly container: string; @@ -156,6 +398,46 @@ export const startCloudPostgres = (input: { // checks none. const namespace = yield* pidNamespace; if (namespace !== undefined) yield* removeAbandoned(dockerEnv, namespace); + // Without a start time, a volume records no owner, and this run sweeps none. + const owner = yield* volumeOwner; + if (owner !== undefined) yield* removeAbandonedVolumes(dockerEnv, owner); + const volume = `${container}-data`; + // Released after the container, which mounts it. Removal by name is a no-op when nothing was + // created; a volume that cannot be removed is left for a later run's sweep. + const volumeCreated = yield* Effect.acquireRelease( + processes.exitCode( + docker([ + "volume", + "create", + "--label", + `${volumeLabels.role}=${volumeRole}`, + ...(owner === undefined + ? [] + : [ + "--label", + `${volumeLabels.boot}=${owner.boot}`, + "--label", + `${volumeLabels.namespace}=${owner.namespace}`, + "--label", + `${volumeLabels.pid}=${owner.pid}`, + "--label", + `${volumeLabels.started}=${owner.started}`, + ]), + volume, + ]), + ), + () => + processes.exitCode(docker(["volume", "rm", "--force", volume])).pipe( + Effect.flatMap((code) => + code === 0 + ? Effect.void + : Console.error(`Cannot remove Cloud test database volume ${volume}`), + ), + Effect.catch(() => Console.error(`Cannot remove Cloud test database volume ${volume}`)), + ), + ); + if (volumeCreated !== 0) + return yield* new CloudPostgresFailed({ operation: "Create the Postgres volume" }); // Acquisition is uninterruptible, so an interrupted run still learns whether Docker created // the container before it removes it. Removal by name is a no-op when nothing was created. const created = yield* Effect.acquireRelease( @@ -176,6 +458,8 @@ export const startCloudPostgres = (input: { "--label", `${pidLabel}=${process.pid}`, ]), + "--volume", + `${volume}:/var/lib/postgresql/data`, "--publish", `127.0.0.1:${input.databasePort}:5432`, "--env", diff --git a/e2e/support/connect-timeout.ts b/e2e/support/connect-timeout.ts new file mode 100644 index 0000000000..fc26e20f1f --- /dev/null +++ b/e2e/support/connect-timeout.ts @@ -0,0 +1,20 @@ +/** + * How long the e2e harness's own outbound connections give each address of a name before trying the + * next: Node's network family auto-selection attempt time-out, 250 ms by default on Node 24.22.1 + * (`net.getDefaultAutoSelectFamilyAttemptTimeout()`). With a name's two IPv4 addresses slow to + * answer and its IPv6 addresses failing at once (a runner without an IPv6 route), Node gives up + * after about 500 ms: the `AggregateError ETIMEDOUT, after 507 ms` of emulator calls on 16 Nov, + * each of which spent one of the call's three attempts. At 2,500 ms that case gives up after about + * 5 s, inside each attempt's own 10 s (support/emulators.ts), and a connect that answers is not + * slowed at all. + * + * The test client only: product processes the harness starts are other processes and keep Node's + * default. Imported first by each entry point that makes outbound calls (run.ts, deployed.ts, + * cleanup-deployed.ts, cli.ts, create-emulators.ts) and by Vitest's setup file (setup.ts), so every + * worker has it too. + */ +import { setDefaultAutoSelectFamilyAttemptTimeout } from "node:net"; + +export const connectAttemptMs = 2_500; + +setDefaultAutoSelectFamilyAttemptTimeout(connectAttemptMs); diff --git a/e2e/support/contracts.ts b/e2e/support/contracts.ts index cfc5d9989c..8e611ed23f 100644 --- a/e2e/support/contracts.ts +++ b/e2e/support/contracts.ts @@ -15,11 +15,24 @@ export const Inventory = Schema.Struct({ }); export const Collector = Schema.Struct({ state: Schema.Literal("running"), url: Schema.String }); /** - * Motel's exported span query projection. Tests assert only delivered telemetry. It keeps every - * delivered field that can carry text, including exception events with their messages and stacks, - * so a privacy check over the whole projection also covers error messages. + * The collector's span query projection (Motel's shapes). Tests assert only delivered telemetry. It + * keeps every delivered field that can carry text, including exception events with their messages + * and stacks, so a privacy check over the whole projection also covers error messages. */ export const SpanQuery = Schema.Struct({ + /** + * The list's own facts. On a trace's spans, `truncated` with `returned` and `spanCount` says the + * trace was answered in part: its earliest `returned` of `spanCount` spans (the collector's + * README, "Fields and markers beyond Motel's answers"). On a search, `truncated` says more spans + * matched than the limit returned. Axiom's adapter sends none. + */ + meta: Schema.optionalKey( + Schema.Struct({ + truncated: Schema.Boolean, + returned: Schema.optionalKey(Schema.Number), + spanCount: Schema.optionalKey(Schema.Number), + }), + ), data: Schema.Array( Schema.Struct({ traceId: Schema.String, @@ -40,9 +53,9 @@ export const SpanQuery = Schema.Struct({ attributes: Schema.Record(Schema.String, Schema.String), }), ), - // Motel keeps only the status code; Axiom also delivers the status message. + // The collector keeps only the status code; Axiom also delivers the status message. statusMessage: Schema.optionalKey(Schema.String), - // Motel omits links; Axiom's adapter must supply the delivered array. + // The collector omits links; Axiom's adapter must supply the delivered array. links: Schema.optionalKey( Schema.Array(Schema.Struct({ traceId: Schema.String, spanId: Schema.String })), ), diff --git a/e2e/support/evidence.ts b/e2e/support/evidence.ts index 89eaf5e4d8..3d42824621 100644 --- a/e2e/support/evidence.ts +++ b/e2e/support/evidence.ts @@ -20,9 +20,10 @@ import { EvidenceEntries } from "../report-model.ts"; import { Target } from "./platform.ts"; import { Collector, SpanQuery } from "./contracts.ts"; import { RecordingFocus } from "./recording-focus.ts"; -import { axiomSpans } from "./axiom.ts"; +import { axiomSpans, stageEnvironment } from "./axiom.ts"; import { scopeOf, type Scope } from "./scope.ts"; import { stepFailureText } from "./failure-text.ts"; +import { lostRequestsOf, stageRequestsOf } from "./lost-connections.ts"; /** Public request measurements contain no request bodies, cookies or credentials. */ export interface RequestEvidence { @@ -120,11 +121,56 @@ export type SpanScope = Scope< /** One delivered span. */ export type DeliveredSpan = (typeof SpanQuery.Type)["data"][number]["span"]; -/** Motel queries read delivered spans through HTTP. No in-process instrumentation assertions. */ +/** The most matches the collector returns for one span search. */ +export const searchLimit = 500; + +/** + * A search the collector cut off at its limit holds only some of the matching spans. A check that + * read it would assert on part of the data: an absence it reports could be a span past the limit. + * So it is a defect, never retried; the search must be narrowed (a trace, an owned id, a time + * window or an attribute) until it fits. + */ +export const refuseTruncated = (found: typeof SpanQuery.Type, url: URL) => + found.meta?.truncated === true + ? Effect.die( + new Error( + `The collector's span search ${url.pathname}${url.search} matched more than ${found.meta.returned ?? searchLimit} spans and answered only those; narrow the search so a check reads all of them`, + ), + ) + : Effect.void; + +/** One page of a log list: its logs, and whether more follow and where. */ +const LogPage = Schema.Struct({ + data: Schema.Array(Schema.Json), + meta: Schema.optionalKey( + Schema.Struct({ + truncated: Schema.Boolean, + nextCursor: Schema.optionalKey(Schema.NullOr(Schema.String)), + }), + ), +}); +/** The most pages a log read takes before it gives up: 20,000 logs at 500 a page. */ +const logPages = 40; + +/** Collector queries read delivered spans through HTTP. No in-process instrumentation assertions. */ +/** The longest one read of the local collector may take. */ +export const collectorReadMs = 5_000; + export class Telemetry extends Context.Service< Telemetry, { + /** + * A trace's delivered spans, whole. A trace the collector answers in part (`meta.truncated`) + * is a defect, never retried: a check must not read part of a trace as all of it. + */ readonly query: (traceId: string) => Effect.Effect; + /** + * A trace's spans as the collector answers them, whole or in part, `meta` included. Only for + * evidence, which keeps the marker beside the spans. + */ + readonly stored: ( + traceId: string, + ) => Effect.Effect; /** * The spans of the scenario's own traces once `ready` holds for them. A Cloud isolate exports * the spans that have ended with whichever request flushes next, so a trace arrives in pieces: @@ -149,6 +195,16 @@ export class Telemetry extends Context.Service< scope: SpanScope, attributes?: Readonly>, ) => Effect.Effect; + /** + * Delivered spans of one operation from this environment in the last `minutes` (whole minutes, + * at least one), whose attributes match exactly. For a check whose subject is environment-wide + * work, such as the webhook source sweep, that must stay under a search's match limit. + */ + readonly searchRecent: ( + operation: string, + attributes: Readonly>, + minutes: number, + ) => Effect.Effect; /** * Delivered spans of one operation from every scenario that shares this environment's * collector: on Cloud, every scenario of the job. Only for a check whose subject is the whole @@ -177,7 +233,7 @@ export class Telemetry extends Context.Service< const target = yield* Target; const fs = yield* FileSystem.FileSystem; const http = yield* HttpClient.HttpClient; - const cloud = yield* axiomSpans; + const cloud = yield* axiomSpans(stageEnvironment(target.metadata.origin)); const attached = target.metadata.target === "cloud" && target.metadata.mode === "attached"; const origin = fs.readFileString(`${target.directory}/data/diagnostics/collector.json`).pipe( Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(Collector))), @@ -185,7 +241,7 @@ export class Telemetry extends Context.Service< ); const safe = (program: Effect.Effect) => program.pipe( - Effect.timeout("5 seconds"), + Effect.timeout(collectorReadMs), Effect.mapError(() => new TelemetryUnavailable()), ); const read = (url: URL) => @@ -199,23 +255,33 @@ export class Telemetry extends Context.Service< operation: string, attributes: Readonly>, traceId: string | undefined, + lookback?: { readonly minutes: number }, ) => attached ? cloud - .search(operation, attributes, traceId) + .search( + operation, + attributes, + traceId, + lookback === undefined ? undefined : Date.now() - lookback.minutes * 60_000, + ) .pipe(Effect.mapError(() => new TelemetryUnavailable())) : safe( Effect.scoped( Effect.gen(function* () { const url = new URL("/api/spans/search", yield* origin); url.searchParams.set("operation", operation); - url.searchParams.set("lookback", "1d"); - url.searchParams.set("limit", "10000"); + url.searchParams.set( + "lookback", + lookback === undefined ? "1d" : `${Math.max(1, Math.ceil(lookback.minutes))}m`, + ); + url.searchParams.set("limit", String(searchLimit)); if (traceId !== undefined) url.searchParams.set("traceId", traceId); for (const [key, value] of Object.entries(attributes)) url.searchParams.set(`attr.${key}`, value); return yield* read(url).pipe( Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(SpanQuery))), + Effect.tap((found) => refuseTruncated(found, url)), ); }), ), @@ -250,7 +316,7 @@ export class Telemetry extends Context.Service< return searchIn(operation, { ...attributes, [attribute]: value }, undefined); }), ); - const query = (id: string) => + const stored = (id: string) => attached ? cloud.trace(id).pipe(Effect.mapError(() => new TelemetryUnavailable())) : safe( @@ -263,12 +329,54 @@ export class Telemetry extends Context.Service< }), ), ); + /** + * Every log of a list, read a page at a time through the collector's cursor, as one + * `{data}` JSON text. A list still cut off with no cursor to continue (the Cloud router's + * merged search has none), or longer than `logPages` pages, is a defect, never retried: a + * check must not read part of it as all. + */ const logs = (url: (base: string) => URL) => attached ? Effect.fail(new TelemetryUnavailable()) - : safe(Effect.scoped(Effect.flatMap(origin, (base) => read(url(base))))); + : safe( + Effect.scoped( + Effect.gen(function* () { + const page = url(yield* origin); + page.searchParams.set("limit", String(searchLimit)); + const data: Array = []; + for (let read_ = 1; ; read_++) { + const found = yield* read(page).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(LogPage))), + ); + data.push(...found.data); + if (found.meta?.truncated !== true) return JSON.stringify({ data }); + const cursor = found.meta.nextCursor; + if (cursor === undefined || cursor === null || read_ >= logPages) + return yield* Effect.die( + new Error( + `The collector's log list ${page.pathname}${page.search} holds more than it answered and cannot be read whole; narrow it so a check reads every log`, + ), + ); + page.searchParams.set("cursor", cursor); + } + }), + ), + ); + const query = (id: string) => + stored(id).pipe( + Effect.tap((found) => + found.meta?.truncated === true + ? Effect.die( + new Error( + `The collector answered trace ${id} in part (${found.meta.returned ?? "?"} of ${found.meta.spanCount ?? "?"} spans); a check cannot read it as the whole trace`, + ), + ) + : Effect.void, + ), + ); return { query, + stored, settled: (traces, ready) => { const list = Effect.isEffect(traces) ? traces @@ -292,6 +400,8 @@ export class Telemetry extends Context.Service< Effect.map((found) => found.data.map(({ span }) => span.tags)), ), search, + searchRecent: (operation, attributes, minutes) => + searchIn(operation, attributes, undefined, { minutes }), searchWholeEnvironment: (operation, attributes) => searchIn(operation, attributes, undefined), logs: (traceId) => logs((base) => new URL(`/api/traces/${traceId}/logs`, base)), @@ -478,6 +588,14 @@ export const scenarioEvidence = ( const outcome = final ? report(cause) : Effect.void; const rows = yield* Ref.get(requests); yield* json("requests.json", rows); + // Stage requests lost in transport, and those sent again because nothing of them was + // written to a connection, with each repeat (lost-connections.ts). + const lost = lostRequestsOf(identity.name); + if (lost.length > 0) yield* json("lost-requests.json", lost); + // Every stage request the scenario sent, through any client, passed or failed: a few + // hundred rows at most for most scenarios (lost-connections.ts). + const stage = stageRequestsOf(identity.name); + if (stage.rows.length > 0) yield* json("stage-requests.json", stage); yield* json("steps.json", steps); const timeline = yield* recording.snapshot; yield* json("recording-timeline.json", timeline); @@ -616,12 +734,12 @@ export const scenarioEvidence = ( const entry = ( traceId: string, kept: ReadonlyArray, - query: typeof telemetry.query, + query: typeof telemetry.stored, ) => query(traceId).pipe( Effect.map((value) => ({ id: traceId, kept, ...value })), Effect.catch(() => - Effect.succeed({ id: traceId, kept, error: "Motel query failed" }), + Effect.succeed({ id: traceId, kept, error: "The collector query failed" }), ), ); // The server exports its spans every second, so the requests just before the failure @@ -632,7 +750,7 @@ export const scenarioEvidence = ( traceId: string, spanId: string, ): Effect.Effect => - telemetry.query(traceId).pipe( + telemetry.stored(traceId).pipe( Effect.flatMap((found) => Effect.gen(function* () { if ( @@ -647,7 +765,7 @@ export const scenarioEvidence = ( ); const read = ({ traceId, spanId, kept }: Planned) => spanId === undefined - ? entry(traceId, kept, telemetry.query) + ? entry(traceId, kept, telemetry.stored) : entry(traceId, kept, (id) => delivered(id, spanId)); const size = (value: unknown) => Buffer.byteLength(JSON.stringify(value)); let used = 0; diff --git a/e2e/support/failure-text.ts b/e2e/support/failure-text.ts index c497e71c88..80cf6d835c 100644 --- a/e2e/support/failure-text.ts +++ b/e2e/support/failure-text.ts @@ -256,8 +256,9 @@ const bounded = (message: string, lineLength = maxLine) => { /** One free text, such as a page error or a console message: masked in full, then bounded. */ export const safeText = (text: string): string => bounded(mask(text)); -// Node and undici codes are short constants, such as ERR_INVALID_PROTOCOL or ECONNREFUSED. -const codePattern = /^[A-Z][A-Z0-9_]{1,63}$/u; +// Node and undici codes are short constants, such as ERR_INVALID_PROTOCOL or ECONNREFUSED. OpenSSL's +// are built from its reason text and can hold a slash: ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE. +const codePattern = /^[A-Z][A-Z0-9_/]{1,63}$/u; const codeDepth = 8; /** The next level of a cause chain: an Effect `HttpClientError`'s `reason`, else `cause`. */ diff --git a/e2e/support/gate-environment.ts b/e2e/support/gate-environment.ts new file mode 100644 index 0000000000..7f7a7c388f --- /dev/null +++ b/e2e/support/gate-environment.ts @@ -0,0 +1,63 @@ +/** + * The environment GitHub Actions gives a gate step of checks.yml (`check` and `e2e`): its `env:` + * block's expressions evaluated against a `needs` object, as the runner would, so a case runs the + * gate on exactly what the workflow passes it. An expression this does not know fails the case + * instead of being guessed at. + */ + +export interface Need { + readonly result: string; + readonly outputs: Readonly>; +} + +/** The `env:` block of the step named `step` in the job `job` of a workflow's text. */ +export const stepEnvironmentBlock = (workflow: string, job: string, step: string) => { + const start = workflow.indexOf(`\n ${job}:\n`); + if (start === -1) throw new Error(`No job ${job}`); + const end = workflow.slice(start + 1).search(/\n {2}[a-z][\w-]*:\n/); + const text = end === -1 ? workflow.slice(start) : workflow.slice(start, start + 1 + end); + const at = text.indexOf(` - name: ${step}\n`); + if (at === -1) throw new Error(`No step ${step} in ${job}`); + const rest = text.slice(at); + const env = rest.indexOf("\n env:\n"); + const run = rest.indexOf("\n run: |\n"); + if (env === -1 || run === -1 || env > run) throw new Error(`No env block in ${job} / ${step}`); + return rest.slice(env + "\n env:\n".length, run); +}; + +/** One `${{ … }}` value, evaluated against `needs`. */ +const evaluate = (expression: string, needs: Readonly>): string => { + if (expression === "join(needs.*.result, ' ')") + return Object.values(needs) + .map((need) => need.result) + .join(" "); + if (expression === "github.token") return "synthetic-token"; + const result = expression.match(/^needs\.([\w-]+)\.result$/); + if (result !== null) return needs[result[1]!]?.result ?? ""; + const output = expression.match(/^needs\.([\w-]+)\.outputs\.([\w-]+)$/); + if (output !== null) return needs[output[1]!]?.outputs[output[2]!] ?? ""; + const set = expression.match(/^needs\.([\w-]+)\.outputs\.([\w-]+) != ''$/); + if (set !== null) return String((needs[set[1]!]?.outputs[set[2]!] ?? "") !== ""); + throw new Error(`An expression gate-environment.ts does not evaluate: ${expression}`); +}; + +/** The variables a gate step gets for `needs`; comments and literal values are kept as written. */ +export const gateEnvironment = ( + workflow: string, + job: string, + step: string, + needs: Readonly>, +): Record => + Object.fromEntries( + stepEnvironmentBlock(workflow, job, step) + .split("\n") + .filter((line) => /^ {10}[A-Za-z_][\w]*: /.test(line)) + .map((line) => { + const [, name, value] = line.match(/^ {10}([A-Za-z_]\w*): (.*)$/)!; + const expression = value!.match(/^\$\{\{ (.*) \}\}$/); + return [ + name!, + expression === null ? value!.replace(/^"(.*)"$/, "$1") : evaluate(expression[1]!, needs), + ]; + }), + ); diff --git a/e2e/support/lifecycle.ts b/e2e/support/lifecycle.ts index 67c3b000bf..b0f55935ea 100644 --- a/e2e/support/lifecycle.ts +++ b/e2e/support/lifecycle.ts @@ -1,7 +1,8 @@ /** Vitest hooks own isolated servers and cleanup; the test deadline owns scenario work. */ -import { Clock, Effect, Exit, FileSystem, Layer, Scope } from "effect"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { Clock, Config, Effect, Exit, FileSystem, Layer, Option, Path, Scope } from "effect"; import { randomBytes } from "node:crypto"; -import { beforeEach, type TestContext } from "vitest"; +import { afterAll, beforeEach, type TestContext } from "vitest"; import { prepareScenario, startScenario } from "../sdk/scenario.ts"; import { createScenario } from "../sdk/session.ts"; import { RuntimeLive, Target } from "./platform.ts"; @@ -10,6 +11,17 @@ import { prepareManagementApp } from "./management-app.ts"; import { prepareOrganization } from "./organization-provisioning.ts"; import { SessionClients } from "./api.ts"; import { scenarios, type TestPlan } from "../test-plan.ts"; +import { saveLostRequests } from "../lost-requests.ts"; +import { + configureLostConnections, + takeAllLostRequests, + forgetStageRequests, + takeLostRequests, + unsentNote, +} from "./lost-connections.ts"; +import { targetHosts } from "./role-hosts.ts"; +import { machineWorkMs, referenceMachineWorkMs, timeoutLine } from "./machine-speed.ts"; +import { hookTimeoutMs } from "../timeouts.ts"; /** An extra Testing SDK scenario acquired by setup. The test may close its scope early. */ export interface SdkScenarioFixture { @@ -50,48 +62,107 @@ export const installScenarioLifecycle = () => let completedAt: number | undefined; let provisioningMs: number | null = null; let save = (_finishedAt: number): Effect.Effect => Effect.void; + let finished = false; + // A scenario that fails while a request of it never got a connection says so in its failure, + // a time-out included: "one stage request was unsent for 31.0 s". + context.onTestFailed(({ task }) => { + const note = unsentNote(task.name); + const first = task.result?.errors?.[0]; + if (note === "" || first === undefined) return; + const original = first.message; + first.message = `${original}; ${note}`; + if (first.stack) first.stack = first.stack.replace(original, first.message); + }); + // The time-out that applied, and, when the scenario or one of its hooks timed out, how fast + // the machine was then (machine-speed.ts). The line is also put on the time-out's own error. + let timing: Record = {}; let discard: Effect.Effect = Effect.void; // A passing scenario's server data is never read again; failures keep theirs for diagnosis. - context.onTestFinished(({ task }) => - Effect.runPromise( + context.onTestFinished(({ task }) => { + finished = true; + timing = { timeoutMs: task.timeout, hookTimeoutMs }; + const timedOut = (task.result?.errors ?? []).flatMap((error) => { + const bound = /^(Test|Hook) timed out in (\d+)ms/.exec(error.message ?? ""); + return bound ? [{ error, kind: bound[1], boundMs: Number(bound[2]) }] : []; + }); + if (timedOut.length > 0) { + const machineMs = machineWorkMs(); + const lines = timedOut.map(({ error, kind, boundMs }) => { + const line = `${kind} ${timeoutLine(boundMs, boundMs, machineMs)}`; + // The first line of the failure, which the job log and ci-failures print, names the + // machine's speed; Vitest's own message follows it. + const original = error.message; + error.message = `${line}\n${original}`; + if (error.stack) error.stack = error.stack.replace(original, error.message); + return line; + }); + timing = { ...timing, timedOut: lines, machineMs, referenceMachineWorkMs }; + console.error(lines.join("\n")); + } + return Effect.runPromise( Scope.close(scope, outcome).pipe( Effect.ensuring(Clock.currentTimeMillis.pipe(Effect.flatMap(save))), + // Closing the scope wrote the scenario's evidence, its stage requests included. + Effect.ensuring(Effect.sync(() => forgetStageRequests(task.name))), Effect.andThen( Effect.suspend(() => (task.result?.state === "pass" ? discard : Effect.void)), ), ), - ), - ); + ); + }); return Effect.runPromise( Effect.gen(function* () { const runtime = yield* Layer.buildWithScope(RuntimeLive, scope); yield* Effect.gen(function* () { const base = yield* Target; const fs = yield* FileSystem.FileSystem; + const paths = yield* Path.Path; const directory = `${base.directory}/report/lifecycle`; yield* fs.makeDirectory(directory, { recursive: true }); + // The stage this scenario sends to: its requests lost in transport are told apart + // and recorded, each on a connection of its own (lost-connections.ts). A target the scenario starts adds + // its own origins as soon as it exists (sdk/scenario.ts). + // A request of which nothing was written is sent again, never at or past the + // scenario's own bound, counted from before its set-up. + configureLostConnections({ + origins: Object.values(targetHosts(base)), + deadline: () => + finished || context.task.timeout <= 0 ? undefined : startedAt + context.task.timeout, + }); save = (finishedAt) => - fs - .writeFileString( - `${directory}/${context.task.id}.json`, - JSON.stringify( - { - title: context.task.name, - setupMs: readyAt === undefined ? finishedAt - startedAt : readyAt - startedAt, - provisioningMs, - scenarioMs: - readyAt === undefined || completedAt === undefined - ? null - : completedAt - readyAt, - cleanupMs: completedAt === undefined ? null : finishedAt - completedAt, - totalMs: finishedAt - startedAt, - }, - null, - 2, - ), - { mode: 0o600 }, + saveLostRequests(base.directory, context.task.id, takeLostRequests(context.task.name)) + .pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, paths), + Effect.orDie, ) - .pipe(Effect.orDie); + .pipe( + Effect.andThen( + fs + .writeFileString( + `${directory}/${context.task.id}.json`, + JSON.stringify( + { + title: context.task.name, + setupMs: + readyAt === undefined ? finishedAt - startedAt : readyAt - startedAt, + provisioningMs, + scenarioMs: + readyAt === undefined || completedAt === undefined + ? null + : completedAt - readyAt, + cleanupMs: completedAt === undefined ? null : finishedAt - completedAt, + totalMs: finishedAt - startedAt, + ...timing, + }, + null, + 2, + ), + { mode: 0o600 }, + ) + .pipe(Effect.orDie), + ), + ); const plan: typeof TestPlan.Type | undefined = Object.values(scenarios).find( (scenario) => scenario.title === context.task.name, ); @@ -175,4 +246,26 @@ export const installScenarioLifecycle = () => { signal: context.signal }, ); }); + +/** + * At the end of each test file, save the lost requests no scenario saved: those made outside any + * scenario (`beforeAll`, `afterAll`, a `layer`'s set-up and teardown) and a scenario's whose start + * failed before its lifecycle could save them, so a burst in set-up still shows in the summary. + */ +export const installLostRequestSaving = () => + afterAll(() => + Effect.runPromise( + Effect.gen(function* () { + const run = yield* Config.String("EXECUTOR_E2E_RUN").pipe(Config.option); + const records = takeAllLostRequests(); + if (Option.isNone(run) || records.length === 0) return; + const file = records.find((lost) => lost.file !== undefined)?.file ?? "file"; + yield* saveLostRequests( + run.value, + `outside-scenarios-${file.replace(/[^A-Za-z0-9.-]/gu, "-")}-${process.pid}-${Date.now()}`, + records, + ); + }).pipe(Effect.provide(NodeServices.layer), Effect.orDie), + ), + ); /* oxlint-enable executor/no-manual-effect-runtime-in-tests */ diff --git a/e2e/support/local-workflow-engines.ts b/e2e/support/local-workflow-engines.ts index 09ccc5fa73..3a196ba3ef 100644 --- a/e2e/support/local-workflow-engines.ts +++ b/e2e/support/local-workflow-engines.ts @@ -34,12 +34,18 @@ export const startLocalProduct = (environment: Readonly> "product", environment, ); - const request = (method: "GET" | "POST", route: string, data?: unknown) => + const request = ( + method: "GET" | "POST", + route: string, + data?: unknown, + headers: Readonly> = {}, + ) => driver(`${method} ${route}`, (signal) => fetch(`${server.origin}${route}`, { method, signal, headers: { + ...headers, authorization: `Bearer ${reveal(target.apiKey)}`, "content-type": "application/json", }, diff --git a/e2e/support/lost-connections.ts b/e2e/support/lost-connections.ts new file mode 100644 index 0000000000..c09ddb873b --- /dev/null +++ b/e2e/support/lost-connections.ts @@ -0,0 +1,670 @@ +/** + * Requests to a stage go on connections of their own, and any lost before a response is recorded. + * + * Deployed runs on 16 November lost stage requests in bursts: reset before any response, never + * reaching a Worker, while the stage kept answering. The recording that followed (#4688, #4695) + * showed run #1082 lose 16, every one on a reused connection, 12 of them written 1-5 ms after that + * connection's previous response, across 14 scenarios in three minutes: the far end closed + * connections right after a response without saying so, and the next request crossed the close. + * Its GETs were sent again and answered; its POSTs and DELETEs could not be, since they may have + * been processed. + * + * So no request to a stage reuses a connection: each goes on its own, through a pool of Node's own + * kind with keep-alive off (undici `pipelining: 0`), and the connection closes after its response, + * streamed or not. A request then has nothing to cross, whatever its method, and nothing is ever + * sent twice. HTTP/2 was not the alternative: undici 7.30.1, Node 24.22.1's bundled client, fails + * the streams in flight on a `GOAWAY` without reading which ones the far end did not process. + * Measured for #1082's 11,612 harness requests at 34-39 ms more per new TLS connection, this adds + * about 25 s to a deployed run's 15.5-minute test phase over 16 workers; on the local lanes the + * stage is a loopback server and the cost is a local connect. + * + * Requests to anything else (hosted emulators, registries, the browser's own) are unchanged. + * + * The recording stays: it is how a loss of another kind would show. It watches the diagnostics + * channels undici publishes (`undici:client:connected` for a new connection, + * `undici:client:sendHeaders` with the socket a request is written to, `undici:request:bodySent`, + * `undici:request:trailers`) and the socket's own `data` event for a response's first byte, matched + * to each stage request by an `x-executor-e2e-request` id header. A stage request that fails in + * transport says whether its connection was new or reused, how many requests the connection had + * carried and how old it was, how much of the request was written, and whether any byte of a + * response arrived (`support/request-failure.ts`). With connections never reused, a loss records + * "new connection": a different fault from the one above, such as a connection the far end refused + * or reset. + * + * Every stage request lost in transport (before any response, or after one began) is kept with its + * test file and scenario. The + * scenario lifecycle saves a scenario's in the run's `report/lost-requests/` + * (`e2e/lost-requests.ts`) and its evidence gets `lost-requests.json`; the setup file saves those + * made outside any scenario (`beforeAll`, `afterAll`, a `layer`'s set-up and teardown) under the + * file, and the suite's preparation saves its own. The job summary (`e2e/lost-request-summary.ts`) + * counts them all, with the times. + * + * A request of which nothing was written to any connection is sent again, whatever its method: its + * connect or TLS handshake failed, so no byte of it left the process. Connections are never reused, + * so in a burst of new connections refused or reset (#1089 lost 9 requests that way, most "never + * written to a connection" after 80-211 ms) every request needed a new one and failed with it. It + * is sent again on a new connection at most 5 times, 1, 2, 4, 8 and 16 s after each failure (31 s + * of waiting, ending inside a scenario's 60 s), and never at or past the scenario's own bound or + * after its caller gave up. Only a failure of the connection itself counts (refused, reset, timed + * out, a failed TLS handshake); one that would fail the same way again (an invalid argument, a + * certificate that does not verify) is not repeated. A request whose headers were written, or whose + * response had begun, is never sent again. Nor is one whose body may have been read (a stream). + * + * Why "nothing written" means no byte left the process, for undici 7.30.1 (Node 24.22.1): + * - The connector hands undici a socket only on `secureConnect` (`connect` for plain HTTP). A + * refused connect or a failed handshake fails the request before undici's `writeH1` runs. + * - In `writeH1`, `undici:client:sendHeaders` is published, synchronously, before the request's + * first write to the socket (`writeBuffer`, `writeStream`, `writeIterable`, `writeBlob`). Nothing + * before it writes to the socket. This module marks the request written in that publish. + * - HTTP/2 is off: the pool sets no `allowH2`, so the connector offers only `http/1.1`. + * - Each attempt carries its own `x-executor-e2e-request` id, and the publish carries the request + * it writes. A concurrent request's write can never mark this one, and an attempt never inherits + * an earlier attempt's state. + * So a request that never got a connection, and one whose connection opened and failed before + * undici wrote to it, both read `written: "nothing"`. A request undici began to write reads + * "headers" or "whole request" even if the write then failed: the error lies only towards not + * sending again. + * + * Each repeat is kept with the request (`repeats`: when each attempt failed, its code and time). A + * request answered after repeats is recorded as "sent again: answered", and one that stays unsent + * as "lost: never written" with why the repeats stopped. Both reach the scenario's evidence, the + * job summary and, when the scenario fails anyway, its failure text. + * + * The test client only: the product's own clients are other processes and are unchanged. + */ +import { subscribe } from "node:diagnostics_channel"; +import { randomBytes } from "node:crypto"; +import { shortPath } from "../lost-requests.ts"; + +export const requestIdHeader = "x-executor-e2e-request"; + +/** What the client saw of one request's connection before it failed. */ +export interface SentFacts { + /** A new connection, or one an earlier request had used. Absent: the request was never written. */ + readonly connection?: "new" | "reused"; + /** How many requests the connection had carried before this one. */ + readonly earlierRequests?: number; + /** How long ago the connection was opened. */ + readonly connectionAgeMs?: number; + /** How long the reused connection had been idle since its last response completed. */ + readonly idleMs?: number; + /** How much of the request was written: nothing, the headers, or the whole request. */ + readonly written: "nothing" | "headers" | "whole request"; + /** Whether any byte of a response arrived before the failure. */ + readonly responseStarted: boolean; +} + +/** + * Where a lost request was lost, or that it was answered after being sent again: only a request of + * which nothing was written is ever sent again. + */ +export type LostOutcome = + | "lost: new connection" + | "lost: reused connection" + | "lost: response had begun" + | "lost: never written" + | "sent again: answered"; + +/** One attempt of a request that failed with nothing written. */ +export interface UnsentAttempt { + readonly at: string; + readonly code?: string; + readonly elapsedMs: number; +} + +/** Why a request that stayed unsent was not sent again. */ +export type RepeatsStopped = + | "5 repeats used" + | "the scenario's bound" + | "the caller gave up" + | "its body cannot be sent twice" + | "its failure is not a connection's"; + +/** The waits before each repeat of a request of which nothing was written. */ +export const unsentRepeatDelaysMs = [1_000, 2_000, 4_000, 8_000, 16_000] as const; + +/** One stage request lost in transport, as the record and the evidence keep it. */ +export interface LostRequest { + readonly at: string; + /** The scenario's title as Vitest names it; absent outside any scenario. */ + readonly scenario?: string; + /** The test file, when a test file was running. */ + readonly file?: string; + readonly method: string; + /** The path only: a query can hold identifiers. */ + readonly path: string; + readonly code?: string; + /** From the first attempt to the last one's end, the repeats' waits included. */ + readonly elapsedMs: number; + readonly sent: SentFacts; + readonly outcome: LostOutcome; + /** The failed attempts of a request sent again, oldest first; absent when it was not. */ + readonly repeats?: ReadonlyArray; + /** Why a request of which nothing was written was not sent again. */ + readonly stopped?: RepeatsStopped; +} + +/** + * One stage request a scenario sent, in its `stage-requests.json`: the method, the path with ids + * shortened, when it started and when its response's headers arrived or it failed, the status or + * the failure's code, the trace id it carried, and whether its connection was new. A request lost + * in transport also says where (`outcome`), and one sent again how many attempts it took. Never a + * query, a body, a header value other than the trace id, a token or a cookie. A request still open + * when the scenario ended has no `endedAt`. + */ +export interface StageRequest { + readonly method: string; + readonly path: string; + readonly startedAt: string; + endedAt?: string; + durationMs?: number; + status?: number; + /** A transport code (`ECONNRESET`), `aborted` when the caller gave up, or the error's class. */ + failure?: string; + traceId?: string; + connection?: "new" | "reused"; + attempts?: number; + outcome?: LostOutcome; + stopped?: RepeatsStopped; +} + +/** The most rows one scenario's record keeps; the oldest are dropped first, and counted. */ +export const stageRequestLimit = 2000; + +/** Each scenario's stage requests, by its name as `currentScenario` gives it. */ +const stageRequests = new Map; dropped: number }>(); + +const startStageRequest = (scenario: string, row: StageRequest) => { + let record = stageRequests.get(scenario); + if (record === undefined) stageRequests.set(scenario, (record = { rows: [], dropped: 0 })); + record.rows.push(row); + if (record.rows.length > stageRequestLimit) { + record.rows.shift(); + record.dropped += 1; + } +}; + +const endStageRequest = (row: StageRequest | undefined, started: number) => { + if (row === undefined) return; + const now = Date.now(); + row.endedAt = new Date(now).toISOString(); + row.durationMs = now - started; +}; + +/** The trace id a request carries in its `traceparent`, when it is one. */ +const traceIdOf = (input: RequestInfo | URL, init: RequestInit | undefined) => { + const header = + new Headers(init?.headers).get("traceparent") ?? + (input instanceof Request ? input.headers.get("traceparent") : null); + const id = header?.split("-")[1]; + return id !== undefined && /^[0-9a-f]{32}$/u.test(id) ? id : undefined; +}; + +/** What a failed request's record says of it: its code, `aborted`, or the error's class. */ +const failureOf = (error: unknown) => + firstCode(error) ?? + (error instanceof Error && error.name === "AbortError" + ? "aborted" + : error instanceof Error + ? error.name + : "other"); + +/** One scenario's record of its stage requests, still unsaved, matched as `lostRequestsOf` is. */ +export const stageRequestsOf = (scenario: string) => { + for (const [name, record] of stageRequests) + if (name.endsWith(scenario)) + return { limit: stageRequestLimit, dropped: record.dropped, rows: [...record.rows] }; + return { limit: stageRequestLimit, dropped: 0, rows: [] as Array }; +}; + +/** Forget one scenario's record once its evidence is written. */ +export const forgetStageRequests = (scenario: string) => { + for (const name of stageRequests.keys()) if (name.endsWith(scenario)) stageRequests.delete(name); +}; + +interface SocketState { + readonly connectedAt: number; + requests: number; + lastCompletedAt?: number; +} +interface Observed { + connection?: "new" | "reused"; + earlierRequests?: number; + connectionAgeMs?: number; + idleMs?: number; + headersWritten: boolean; + bodyWritten: boolean; + responseStarted: boolean; + /** Stops watching the socket for this request's first response byte. */ + unwatch?: () => void; +} + +const sockets = new WeakMap(); +const socketOf = new WeakMap(); +const observed = new Map(); +const stageOrigins = new Set(); +const kept: Array = []; +let currentScenario: () => string | undefined = () => undefined; +let currentFile: () => string | undefined = () => undefined; +/** When the running scenario's bound ends (epoch ms); no repeat starts at or past it. */ +let currentDeadline: () => number | undefined = () => undefined; +/** Requests waiting to be sent again now, for a scenario that fails meanwhile. */ +const waiting = new Set<{ + readonly scenario?: string; + readonly since: number; + readonly path: string; +}>(); +/** + * Set on `globalThis` once a copy of this module has wrapped `fetch`. A second copy of the module in + * one process (a module graph loaded twice) must not wrap it again: its id header would replace the + * first wrapper's, and the first would then read a written request as never written. + */ +const installedKey = Symbol.for("executor.e2e.lostConnections.installed"); + +/** The id header of an undici request, whose headers are a flat name, value array or a string. */ +const idOf = (request: { headers?: unknown }): string | undefined => { + const headers = request.headers; + if (Array.isArray(headers)) { + for (let index = 0; index + 1 < headers.length; index += 2) + if (String(headers[index]).toLowerCase() === requestIdHeader) + return String(headers[index + 1]); + return undefined; + } + if (typeof headers === "string") + return new RegExp(`^${requestIdHeader}:\\s*([0-9a-f]+)\\s*$`, "imu").exec(headers)?.[1]; + return undefined; +}; + +const entryOf = (message: unknown) => { + const id = idOf((message as { request: { headers?: unknown } }).request); + return id === undefined ? undefined : observed.get(id); +}; + +const watch = () => { + subscribe("undici:client:connected", (message) => { + sockets.set((message as { socket: object }).socket, { connectedAt: Date.now(), requests: 0 }); + }); + subscribe("undici:client:sendHeaders", (message) => { + const { request, socket } = message as { request: object; socket: object }; + const state = sockets.get(socket) ?? { connectedAt: Date.now(), requests: 0 }; + sockets.set(socket, state); + socketOf.set(request, socket); + const entry = entryOf(message); + if (entry !== undefined) { + const now = Date.now(); + entry.connection = state.requests > 0 ? "reused" : "new"; + entry.earlierRequests = state.requests; + entry.connectionAgeMs = now - state.connectedAt; + if (state.requests > 0 && state.lastCompletedAt !== undefined) + entry.idleMs = now - state.lastCompletedAt; + entry.headersWritten = true; + // undici reads the socket on `readable`, so a `data` listener sees each read without + // changing how undici reads. + const stream = socket as NodeJS.ReadableStream; + const onData = () => { + entry.responseStarted = true; + }; + stream.once("data", onData); + entry.unwatch = () => stream.removeListener("data", onData); + } + state.requests += 1; + }); + subscribe("undici:request:bodySent", (message) => { + const entry = entryOf(message); + if (entry !== undefined) entry.bodyWritten = true; + }); + subscribe("undici:request:trailers", (message) => { + const socket = socketOf.get((message as { request: object }).request); + const state = socket === undefined ? undefined : sockets.get(socket); + if (state !== undefined) state.lastCompletedAt = Date.now(); + }); +}; + +const factsOf = (entry: Observed): SentFacts => ({ + ...(entry.connection !== undefined && { connection: entry.connection }), + ...(entry.earlierRequests !== undefined && { earlierRequests: entry.earlierRequests }), + ...(entry.connectionAgeMs !== undefined && { connectionAgeMs: entry.connectionAgeMs }), + ...(entry.idleMs !== undefined && { idleMs: entry.idleMs }), + written: !entry.headersWritten ? "nothing" : entry.bodyWritten ? "whole request" : "headers", + responseStarted: entry.responseStarted, +}); + +/** Where a request with these facts was lost. */ +export const lostOutcome = (facts: SentFacts): LostOutcome => + facts.responseStarted + ? "lost: response had begun" + : facts.connection === "reused" + ? "lost: reused connection" + : facts.connection === "new" + ? "lost: new connection" + : "lost: never written"; + +/** The facts a failed stage request carries on its error, for `request-failure.ts` to read. */ +export const sentFactsKey = Symbol.for("executor.e2e.sentFacts"); +/** What a lost request's error carries: its last attempt's facts, and its repeats when nothing was written. */ +export type SentOnError = SentFacts & { + readonly outcome?: LostOutcome; + /** How many attempts failed with nothing written, the first included. */ + readonly repeats?: number; + /** From the first attempt to the last one's end. */ + readonly waitedMs?: number; + readonly stopped?: RepeatsStopped; +}; +export const sentFacts = (error: unknown): SentOnError | undefined => + typeof error === "object" && error !== null && sentFactsKey in error + ? (error as Record)[sentFactsKey] + : undefined; + +/** + * Node and undici codes, such as ECONNRESET. OpenSSL's are built from its reason text and can hold a + * slash: a TLS alert in the handshake is `ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE`. + */ +const codePattern = /^[A-Z][A-Z0-9_/]{1,63}$/u; + +const firstCode = (error: unknown): string | undefined => { + let current: unknown = error; + for (let level = 0; level < 6 && typeof current === "object" && current !== null; level++) { + const code = (current as { code?: unknown }).code; + if (typeof code === "string" && codePattern.test(code)) return code; + current = (current as { cause?: unknown }).cause; + } + return undefined; +}; + +/** A transport failure with no response, as opposed to an abort the caller asked for. */ +const lostInTransport = (error: unknown) => + error instanceof TypeError && + !(error.cause instanceof DOMException) && + firstCode(error) !== undefined; + +type Pool = { readonly close: () => Promise }; +let stagePool: Promise | undefined; + +/** + * The stage's pool: Node's own dispatcher class with keep-alive off, so each request opens its own + * connection and the connection closes after the response. Node creates its global dispatcher, and + * so makes its class reachable, on its first `fetch`; a `data:` URL does that without a connection. + */ +const poolFor = (original: typeof fetch): Promise => + (stagePool ??= ( + (globalThis as Record)[globalDispatcher] === undefined + ? original("data:,").then((response) => response.arrayBuffer()) + : Promise.resolve() + ).then(() => { + const global = (globalThis as Record)[globalDispatcher] as { + constructor: new (options: { pipelining: number }) => Pool; + }; + return new global.constructor({ pipelining: 0 }); + })); +const globalDispatcher = Symbol.for("undici.globalDispatcher.1"); + +const tagged = (input: RequestInfo | URL, init: RequestInit | undefined, id: string) => { + const headers = new Headers(input instanceof Request ? input.headers : undefined); + new Headers(init?.headers).forEach((value, name) => headers.set(name, value)); + headers.set(requestIdHeader, id); + return { ...init, headers }; +}; + +/** + * Whether a request's body can be given to `fetch` again unchanged: none, or one `fetch` reads + * afresh each time (a string, bytes, a Blob, form fields). A stream, an async iterable or anything + * else may have been read by the first attempt. + */ +const replayable = (input: RequestInfo | URL, init: RequestInit | undefined) => { + if (input instanceof Request && input.body !== null && init?.body === undefined) return false; + const body = init?.body; + return ( + body === null || + body === undefined || + typeof body === "string" || + body instanceof ArrayBuffer || + ArrayBuffer.isView(body) || + body instanceof Blob || + body instanceof URLSearchParams || + body instanceof FormData + ); +}; + +/** + * Failures of the connection itself: refused, reset, timed out, unreachable, closed, or a TLS + * handshake that failed. Anything else that fails before a write (an invalid argument, a + * certificate that does not verify) would fail again the same way, so it is not repeated. + */ +const connectionFailure = + /^(?:ECONNREFUSED|ECONNRESET|EPIPE|ETIMEDOUT|EHOSTUNREACH|ENETUNREACH|EAI_AGAIN|UND_ERR_CONNECT_TIMEOUT|UND_ERR_SOCKET|UND_ERR_CLOSED|EPROTO|ERR_SSL_[\w/]+)$/u; + +/** Waits, unless the caller gives up first: then false. */ +const pause = (milliseconds: number, signal: AbortSignal | null | undefined) => + new Promise((resolve) => { + if (signal?.aborted === true) return resolve(false); + const timer = setTimeout(() => { + signal?.removeEventListener("abort", stop); + resolve(true); + }, milliseconds); + const stop = () => { + clearTimeout(timer); + resolve(false); + }; + signal?.addEventListener("abort", stop, { once: true }); + }); + +/** + * `fetch` that gives each stage request a connection of its own, records a lost one, and sends again + * one of which nothing was written to any connection. + */ +export const lostConnectionFetch = (original: typeof fetch): typeof fetch => + ((input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (!stageOrigins.has(url.origin)) return original(input, init); + const method = ( + init?.method ?? (input instanceof Request ? input.method : "GET") + ).toUpperCase(); + const signal = init?.signal ?? (input instanceof Request ? input.signal : undefined); + const scenario = currentScenario(); + const file = currentFile(); + const traceId = scenario === undefined ? undefined : traceIdOf(input, init); + const row: StageRequest | undefined = + scenario === undefined + ? undefined + : { + method, + path: shortPath(url.pathname), + startedAt: new Date().toISOString(), + ...(traceId !== undefined && { traceId }), + }; + if (row !== undefined) startStageRequest(scenario!, row); + const record = (fields: Omit) => + kept.push({ + at: new Date().toISOString(), + ...(scenario !== undefined && { scenario }), + ...(file !== undefined && { file }), + method, + path: url.pathname, + ...fields, + }); + const sent = Date.now(); + return poolFor(original).then((dispatcher) => { + const started = Date.now(); + const repeats: Array = []; + // Each attempt is its own request to undici, with its own id and record of what was written. + const attempt = (): Promise => { + const id = randomBytes(8).toString("hex"); + const entry: Observed = { + headersWritten: false, + bodyWritten: false, + responseStarted: false, + }; + observed.set(id, entry); + const attemptStarted = Date.now(); + return original(input, { ...tagged(input, init, id), dispatcher } as RequestInit) + .finally(() => { + // This attempt is over: its facts stay in `entry`, read below. + entry.unwatch?.(); + observed.delete(id); + }) + .then( + (response) => { + if (row !== undefined) { + row.status = response.status; + if (entry.connection !== undefined) row.connection = entry.connection; + if (repeats.length > 0) row.attempts = repeats.length + 1; + endStageRequest(row, sent); + } + if (repeats.length > 0) + record({ + ...(repeats[0]!.code !== undefined && { code: repeats[0]!.code }), + elapsedMs: Date.now() - started, + sent: { written: "nothing", responseStarted: false }, + outcome: "sent again: answered", + repeats, + }); + return response; + }, + (error: unknown) => { + if (!lostInTransport(error)) { + if (row !== undefined && row.endedAt === undefined) { + row.failure = failureOf(error); + endStageRequest(row, sent); + } + throw error; + } + const facts = factsOf(entry); + const code = firstCode(error); + const lost = (stopped?: RepeatsStopped) => { + const outcome = lostOutcome(facts); + if (row !== undefined) { + row.failure = code ?? "other"; + row.outcome = outcome; + if (facts.connection !== undefined) row.connection = facts.connection; + if (repeats.length > 1) row.attempts = repeats.length; + if (stopped !== undefined) row.stopped = stopped; + endStageRequest(row, sent); + } + record({ + ...(code !== undefined && { code }), + elapsedMs: Date.now() - started, + sent: facts, + outcome, + ...(repeats.length > 0 && { repeats: [...repeats] }), + ...(stopped !== undefined && { stopped }), + }); + Object.defineProperty(error, sentFactsKey, { + value: { + ...facts, + outcome, + repeats: repeats.length, + waitedMs: Date.now() - started, + ...(stopped !== undefined && { stopped }), + }, + }); + return error; + }; + // Undici wrote some of it, or a response began: it may have been processed. + if (facts.written !== "nothing") throw lost(); + repeats.push({ + at: new Date().toISOString(), + ...(code !== undefined && { code }), + elapsedMs: Date.now() - attemptStarted, + }); + if (code === undefined || !connectionFailure.test(code)) + throw lost("its failure is not a connection's"); + if (!replayable(input, init)) throw lost("its body cannot be sent twice"); + const delay = unsentRepeatDelaysMs[repeats.length - 1]; + if (delay === undefined) throw lost("5 repeats used"); + const deadline = currentDeadline(); + if (deadline !== undefined && Date.now() + delay >= deadline) + throw lost("the scenario's bound"); + const wait = { + ...(scenario !== undefined && { scenario }), + since: started, + path: url.pathname, + }; + waiting.add(wait); + return pause(delay, signal).then((waited) => { + waiting.delete(wait); + if (!waited) throw lost("the caller gave up"); + return attempt(); + }); + }, + ); + }; + return attempt(); + }); + }) as typeof fetch; + +/** + * A failed scenario's note on its requests that never got a connection: those that stayed unsent, + * and those still waiting to be sent again when it failed. Empty when there are none. + */ +export const unsentNote = (scenario: string, now = Date.now()) => { + const unsent = [ + ...kept + .filter( + (lost) => + lost.scenario?.endsWith(scenario) === true && lost.outcome === "lost: never written", + ) + .map((lost) => lost.elapsedMs), + ...[...waiting] + .filter((wait) => wait.scenario?.endsWith(scenario) === true) + .map((wait) => now - wait.since), + ]; + if (unsent.length === 0) return ""; + const longest = (Math.max(...unsent) / 1000).toFixed(1); + return unsent.length === 1 + ? `one stage request was unsent for ${longest} s: its connect or TLS handshake failed on every attempt (lost-requests.json)` + : `${unsent.length} stage requests were unsent, the longest for ${longest} s: their connects or TLS handshakes failed on every attempt (lost-requests.json)`; +}; + +/** + * Start a scenario's stage origins afresh with the run's own, and say how a record names its + * scenario and test file. Each target a scenario then starts adds its origins (`addStageOrigins`) + * as soon as it exists, before any request is sent to it. + */ +export const configureLostConnections = (options: { + readonly origins: Iterable; + readonly scenario?: () => string | undefined; + readonly file?: () => string | undefined; + /** When the running scenario's bound ends (epoch ms): no repeat starts at or past it. */ + readonly deadline?: () => number | undefined; +}) => { + stageOrigins.clear(); + addStageOrigins(options.origins); + if (options.scenario !== undefined) currentScenario = options.scenario; + if (options.file !== undefined) currentFile = options.file; + if (options.deadline !== undefined) currentDeadline = options.deadline; +}; + +/** Add origins a scenario's own target serves. */ +export const addStageOrigins = (origins: Iterable) => { + for (const origin of origins) stageOrigins.add(new URL(origin).origin); +}; + +/** Watch the channels and wrap `globalThis.fetch`, once per process. */ +export const installLostConnections = () => { + const global = globalThis as Record; + if (global[installedKey] === true) return; + global[installedKey] = true; + watch(); + globalThis.fetch = lostConnectionFetch(globalThis.fetch); +}; + +/** The lost requests this process kept for one scenario, by its title, still unsaved. */ +export const lostRequestsOf = (scenario: string) => + kept.filter((lost) => lost.scenario !== undefined && lost.scenario.endsWith(scenario)); + +/** + * Remove and return the records a saver writes: one scenario's by its title, or, with no title, + * those made outside any scenario. Each record is saved once. + */ +export const takeLostRequests = (scenario?: string) => { + const taken = kept.filter((lost) => + scenario === undefined + ? lost.scenario === undefined + : lost.scenario !== undefined && lost.scenario.endsWith(scenario), + ); + for (const lost of taken) kept.splice(kept.indexOf(lost), 1); + return taken; +}; + +/** Remove and return every record still unsaved, for the saver at the end of a test file. */ +export const takeAllLostRequests = () => kept.splice(0, kept.length); diff --git a/e2e/support/machine-speed.ts b/e2e/support/machine-speed.ts new file mode 100644 index 0000000000..b6499ece32 --- /dev/null +++ b/e2e/support/machine-speed.ts @@ -0,0 +1,59 @@ +/** + * How fast this machine is now, against a reference. CI's runners do the same work up to about + * three times slower on weekday afternoons than at night or at weekends, on every runner size + * (`.reference` report runner-speed-16-nov, 9-16 Nov). The catalog-scale scenario calibrates its + * search budget with it, and a scenario or hook that times out records it (lifecycle.ts), so a + * time-out in a slow hour reads as one. + */ +import { createHash } from "node:crypto"; +import { Effect } from "effect"; + +/** + * `machineWork` on the reference machine: CI's `blacksmith-8vcpu` scale lane in its normal hours + * (weekends, and weekdays 00-12Z), where the catalog-scale search took a median 5.0 s (38 runs, + * 9-16 Nov). Derived from a host whose search took 18.0-19.9 times its `machineWork`: 5,000 ms / + * 18.9. Re-derive it from CI itself: the median `machine` of the "Search the whole catalog, + * calibrated" line in the scale lane's job logs, over normal-hours runs. + */ +export const referenceMachineWorkMs = 265; + +const machineBuffer = Buffer.alloc(1 << 20, 7); + +/** + * A fixed piece of CPU work with no product code in it: an integer loop and SHA-256 over a fixed + * 1 MiB buffer, three times, in milliseconds (about 0.3 s). It runs in the test process; the + * server's processes slow with it (in CI's slow hours the scale lane's `bun` and workerd CPU seconds + * stretched 2.3x and 2.7x for the same work, the test process 2.0x, so it errs low). + */ +export const machineWorkMs = (): number => { + const started = performance.now(); + let digest = ""; + for (let pass = 0; pass < 3; pass++) { + let x = 0x9e3779b9; + for (let i = 0; i < 60_000_000; i++) { + x ^= x << 13; + x ^= x >>> 17; + x ^= x << 5; + } + digest += String(x); + for (let i = 0; i < 72; i++) + digest = createHash("sha256").update(machineBuffer).update(digest).digest("hex"); + } + if (digest.length !== 64) throw new Error("machine work produced no digest"); + return Math.round(performance.now() - started); +}; + +export const machineWork = Effect.sync(machineWorkMs); + +/** + * What a time-out says about the machine, as one line: `timed out at 120 s (bound 120 s; machine + * 1.9× slower than reference: 615 ms against 265 ms)`, or that it ran at reference speed or faster. + */ +export const timeoutLine = (boundMs: number, elapsedMs: number, machineMs: number) => { + const slowdown = machineMs / referenceMachineWorkMs; + const speed = + slowdown >= 1.1 + ? `machine ${slowdown.toFixed(1)}× slower than reference: ${machineMs} ms against ${referenceMachineWorkMs} ms` + : `machine at reference speed or faster: ${machineMs} ms against ${referenceMachineWorkMs} ms, so not a slow runner`; + return `timed out at ${Math.round(elapsedMs / 1000)} s (bound ${Math.round(boundMs / 1000)} s; ${speed})`; +}; diff --git a/e2e/support/native-self-host.ts b/e2e/support/native-self-host.ts index 70a91420c4..3803fbc77e 100644 --- a/e2e/support/native-self-host.ts +++ b/e2e/support/native-self-host.ts @@ -58,7 +58,7 @@ export const nativeClientProxy = (upstream: string) => /** * One request to the collector's private Unix socket. The packaged host does not publish a TCP - * port for Motel. + * port for the collector. */ export const collectorRequest = (socketPath: string, requestPath: string, body?: string) => Effect.tryPromise({ @@ -107,7 +107,7 @@ export const nativeSelfHost = (environment: Readonly>) => Config.withDefault(path.resolve(".local/native-auth/executor-host")), ); const runtime = path.resolve("apps/hosted/self-host/dist/workerd"); - for (const artifact of [binary, `${runtime}/workerd.capnp`, `${runtime}/motel.capnp`]) + for (const artifact of [binary, `${runtime}/workerd.capnp`, `${runtime}/executor-collector`]) if (!(yield* fs.exists(artifact))) return yield* Effect.die( `Missing ${artifact}. Prepare the native self-host artifacts as described in e2e/README.md; rebuild them after source changes.`, @@ -137,7 +137,7 @@ export const nativeSelfHost = (environment: Readonly>) => EXECUTOR_DATA_DIR: dataDirectory, EXECUTOR_MOTEL_DATA_DIR: motelDirectory, EXECUTOR_RUNTIME_DIR: runtime, - // Leave OTEL_EXPORTER_OTLP_* unset so the product uses its MOTEL binding. Setting an + // Leave OTEL_EXPORTER_OTLP_* unset so the product uses its COLLECTOR binding. Setting an // endpoint would export to that URL instead of the private Unix socket. ...fullTracing, ...environment, diff --git a/e2e/support/npm-registry.ts b/e2e/support/npm-registry.ts index 2a249b27fa..928e9aa64b 100644 --- a/e2e/support/npm-registry.ts +++ b/e2e/support/npm-registry.ts @@ -24,6 +24,72 @@ const upstream = "https://registry.npmjs.org"; */ export const stalledPackage = "@executor-fixture/stalled-package"; const stalledPath = `/@${encodeURIComponent(stalledPackage.slice(1))}`; + +/** + * A synthetic package whose one module is far over the build's 2,000,000-byte limit on the app's + * own files: dense generated objects, which cost the compiler what a large bundled library does. + * Package files are not limited, so an app that imports it builds. The archive is made here. + */ +export const largeModulePackage = { + name: "@executor-fixture/large-module", + version: "1.0.0", + bytes: 3_000_000, +} as const; +/** `index.js` of exactly `bytes` bytes, exporting `count`, the number of its objects. */ +const largeModule = (() => { + const entry = (at: number) => + ` { id: ${at}, name: "item ${at}", tags: ["a", "b"], nested: { x: ${at}, y: "z" } },\n`; + const open = "const items = [\n"; + const close = "];\nexport const count = items.length;\n"; + const entries: string[] = []; + let used = open.length + close.length + 3; + while (used + entry(entries.length).length <= largeModulePackage.bytes) { + used += entry(entries.length).length; + entries.push(entry(entries.length)); + } + return { + source: `${open}${entries.join("")}${close}//${" ".repeat(largeModulePackage.bytes - used)}\n`, + count: entries.length, + }; +})(); +/** The number of objects the large module exports as `count`. */ +export const largeModuleCount = largeModule.count; +/** A gzipped ustar archive of `files`, each under `package/`, as npm packs one. */ +const packArchive = (files: Readonly>) => { + const blocks: Buffer[] = []; + for (const [name, text] of Object.entries(files)) { + const body = Buffer.from(text); + const header = Buffer.alloc(512); + header.write(`package/${name}`, 0, 100); + header.write("0000644\0", 100); + header.write("0000000\0", 108); + header.write("0000000\0", 116); + header.write(`${body.length.toString(8).padStart(11, "0")}\0`, 124); + header.write("00000000000\0", 136); + header.fill(" ", 148, 156); + header.write("0", 156); + header.write("ustar\0", 257); + header.write("00", 263); + const sum = header.reduce((total, byte) => total + byte, 0); + header.write(`${sum.toString(8).padStart(6, "0")}\0 `, 148); + blocks.push(header, body, Buffer.alloc((512 - (body.length % 512)) % 512)); + } + blocks.push(Buffer.alloc(1024)); + const tar = Buffer.concat(blocks); + return Effect.promise(() => + new Response(new Blob([tar]).stream().pipeThrough(new CompressionStream("gzip"))).arrayBuffer(), + ).pipe(Effect.map((gzipped) => new Uint8Array(gzipped))); +}; +const largeModuleArchive = packArchive({ + "package.json": JSON.stringify({ + name: largeModulePackage.name, + version: largeModulePackage.version, + type: "module", + main: "index.js", + exports: { ".": "./index.js" }, + }), + "index.js": largeModule.source, +}); const JsonObject = Schema.Record(Schema.String, Schema.Json); const Manifest = Schema.Struct({ name: Schema.Literal("apps"), version: Schema.NonEmptyString }); class RegistryFailed extends Schema.TaggedError()("RegistryFailed", { @@ -142,6 +208,8 @@ const lockedReply = (staged: Staged) => ...(name === "apps" ? { [staged.version]: staged.entry(base) } : {}), }, }); + const largeArchivePath = tarballPath(largeModulePackage.name, largeModulePackage.version); + const largeArchive = yield* largeModuleArchive; const replies: Replies = (url, base) => { if (url === staged.tarballPath) return Effect.succeed({ @@ -149,6 +217,29 @@ const lockedReply = (staged: Staged) => type: "application/octet-stream", body: staged.bytes, }); + if (url === largeArchivePath) + return Effect.succeed({ + status: 200, + type: "application/octet-stream", + body: largeArchive, + }); + if (decodeURIComponent(url.slice(1)) === largeModulePackage.name) + return Effect.succeed( + json(200, { + name: largeModulePackage.name, + "dist-tags": { latest: largeModulePackage.version }, + versions: { + [largeModulePackage.version]: { + name: largeModulePackage.name, + version: largeModulePackage.version, + dist: { + tarball: `${base}${largeArchivePath}`, + integrity: integrityOf(largeArchive), + }, + }, + }, + }), + ); const archive = archives.get(url); if (archive !== undefined) return read(archive.name, archive.version).pipe( diff --git a/e2e/support/openapi-import.ts b/e2e/support/openapi-import.ts index 7b2f4d1fca..f89e2a653a 100644 --- a/e2e/support/openapi-import.ts +++ b/e2e/support/openapi-import.ts @@ -314,6 +314,130 @@ export const requestEchoUpstream = Effect.gen(function* () { return { origin: `http://127.0.0.1:${server.address.port}` }; }); +/** A HuJSON policy file: a comment and trailing commas, so it is not valid JSON. */ +export const hujsonPolicy = `// Engineers reach CI runners. +{ + "grants": [ + { "src": ["group:eng"], "dst": ["tag:ci"], "ip": ["*"], }, + ], +} +`; + +/** + * JSON answers that are not `application/json`, by path: AWS's JSON protocols and a `+json` type + * outside `application`. + */ +export const jsonAnswers = { + "/aws/json-1.0": { + contentType: "application/x-amz-json-1.0", + body: { TableNames: ["synthetic-table"] }, + }, + "/aws/json-1.1": { + contentType: "application/x-amz-json-1.1", + body: { StreamNames: ["synthetic-stream"] }, + }, + "/scene": { contentType: "model/gltf+json", body: { asset: { version: "2.0" } } }, +} as const; + +/** + * An API whose bodies are JSON or text by media type. Its policy file is shaped like Tailscale's: + * it takes the policy as JSON, HuJSON or a JSON merge patch, and answers with it as HuJSON. AWS's + * JSON protocols and a glTF scene take and answer JSON under their own types (`jsonAnswers`). It + * records each request: method, path, Content-Type and body as sent. + */ +export const mediaTypeUpstream = Effect.gen(function* () { + const requests = yield* Ref.make< + readonly { + readonly method: string; + readonly url: string; + readonly contentType: string | null; + readonly body: string; + }[] + >([]); + const services = yield* Layer.build( + HttpRouter.serve( + Layer.mergeAll( + HttpRouter.add( + "GET", + "/policy", + Effect.succeed( + HttpServerResponse.text(hujsonPolicy, { contentType: "application/hujson" }), + ), + ), + HttpRouter.add( + "POST", + "/*", + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const observed = { + method: request.method, + url: request.url, + contentType: request.headers["content-type"] ?? null, + body: yield* request.text, + }; + yield* Ref.update(requests, (previous) => [...previous, observed]); + const answer = Object.entries(jsonAnswers).find(([path]) => path === request.url)?.[1]; + return answer === undefined + ? yield* HttpServerResponse.json({}) + : HttpServerResponse.text(JSON.stringify(answer.body), { + contentType: answer.contentType, + }); + }), + ), + ), + { disableLogger: true, disableListenLog: true }, + ).pipe(Layer.provideMerge(NodeHttpServer.layer(createServer, { host: "127.0.0.1", port: 0 }))), + ); + const server = yield* HttpServer.HttpServer.pipe(Effect.provideContext(services)); + if (!("port" in server.address)) return yield* Effect.die("Fixture must listen on TCP"); + const origin = `http://127.0.0.1:${server.address.port}`; + const object = { type: "object" }; + /** An operation that takes and answers one JSON media type. */ + const jsonOperation = (operationId: string, type: string) => ({ + post: { + operationId, + requestBody: { required: true, content: { [type]: { schema: object } } }, + responses: { "200": { description: "OK", content: { [type]: { schema: object } } } }, + }, + }); + const document = { + openapi: "3.1.0", + info: { title: "Media types", version: "1" }, + servers: [{ url: origin }], + paths: { + "/policy": { + get: { + operationId: "getPolicyFile", + responses: { + "200": { + description: "The policy file", + content: { "application/hujson": { schema: { type: "string" } } }, + }, + }, + }, + post: { + operationId: "setPolicyFile", + requestBody: { + required: true, + content: { + "application/json": { + schema: { type: "object", properties: { grants: { type: "array" } } }, + }, + "application/hujson": { schema: { type: "string" } }, + "application/merge-patch+json": { schema: { type: "object" } }, + }, + }, + responses: ok, + }, + }, + "/aws/json-1.0": jsonOperation("listTables", "application/x-amz-json-1.0"), + "/aws/json-1.1": jsonOperation("listStreams", "application/x-amz-json-1.1"), + "/scene": jsonOperation("putScene", "model/gltf+json"), + }, + }; + return { origin, document, requests: Ref.get(requests) }; +}); + /** * Shapes Executor v1 imported that the OpenAPI specification forbids or leaves unclear: schema * references to a component whose name is not URI-encoded, into another component's properties, diff --git a/e2e/support/operator-oauth.ts b/e2e/support/operator-oauth.ts index 262f0edb72..c590d0b03e 100644 --- a/e2e/support/operator-oauth.ts +++ b/e2e/support/operator-oauth.ts @@ -10,5 +10,15 @@ export const OperatorOAuthFixture = Schema.Struct({ clientId: Schema.String, clientSecret: Schema.String, }); +/** + * Synthetic credentials for the reviewed Executor for Google client, which every managed local + * Cloud turns on, so scenarios can check which sign-ins Cloud's own Google registration is offered + * for. Nothing signs in with them: only providers on Google's own endpoints match the client, and + * no scenario follows its authorization URL. + */ +export const syntheticCloudGoogle = { + clientId: "synthetic-cloud-google-client.apps.googleusercontent.com", + clientSecret: "synthetic-cloud-google-secret", +} as const; /** Where a managed Cloud run records its operator OAuth fixture, under the run directory. */ export const operatorOAuthFile = "operator-oauth.json"; diff --git a/e2e/support/otlp-collector.ts b/e2e/support/otlp-collector.ts index 5642aa5550..660bcdbce7 100644 --- a/e2e/support/otlp-collector.ts +++ b/e2e/support/otlp-collector.ts @@ -14,25 +14,30 @@ import { import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { createServer } from "node:http"; import { SpanQuery } from "./contracts.ts"; +import { refuseTruncated, searchLimit } from "./evidence.ts"; -/** workerd's control message once the collector socket accepts connections. */ -const Listening = Schema.Struct({ - event: Schema.Literal("listen"), - socket: Schema.Literal("motel"), - port: Schema.Number, -}); +/** The collector's one line on stdout once it accepts connections on its TCP port. */ +const Listening = Schema.Struct({ event: Schema.Literal("listening"), port: Schema.Number }); + +/** Bounds a scenario sets on its own collector, as the collector's flags name them. */ +export interface CollectorBounds { + readonly maxStoreMb?: number | undefined; + readonly retentionHours?: number | undefined; + readonly maxPending?: number | undefined; + readonly maxExportBytes?: number | undefined; +} /** - * Serve one Motel from the bundle `e2e:prepare` built, storing its data in `diagnostics/` - * and its output in `.log`. Every target in a run serves the same bundle directory, and a - * build replaces it, so nothing in a run builds it. `bounds` replaces the values of the bundle's - * own `MOTEL_OTEL_*` settings in a copy of its config, beside a copy of its worker. + * Serve one collector from the build `e2e:prepare` made, storing its data in `diagnostics/` + * and its log in `.log`. Every target in a run starts the same executable, and a build + * replaces it, so nothing in a run builds it. `bounds` are passed as its flags; the rest keep the + * collector's defaults, which are the product's. */ -const serveMotel = ( +const serveCollector = ( directory: string, name: string, log: string, - bounds: Readonly> = {}, + bounds: CollectorBounds = {}, ) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -40,61 +45,37 @@ const serveMotel = ( const processes = yield* ChildProcessSpawner.ChildProcessSpawner; const data = path.join(directory, "data/diagnostics", name); yield* fs.makeDirectory(data, { recursive: true }); - const bundle = path.resolve("packages/telemetry/dist/motel"); - if (!(yield* fs.exists(path.join(bundle, "motel.capnp")))) + const executable = path.resolve( + "packages/telemetry/dist/collector", + process.platform === "win32" ? "executor-collector.exe" : "executor-collector", + ); + if (!(yield* fs.exists(executable))) return yield* Effect.die("Run bun run e2e:prepare to build the telemetry collector first."); - let config = path.join(bundle, "motel.capnp"); - if (Object.keys(bounds).length > 0) { - let text = yield* fs.readFileString(config); - for (const [setting, value] of Object.entries(bounds)) { - const binding = `(name="${setting}",text="`; - if (!text.includes(binding)) - return yield* Effect.die(`The collector's config has no ${setting} setting.`); - text = text.replace( - new RegExp(`\\(name="${setting}",text="[^"]*"\\)`), - `${binding}${value}")`, - ); - } - config = path.join(directory, `${name}.capnp`); - yield* fs.writeFileString(config, text, { mode: 0o600 }); - yield* fs.copyFile(path.join(bundle, "motel.mjs"), path.join(directory, "motel.mjs")); - } - // The bundle's config names its directories and socket; this supplies their paths and a free port. + const flags = [ + ["--max-store-mb", bounds.maxStoreMb], + ["--retention-hours", bounds.retentionHours], + ["--max-pending", bounds.maxPending], + ["--max-export-bytes", bounds.maxExportBytes], + ].flatMap(([flag, value]) => (value === undefined ? [] : [`${flag}=${value}`])); const child = yield* processes.spawn( - ChildProcess.make( - path.resolve("node_modules/.bin", process.platform === "win32" ? "workerd.exe" : "workerd"), - [ - "serve", - "--control-fd=3", - `--directory-path=motel-data=${data}`, - `--directory-path=motel-assets=${path.join(bundle, "web/dist")}`, - "--socket-addr=motel=127.0.0.1:0", - config, - ], - { - cwd: data, - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - additionalFds: { fd3: { type: "output" } }, - forceKillAfter: "3 seconds", - }, - ), + ChildProcess.make(executable, ["serve", `--data=${data}`, "--listen=127.0.0.1:0", ...flags], { + cwd: directory, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + // It stops within 2.5 seconds of SIGTERM. + forceKillAfter: "3 seconds", + }), ); - yield* Effect.forEach( - [child.stdout, child.stderr], - (output) => - output.pipe( - Stream.decodeText, - Stream.runForEach((text) => - fs.writeFileString(path.join(directory, log), text, { flag: "a", mode: 0o600 }), - ), - Effect.forkScoped, - ), - { discard: true }, + yield* child.stderr.pipe( + Stream.decodeText, + Stream.runForEach((text) => + fs.writeFileString(path.join(directory, log), text, { flag: "a", mode: 0o600 }), + ), + Effect.forkScoped, ); const ready = yield* Deferred.make(); - yield* child.getOutputFd(3).pipe( + yield* child.stdout.pipe( Stream.decodeText, Stream.splitLines, Stream.runForEach((line) => @@ -105,7 +86,17 @@ const serveMotel = ( ), Effect.forkScoped, ); - return yield* Deferred.await(ready).pipe(Effect.timeout("20 seconds")); + // The listening line, or the collector's exit if it could not start: it says why in its log. + return yield* Deferred.await(ready).pipe( + Effect.raceFirst( + child.exitCode.pipe( + Effect.flatMap((code) => + Effect.die(`The collector exited ${code} before listening; see ${log}.`), + ), + ), + ), + Effect.timeout("20 seconds"), + ); }); /** Name the collector's query URL in `data/diagnostics/collector.json`, where readers find it. */ @@ -120,13 +111,10 @@ const announce = (directory: string, url: string) => ); }); -/** One Motel, as a scenario that owns its own collector uses it, with any `bounds` it sets. */ -export const serveOtlpCollector = ( - directory: string, - bounds: Readonly> = {}, -) => +/** One collector, as a scenario that owns its own collector uses it, with any `bounds` it sets. */ +export const serveOtlpCollector = (directory: string, bounds: CollectorBounds = {}) => Effect.gen(function* () { - const url = yield* serveMotel(directory, "motel", "collector.log", bounds); + const url = yield* serveCollector(directory, "collector", "collector.log", bounds); yield* announce(directory, url); return url; }); @@ -146,10 +134,15 @@ export const ownCollectorSpans = ( Effect.gen(function* () { const url = new URL("/api/spans/search", collector); if (window.operation !== undefined) url.searchParams.set("operation", window.operation); - url.searchParams.set("lookback", "1h"); - url.searchParams.set("limit", "10000"); + // The collector looks back whole minutes from now: as few as cover the window's start. + const minutes = Math.max(1, Math.ceil((Date.now() - window.from) / 60_000) + 1); + url.searchParams.set("lookback", `${minutes}m`); + url.searchParams.set("limit", String(searchLimit)); const response = yield* http.get(url.href); - return yield* response.json.pipe(Effect.flatMap(Schema.decodeUnknownEffect(SpanQuery))); + return yield* response.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(SpanQuery)), + Effect.tap((found) => refuseTruncated(found, url)), + ); }), ).pipe( Effect.map(({ data }) => @@ -203,6 +196,8 @@ const LogExport = Schema.fromJsonString( /** The parts of a query answer the router merges: its items, and the limit that bounds them. */ const Listing = Schema.StructWithRest(Schema.Struct({ data: Schema.Array(Schema.Json) }), [Json]); const SpanItem = Schema.Struct({ span: Schema.Struct({ startTime: Schema.String }) }); +/** A list the collector cut off at its limit. */ +const Truncated = Schema.Struct({ meta: Schema.Struct({ truncated: Schema.Literal(true) }) }); /** * Which collector holds a trace. OTLP JSON names a trace in hex, or in base64 from exporters that @@ -217,26 +212,27 @@ const shardOf = (traceId: string | undefined, shards: number) => { }; /** - * Several Motels behind one OTLP and query endpoint, each holding the traces whose ID it is given. + * Several collectors behind one OTLP and query endpoint, each holding the traces whose ID it is + * given. * - * One Motel stores about 1,500 spans a second: SQLite indexes every attribute of every span on a - * single thread. The managed Cloud job's six workers send more than that while they provision their - * organizations, so one collector refused exports with 429 until the product's three-second export - * budget ran out and dropped them, and scenarios waiting for those spans failed. A deployed stage - * exports to a collector that keeps up. Splitting the traces gives each Motel a share of the load and - * of the data its queries read. + * The managed Cloud job's six workers export together while they provision their organizations. + * Motel, the collector this harness first ran, stored about 1,500 spans a second and refused them; + * so the job split the traces over several. The Rust collector stores about 24,000 spans a second + * on a desktop (its README, "Disk use and the write-ahead log"); whether one would keep up with the + * job has not been measured, so the split and its count are unchanged. * - * Each export is split by trace and sent to the Motels holding those traces. When one refuses its + * Each export is split by trace and sent to the collectors holding those traces. When one refuses its * part, the export is answered with that refusal, so the exporter resends it as it would to one - * Motel; spans are stored by their IDs, so the parts already stored are replaced, not repeated. A - * trace's reads go to its Motel; a search across traces asks every Motel and merges the answers. + * collector; spans are stored by their IDs, so the parts already stored are replaced, not repeated. A + * trace's reads go to its collector; a search across traces asks every collector and merges the + * answers. */ export const serveShardedOtlpCollector = (directory: string, shards: number) => Effect.gen(function* () { const http = yield* HttpClient.HttpClient; - const motels = yield* Effect.forEach( + const collectors = yield* Effect.forEach( Array.from({ length: shards }, (_, index) => index), - (index) => serveMotel(directory, `motel-${index}`, `collector-${index}.log`), + (index) => serveCollector(directory, `collector-${index}`, `collector-${index}.log`), { concurrency: "unbounded" }, ); const answer = (response: HttpClientResponse.HttpClientResponse) => @@ -252,16 +248,18 @@ export const serveShardedOtlpCollector = (directory: string, shards: number) => }), ), ); - const post = (motel: string, path: string, body: HttpBody.HttpBody) => - Effect.scoped(http.post(new URL(path, motel).href, { body }).pipe(Effect.flatMap(answer))); - const get = (motel: string, url: string) => - Effect.scoped(http.get(new URL(url, motel).href).pipe(Effect.flatMap(answer))); + const post = (collector: string, path: string, body: HttpBody.HttpBody) => + Effect.scoped( + http.post(new URL(path, collector).href, { body }).pipe(Effect.flatMap(answer)), + ); + const get = (collector: string, url: string) => + Effect.scoped(http.get(new URL(url, collector).href).pipe(Effect.flatMap(answer))); - /** Send each Motel its part; the first refusal, if any, answers the whole export. */ + /** Send each collector its part; the first refusal, if any, answers the whole export. */ const ingest = (parts: ReadonlyMap, path: string) => Effect.forEach( [...parts], - ([shard, part]) => post(motels[shard]!, path, HttpBody.jsonUnsafe(part)), + ([shard, part]) => post(collectors[shard]!, path, HttpBody.jsonUnsafe(part)), { concurrency: "unbounded" }, ).pipe( Effect.map( @@ -357,7 +355,7 @@ export const serveShardedOtlpCollector = (directory: string, shards: number) => const text = new TextDecoder().decode(bytes); const path = new URL(request.url, "http://collector").pathname; const unchanged = post( - motels[0]!, + collectors[0]!, path, HttpBody.uint8Array(bytes, request.headers["content-type"]), ); @@ -401,25 +399,25 @@ export const serveShardedOtlpCollector = (directory: string, shards: number) => return yield* unchanged; }); - /** A trace's reads go to the Motel holding it. */ + /** A trace's reads go to the collector holding it. */ const trace = Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest; const id = new URL(request.url, "http://collector").pathname.split("/")[3]; - return yield* get(motels[shardOf(id, shards)]!, request.url); + return yield* get(collectors[shardOf(id, shards)]!, request.url); }); - /** A search names one trace, or asks every Motel and returns the newest matches first. */ + /** A search names one trace, or asks every collector and returns the newest matches first. */ const search = Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest; const url = new URL(request.url, "http://collector"); const traceId = url.searchParams.get("traceId"); - if (traceId !== null) return yield* get(motels[shardOf(traceId, shards)]!, request.url); - // Each Motel's matches, or the answer of the first that could not search. + if (traceId !== null) return yield* get(collectors[shardOf(traceId, shards)]!, request.url); + // Each collector's matches, or the answer of the first that could not search. const answers = yield* Effect.forEach( - motels, - (motel) => + collectors, + (collector) => Effect.scoped( http - .get(new URL(request.url, motel).href) + .get(new URL(request.url, collector).href) .pipe( Effect.flatMap( ( @@ -446,15 +444,21 @@ export const serveShardedOtlpCollector = (directory: string, shards: number) => found._tag === "Some" ? found.value.span.startTime : "", ); const limit = Number(url.searchParams.get("limit") ?? Number.POSITIVE_INFINITY); - const data = listings + const matched = listings .flatMap((listing) => listing.data) - .sort((a, b) => start(b).localeCompare(start(a))) - .slice(0, limit); - return HttpServerResponse.jsonUnsafe({ data }); + .sort((a, b) => start(b).localeCompare(start(a))); + const data = matched.slice(0, limit); + // Cut off when any collector had more than it returned, or the merge kept fewer than matched. + const truncated = + matched.length > data.length || + listings.some((listing) => + Schema.decodeUnknownOption(Truncated)(listing).pipe(Option.isSome), + ); + return HttpServerResponse.jsonUnsafe({ data, meta: { truncated, returned: data.length } }); }); - const ingestStatus = Effect.forEach(motels, (motel) => + const ingestStatus = Effect.forEach(collectors, (collector) => Effect.scoped( - http.get(new URL("/api/ingest", motel).href).pipe(Effect.flatMap((r) => r.json)), + http.get(new URL("/api/ingest", collector).href).pipe(Effect.flatMap((r) => r.json)), ), ).pipe(Effect.map((shards) => HttpServerResponse.jsonUnsafe({ shards }))); diff --git a/e2e/support/platform.ts b/e2e/support/platform.ts index 3865f68d49..9278d34731 100644 --- a/e2e/support/platform.ts +++ b/e2e/support/platform.ts @@ -53,9 +53,15 @@ export class Target extends Context.Service< return { metadata, directory, - preparedScenarios: yield* Config.String("E2E_PREPARED_SCENARIOS").pipe( - Config.withDefault("{}"), - Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(PreparedScenarios))), + // The suite writes the Cloud organizations it prepared to the run's directory; a file, + // not an environment variable, since the list grows with the suite past the 128 KiB a + // variable may hold. A run with nothing prepared has none. + preparedScenarios: yield* Schema.decodeUnknownEffect( + Schema.fromJsonString(PreparedScenarios), + )( + (yield* fs.exists(`${directory}/prepared-scenarios.json`)) + ? yield* fs.readFileString(`${directory}/prepared-scenarios.json`) + : "{}", ), apiKey, rows, diff --git a/e2e/support/request-failure.ts b/e2e/support/request-failure.ts index c32e326eb5..e0646d06a9 100644 --- a/e2e/support/request-failure.ts +++ b/e2e/support/request-failure.ts @@ -14,6 +14,7 @@ import { Clock, Duration, Effect, Option, Redacted, Result, Schema } from "effec import type { HttpClientError, HttpClientResponse } from "effect/http"; import { errorCode, nextCause } from "./failure-text.ts"; import { platformSignatures } from "../platform-signatures.ts"; +import { sentFacts } from "./lost-connections.ts"; export const RequestStage = Schema.Literals([ // No response headers arrived: connect, DNS, TLS, a reset before the answer, or a request @@ -60,9 +61,66 @@ export const RequestFailure = Schema.Struct({ elapsedMs: Schema.Number, /** A platform failure this is exactly one of; see `platformSignatures`. */ signature: Schema.optional(Schema.Literals(["runtime-502", "lost"])), + /** + * For a stage request that failed in transport: whether its connection was new or reused, how + * many requests it had carried, how old and how long idle it was, how much of the request was + * written, and whether a response began (`support/lost-connections.ts`). Only a request of + * which nothing was written is sent again: how many attempts failed so, over how long, and why + * the repeats stopped. + */ + sent: Schema.optional( + Schema.Struct({ + connection: Schema.optional(Schema.Literals(["new", "reused"])), + earlierRequests: Schema.optional(Schema.Number), + connectionAgeMs: Schema.optional(Schema.Number), + idleMs: Schema.optional(Schema.Number), + written: Schema.Literals(["nothing", "headers", "whole request"]), + responseStarted: Schema.Boolean, + attempts: Schema.optional(Schema.Number), + waitedMs: Schema.optional(Schema.Number), + stopped: Schema.optional(Schema.String), + }), + ), }); export type RequestFailure = typeof RequestFailure.Type; +/** The facts `lost-connections.ts` put on the first error along a cause chain that carries them. */ +const sentOf = (cause: unknown): RequestFailure["sent"] => { + let current: unknown = cause; + for (let level = 0; level < depth && typeof current === "object" && current !== null; level++) { + const facts = sentFacts(current); + if (facts !== undefined) + return { + ...(facts.connection !== undefined && { connection: facts.connection }), + ...(facts.earlierRequests !== undefined && { earlierRequests: facts.earlierRequests }), + ...(facts.connectionAgeMs !== undefined && { connectionAgeMs: facts.connectionAgeMs }), + ...(facts.idleMs !== undefined && { idleMs: facts.idleMs }), + written: facts.written, + responseStarted: facts.responseStarted, + ...(facts.repeats !== undefined && facts.repeats > 0 && { attempts: facts.repeats }), + ...(facts.waitedMs !== undefined && { waitedMs: facts.waitedMs }), + ...(facts.stopped !== undefined && { stopped: facts.stopped }), + }; + current = nextCause(current); + } + return undefined; +}; + +/** + * `on a new connection 12 ms old, whole request written, no response`, or for a reused one + * `on a reused connection (3 earlier requests, 4.2 s old) idle 0.0 s, …`. + */ +const sentText = (sent: NonNullable) => + [ + sent.connection === undefined + ? `never written to a connection${sent.attempts === undefined || sent.attempts < 2 ? "" : ` on any of ${sent.attempts} attempts over ${((sent.waitedMs ?? 0) / 1000).toFixed(1)} s`}${sent.stopped === undefined ? "" : `, not sent again: ${sent.stopped}`}` + : `on a ${sent.connection} connection${sent.connection === "reused" && sent.earlierRequests !== undefined ? ` (${sent.earlierRequests} earlier requests${sent.connectionAgeMs === undefined ? "" : `, ${(sent.connectionAgeMs / 1000).toFixed(1)} s old`})` : sent.connectionAgeMs === undefined ? "" : ` ${sent.connectionAgeMs} ms old`}${sent.idleMs === undefined ? "" : ` idle ${(sent.idleMs / 1000).toFixed(1)} s`}`, + sent.connection === undefined ? undefined : `${sent.written} written`, + sent.responseStarted ? "a response had begun" : "no response", + ] + .filter((part) => part !== undefined) + .join(", "); + /** * Two ways a request to a stage fails in Cloudflare's platform rather than in our Worker, as seen * on deployed runs. A failure that is exactly one of them says so at the start of its first line, @@ -157,7 +215,7 @@ const connectionCodes: ReadonlyArray = [ ], [ "tls", - /^(?:EPROTO|ERR_TLS_\w+|ERR_SSL_\w+|CERT_\w+|UNABLE_TO_\w+|DEPTH_ZERO_SELF_SIGNED_CERT|SELF_SIGNED_CERT_IN_CHAIN)$/u, + /^(?:EPROTO|ERR_TLS_\w+|ERR_SSL_[\w/]+|CERT_\w+|UNABLE_TO_\w+|DEPTH_ZERO_SELF_SIGNED_CERT|SELF_SIGNED_CERT_IN_CHAIN)$/u, ], ["reset", /^(?:ECONNRESET|EPIPE)$/u], ["closed", /^(?:UND_ERR_SOCKET|UND_ERR_CLOSED|ECONNABORTED|ERR_STREAM_PREMATURE_CLOSE)$/u], @@ -197,6 +255,7 @@ export const requestFailureText = (failure: RequestFailure) => failure.ray === undefined ? undefined : `ray ${failure.ray}${failure.colo === undefined ? "" : `-${failure.colo}`}`, + failure.sent === undefined ? undefined : sentText(failure.sent), ] .filter((part) => part !== undefined) .join(", "); @@ -237,6 +296,8 @@ export const readResponse = ( ...edge, ...chain, elapsedMs: now - started, + ...(stage === "before response" && + sentOf(cause) !== undefined && { sent: sentOf(cause) }), }; return Effect.fail( new CallFailed({ diff --git a/e2e/support/scenario-deadline.ts b/e2e/support/scenario-deadline.ts new file mode 100644 index 0000000000..9a8d8c4879 --- /dev/null +++ b/e2e/support/scenario-deadline.ts @@ -0,0 +1,11 @@ +/** When the running scenario's own time-out ends, for a wait that must report before it does. */ +import { Context } from "effect"; + +/** + * The epoch milliseconds at which Vitest stops the running scenario: when its body started plus + * its time-out. Undefined outside a case (an interactive session has none), where a wait keeps no + * deadline of its own beyond the one its caller gives it. + */ +export const ScenarioDeadline = Context.Reference("e2e/ScenarioDeadline", { + defaultValue: () => undefined, +}); diff --git a/e2e/support/stage-connections.ts b/e2e/support/stage-connections.ts new file mode 100644 index 0000000000..3e386f8f56 --- /dev/null +++ b/e2e/support/stage-connections.ts @@ -0,0 +1,15 @@ +/** + * Installs the stage-request wrapper (`support/lost-connections.ts`) the moment it is imported: + * each request to a stage origin then goes on a connection of its own and a lost one is recorded. + * + * Effect's `FetchHttpClient` reads `globalThis.fetch` once and keeps it for the life of the + * process, so the wrapper has to be in place before anything can make a request: a process that + * installed it later kept Node's keep-alive pool for every request after the first (the deployed + * suite's `/health` check came before the preparation installed it). So every entry point that + * makes outbound calls (run.ts, deployed.ts, cleanup-deployed.ts, cli.ts, create-emulators.ts) and + * Vitest's setup file (setup.ts) import this second, right after `connect-timeout.ts`. Which + * origins are a stage is decided at each request, so installing before they are known is enough. + */ +import { installLostConnections } from "./lost-connections.ts"; + +installLostConnections(); diff --git a/e2e/support/throttling-collector.ts b/e2e/support/throttling-collector.ts index 5bcd93a03a..9d8d741c48 100644 --- a/e2e/support/throttling-collector.ts +++ b/e2e/support/throttling-collector.ts @@ -1,7 +1,7 @@ /** - * A scenario-owned OTLP destination: its own Motel behind a proxy that refuses chosen exports the + * A scenario-owned OTLP destination: its own collector behind a proxy that refuses chosen exports the * way a collector shedding load does. Exports carrying no planned marker, and a marked export once - * its refusals run out, reach Motel unchanged. Nothing is shared with the target's collector. + * its refusals run out, reach the collector unchanged. Nothing is shared with the target's collector. */ import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; import { Clock, Effect, FileSystem, Layer, Ref, Schema } from "effect"; @@ -68,11 +68,11 @@ export const throttlingCollector = Effect.gen(function* () { directory: target.directory, prefix: "collector", }); - const motel = yield* serveOtlpCollector(directory); + const collector = yield* serveOtlpCollector(directory); const plans = yield* Ref.make>>(new Map()); const arrivals = yield* Ref.make>>(new Map()); const appTraces = yield* Ref.make>>(new Map()); - /** Record the traces of app isolate spans that reached Motel, by the app they came from. */ + /** Record the traces of app isolate spans that reached the collector, by the app they came from. */ const recordAppTraces = (text: string) => Ref.update(appTraces, (all) => { const decoded = Schema.decodeUnknownOption(AppTraceExport)(text); @@ -93,7 +93,7 @@ export const throttlingCollector = Effect.gen(function* () { const forward = (path: string, bytes: Uint8Array, contentType: string | undefined) => Effect.scoped( Effect.gen(function* () { - const response = yield* http.post(new URL(path, motel).href, { + const response = yield* http.post(new URL(path, collector).href, { body: HttpBody.uint8Array(bytes, contentType), }); return HttpServerResponse.uint8Array(new Uint8Array(yield* response.arrayBuffer), { @@ -150,15 +150,24 @@ export const throttlingCollector = Effect.gen(function* () { /** Every export containing `marker` that reached the proxy. */ arrivals: (marker: string) => Ref.get(arrivals).pipe(Effect.map((all) => all.get(marker) ?? [])), - /** The traces whose app isolate spans for `app` reached Motel. */ + /** The traces whose app isolate spans for `app` reached the collector. */ appTraces: (app: string) => Ref.get(appTraces).pipe(Effect.map((all) => all.get(app) ?? new Set())), - /** The spans of one trace that Motel stored. */ + /** The spans of one trace that the collector stored, whole: a trace answered in part dies. */ query: (traceId: string) => Effect.scoped( - http.get(new URL(`/api/traces/${traceId}/spans`, motel).href).pipe( + http.get(new URL(`/api/traces/${traceId}/spans`, collector).href).pipe( Effect.flatMap((response) => response.json), Effect.flatMap(Schema.decodeUnknownEffect(SpanQuery)), + Effect.tap((found) => + found.meta?.truncated === true + ? Effect.die( + new Error( + `The collector answered trace ${traceId} in part (${found.meta.returned ?? "?"} of ${found.meta.spanCount ?? "?"} spans); a check cannot read it as the whole trace`, + ), + ) + : Effect.void, + ), ), ), }; diff --git a/e2e/support/webhook-sources.ts b/e2e/support/webhook-sources.ts index fd0c9c8287..fe1a1d608a 100644 --- a/e2e/support/webhook-sources.ts +++ b/e2e/support/webhook-sources.ts @@ -2,6 +2,17 @@ import { expect } from "@effect/vitest"; import { Clock, Deferred, Effect, Schema } from "effect"; import { Api, body, type Session } from "./api.ts"; +import { axiomQueryMs } from "./axiom.ts"; +import { collectorReadMs, Evidence, Telemetry } from "./evidence.ts"; +import { Target } from "./platform.ts"; +import { ScenarioDeadline } from "./scenario-deadline.ts"; +import { + reconcileSpan, + sourcesSpan, + type SweepPace, + sweepSpan, + waitOnSweep, +} from "./webhook-sweep.ts"; /** * One account linked to an app as its webhook source, as the caller may see it. An account with an @@ -47,27 +58,135 @@ export const webhookSubscriptions = (actor: Session, appPath: string) => return yield* body(Schema.Array(WebhookSubscriptionRow), response); }); +/** The app ID in an app path, `/api/organizations//apps/`. */ +const appOf = (appPath: string) => appPath.split("/apps/")[1]?.split("/")[0] ?? ""; + +/** + * How often the waits read, per target. The product is read every 250 ms. Spans come from Axiom + * on a deployed stage: measured on 16 November's test stages, a span was readable a median + * 0.05-0.7 s and at most 18 s after it ended, and every query counts against the organization's + * 1,800 a minute (`x-ratelimit-limit`), shared with all its other readers, so passes are read every + * 10 s there. A local collector receives a span within its exporter's 1 s interval and costs no + * quota. Each `queryMs` is the reader's own time-out, so no span read runs past the report. + */ +export const deployedPace: SweepPace = { + readSpacingMs: 250, + spanSpacingMs: 10_000, + deliveryMs: 20_000, + queryMs: axiomQueryMs, + reportMarginMs: 3_000, + servers: "own", +}; +export const localPace: SweepPace = { + readSpacingMs: 250, + spanSpacingMs: 2_000, + deliveryMs: 5_000, + queryMs: collectorReadMs, + reportMarginMs: 3_000, + servers: "shared", +}; +/** Each Cloud environment has its own collector; see `skippingPass`. */ +export const managedCloudPace: SweepPace = { ...localPace, servers: "own" }; + +/** The pace for a run's target: a deployed stage, a managed Cloud, or a self-hosted or local one. */ +export const sweepPaceFor = (metadata: { + readonly target: string; + readonly mode?: string | undefined; +}): SweepPace => + metadata.target !== "cloud" + ? localPace + : metadata.mode === "attached" + ? deployedPace + : managedCloudPace; + +/** + * Read `attempt` until it is settled while the webhook source sweep can still reach the app's + * pending sources; see `waitOnSweep` in `webhook-sweep.ts`. It passes on the product's read alone; + * the sweep's spans only make it fail sooner, on a pass that skipped a source. For any state the + * sweep's reconciliation of the app's sources brings about, such as a deploy's subscriptions. + */ +export const settleOnSweep = ( + actor: Session, + appPath: string, + what: string, + attempt: Effect.Effect< + { readonly done: boolean; readonly value: A; readonly shown: string }, + E, + R + >, +) => + Effect.gen(function* () { + const evidence = yield* Evidence, + telemetry = yield* Telemetry, + target = yield* Target; + const deadline = yield* ScenarioDeadline; + const app = appOf(appPath); + const minutesSince = (since: number) => + Effect.map(Clock.currentTimeMillis, (now) => (now - since) / 60_000); + const pace = sweepPaceFor(target.metadata); + return yield* waitOnSweep({ + what, + deadline, + pace, + reads: { + attempt, + pending: Effect.gen(function* () { + const sources = yield* webhookSources(actor, appPath); + const read = (yield* evidence.requests).at(-1); + return { + accounts: sources.filter((source) => source.status === "pending"), + shown: sources.map(({ account, label, status, failure }) => ({ + account, + label, + status, + failure, + })), + trace: read?.traceId, + }; + }), + passes: (since) => + minutesSince(since).pipe( + Effect.flatMap((minutes) => telemetry.searchRecent(sweepSpan, {}, minutes)), + Effect.map((found) => found.data), + ), + served: (account, since) => + minutesSince(since).pipe( + Effect.flatMap((minutes) => + telemetry.searchRecent( + reconcileSpan, + { "executor.app.id": app, "executor.account.id": account }, + minutes, + ), + ), + Effect.map((found) => found.data), + ), + readSpan: (trace) => + telemetry + .search(sourcesSpan, { traces: [trace] }) + .pipe(Effect.map((found) => found.data[0]?.span)), + }, + }); + }); + /** * Wait for background registration until `settled` accepts the sources `actor` sees, and return - * them. Fails with the last observed sources after 30 seconds. + * them; see `settleOnSweep` for when it gives up. */ export const settleWebhookSources = ( actor: Session, appPath: string, settled: (sources: readonly WebhookSourceRow[]) => boolean, ) => - Effect.gen(function* () { - const deadline = (yield* Clock.currentTimeMillis) + 30_000; - for (;;) { - const sources = yield* webhookSources(actor, appPath); - if (settled(sources)) return sources; - expect( - yield* Clock.currentTimeMillis, - `Webhook sources did not settle: ${JSON.stringify(sources)}`, - ).toBeLessThan(deadline); - yield* Effect.sleep("100 millis"); - } - }); + settleOnSweep( + actor, + appPath, + "Webhook sources", + Effect.map(webhookSources(actor, appPath), (sources) => ({ + done: settled(sources), + value: sources, + shown: JSON.stringify(sources), + })), + ); /** Exactly these accounts are sources, each in the given status and failure. */ export const sourcesAre = @@ -87,40 +206,43 @@ export const sourcesAre = /** * Wait for background registration until `settled` accepts the subscriptions `actor` sees, and - * return them. Fails with the last observed subscriptions after 30 seconds. + * return them; their source's reconciliation registers them, so it waits on the sweep as + * `settleWebhookSources` does. */ export const settleWebhookSubscriptions = ( actor: Session, appPath: string, settled: (subscriptions: readonly WebhookSubscriptionRow[]) => boolean, ) => - Effect.gen(function* () { - const deadline = (yield* Clock.currentTimeMillis) + 30_000; - for (;;) { - const subscriptions = yield* webhookSubscriptions(actor, appPath); - if (settled(subscriptions)) return subscriptions; - expect( - yield* Clock.currentTimeMillis, - `Webhook subscriptions did not settle: ${JSON.stringify(subscriptions)}`, - ).toBeLessThan(deadline); - yield* Effect.sleep("100 millis"); - } - }); + settleOnSweep( + actor, + appPath, + "Webhook subscriptions", + Effect.map(webhookSubscriptions(actor, appPath), (subscriptions) => ({ + done: settled(subscriptions), + value: subscriptions, + shown: JSON.stringify(subscriptions), + })), + ); /** * Remove the app at `appPath` once its webhook subscriptions have stopped: removal is refused - * while any still runs, and removing sources are not listed. Fails after 30 seconds. + * while any still runs, and the sweep's reconciliation of its sources stops them. Waits on the + * sweep as `settleWebhookSources` does. */ export const removeAppWhenStopped = (actor: Session, appPath: string) => Effect.gen(function* () { const api = yield* Api; - const deadline = (yield* Clock.currentTimeMillis) + 30_000; - for (;;) { - const removed = yield* api.request(actor, "DELETE", appPath); - if (removed.status !== 409) return removed.status; - expect(yield* Clock.currentTimeMillis, JSON.stringify(removed.body)).toBeLessThan(deadline); - yield* Effect.sleep("100 millis"); - } + return yield* settleOnSweep( + actor, + appPath, + "App removal", + Effect.map(api.request(actor, "DELETE", appPath), (removed) => ({ + done: removed.status !== 409, + value: removed.status, + shown: JSON.stringify(removed.body ?? null), + })), + ); }); const isConflict = Schema.is(Schema.Struct({ _tag: Schema.Literal("WebhookConflict") })); diff --git a/e2e/support/webhook-sweep.ts b/e2e/support/webhook-sweep.ts new file mode 100644 index 0000000000..0cd61441a7 --- /dev/null +++ b/e2e/support/webhook-sweep.ts @@ -0,0 +1,295 @@ +/** + * The webhook source sweep as its spans show it, and the wait that reads it. The SDK's `tick` is + * one pass: an `sdk.webhooks.sweep` span that records how many pending sources it took (`taken`) + * and the most it may take (`limit`), with one `sdk.webhooks.reconcileSource` span per source it + * reconciles. Passes run one after another (one worker on a self-hosted server, one coordinator + * on Cloud) and take pending, unleased sources first in, first out, so a pass that started while a + * source was pending and unleased, and took fewer than its limit, took that source. + * + * Spans may only make a wait fail sooner, on a proven skip. A wait never needs them to pass: it + * passes on the product's own read, and a span read that fails or comes late only leaves the wait + * to its bound. + */ +import { Clock, Effect } from "effect"; +import type { SpanQuery } from "./contracts.ts"; + +export const sweepSpan = "sdk.webhooks.sweep"; +export const reconcileSpan = "sdk.webhooks.reconcileSource"; +/** The SDK's read of an app's sources, in the trace of the product request that read them. */ +export const sourcesSpan = "sdk.webhooks.sources"; +/** + * The SDK's webhook source service that ran a pass or a read of an app's sources: one per + * self-hosted server, one per Worker on Cloud. + */ +export const instanceTag = "executor.webhooks.instance"; + +export type Delivered = (typeof SpanQuery.Type)["data"][number]; +export type DeliveredSpan = Delivered["span"]; + +export const spanStart = (span: DeliveredSpan) => Date.parse(span.startTime); +export const spanEnd = (span: DeliveredSpan) => spanStart(span) + span.durationMs; +export const sweepTaken = (span: DeliveredSpan) => + Number(span.tags["executor.webhooks.sweep.taken"] ?? Number.NaN); +export const sweepLimit = (span: DeliveredSpan) => + Number(span.tags["executor.webhooks.sweep.limit"] ?? Number.NaN); +/** A pass that took fewer pending sources than its limit, so it left none behind. */ +export const hasRoom = (span: DeliveredSpan) => sweepTaken(span) < sweepLimit(span); + +const byStart = (a: Delivered, b: Delivered) => spanStart(a.span) - spanStart(b.span); + +/** A product read that showed the source pending, on the server's clock. */ +export interface PendingRead { + readonly start: number; + readonly end: number; + /** The webhook source service that answered it (`executor.webhooks.instance`). */ + readonly instance: string | undefined; +} + +/** + * The first pass that skipped the source, from the source's own history: a pass of the read's own + * server that had room, reconciled nothing of the source's, and started after a read that showed + * the source pending, with no reconciliation + * of the source between that read and the pass. A reconciliation that ends after the read began + * may have served the source and left it ready, so the source could have been queued again only + * after the pass started; that read then proves nothing, and a later one is needed. + * + * `servedReadAt` is when the source's reconciliations were read: a pass is judged only once + * `deliveryMs` has passed since it ended, so every reconciliation before it has been delivered. + * + * Limit: a source whose reconciliation was cut off, its server killed mid-reconcile, keeps its + * lease (120 s, set in the SDK's `reconcile`) and still reads pending; a pass with room cannot + * take it until the lease ends, and this judges that pass a skip. No scenario settles such a + * source today. One that does must wait out the lease before using these waits, or this must + * first learn the lease from the product. + * + * `servers: "shared"` is for a collector other servers may feed, as several self-hosted servers in + * one job do: a pass counts only if the service that ran it answered the read. Each Cloud + * environment has its own collector, and a deployed stage's spans are read by its stage, so there + * (`"own"`) every pass is of the read's own coordinator, though another Worker answered the read. + */ +export const skippingPass = (input: { + readonly passes: ReadonlyArray; + readonly served: ReadonlyArray; + readonly reads: ReadonlyArray; + readonly servedReadAt: number; + readonly deliveryMs: number; + readonly servers: "own" | "shared"; +}) => + input.passes + .filter( + ({ span, traceId }) => + hasRoom(span) && + spanEnd(span) + input.deliveryMs <= input.servedReadAt && + !input.served.some((reconcile) => reconcile.traceId === traceId), + ) + .toSorted(byStart) + .find((pass) => { + const start = spanStart(pass.span); + const read = input.reads + .filter( + (read) => + read.end < start && + (input.servers === "own" || read.instance === pass.span.tags[instanceTag]), + ) + .toSorted((a, b) => b.end - a.end)[0]; + return ( + read !== undefined && + !input.served.some(({ span }) => spanEnd(span) > read.start && spanStart(span) < start) + ); + }); + +/** How the wait reads: the product every `readSpacingMs`, spans at most every `spanSpacingMs`. */ +export interface SweepPace { + readonly readSpacingMs: number; + readonly spanSpacingMs: number; + /** How long after a span ends it is surely readable. */ + readonly deliveryMs: number; + /** The longest one span read may take (its own time-out); none starts closer to the bound. */ + readonly queryMs: number; + /** Time kept before the scenario's time-out to report, from what was already read. */ + readonly reportMarginMs: number; + /** Whether other servers may feed the same collector; see `skippingPass`. */ + readonly servers: "own" | "shared"; +} + +/** What the wait reads. Product reads fail the wait as they would any scenario; span reads never. */ +export interface SweepReads { + /** The scenario's condition, read from the product. */ + readonly attempt: Effect.Effect< + { readonly done: boolean; readonly value: A; readonly shown: string }, + E, + R + >; + /** The app's pending sources, from the product, and the trace of that read. */ + readonly pending: Effect.Effect< + { + readonly accounts: ReadonlyArray<{ readonly account: string; readonly label: string }>; + readonly shown: unknown; + readonly trace: string | undefined; + }, + EP, + RP + >; + /** Passes that started since a time (epoch milliseconds). */ + readonly passes: (since: number) => Effect.Effect, unknown, RS>; + /** One source's reconciliations since a time. */ + readonly served: ( + account: string, + since: number, + ) => Effect.Effect, unknown, RS>; + /** The span of the product's read of the app's sources in one trace, once delivered. */ + readonly readSpan: (trace: string) => Effect.Effect; +} + +interface Watched { + readonly label: string; + /** Test-clock time the source was first seen pending in this stretch. */ + readonly since: number; + /** Reads that showed it pending, newest last, and their server times once read. */ + readonly reads: Array<{ readonly trace: string; readonly at: number; server?: PendingRead }>; +} + +/** + * Read `reads.attempt` until it is settled. A pending source behind full passes is queued, not + * lost, and is waited for until `reportMarginMs` before `deadline`, which then fails with what was + * read. A pass that skipped it (`skippingPass`) fails the wait at once, naming the pass. + */ +export const waitOnSweep = (input: { + readonly what: string; + readonly reads: SweepReads; + readonly deadline: number | undefined; + readonly pace: SweepPace; +}) => + Effect.gen(function* () { + const { reads, pace, what } = input; + const watched = new Map(); + let passes: ReadonlyArray = []; + let passReads = 0; + let failedReads = 0; + let spansReadAt = Number.NEGATIVE_INFINITY; + /** A span read: a failure is counted and leaves the wait to the product's reads. */ + /** + * A span read, made only while it can finish before the report is due (its own time-out, + * `queryMs`, before the bound less `reportMarginMs`). A failure is counted and leaves the wait + * to the product's reads. + */ + const tolerant = (read: Effect.Effect) => + Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + if ( + input.deadline !== undefined && + now + pace.queryMs >= input.deadline - pace.reportMarginMs + ) + return undefined; + return yield* read.pipe( + Effect.map((value): T | undefined => value), + Effect.catchCause(() => + Effect.sync(() => { + failedReads += 1; + return undefined; + }), + ), + ); + }); + const report = (shown: string, sources: unknown) => + JSON.stringify({ + [what]: JSON.parse(shown) as unknown, + sources, + pending: [...watched].map(([account, entry]) => { + const after = passes.filter(({ span }) => spanStart(span) > entry.since); + return { + account, + label: entry.label, + passesSince: after.length, + withRoom: after.filter(({ span }) => hasRoom(span)).length, + }; + }), + ...(passReads === 0 && failedReads > 0 + ? { passes: `could not be read (${failedReads} failed reads)` } + : failedReads > 0 + ? { failedReads } + : {}), + }); + for (;;) { + const current = yield* reads.attempt; + if (current.done) return current.value; + const pending = yield* reads.pending; + const now = yield* Clock.currentTimeMillis; + for (const account of watched.keys()) + if (!pending.accounts.some((source) => source.account === account)) watched.delete(account); + for (const { account, label } of pending.accounts) { + const entry = watched.get(account) ?? { label, since: now, reads: [] }; + watched.set(account, entry); + if (pending.trace !== undefined) entry.reads.push({ trace: pending.trace, at: now }); + // A skip is judged from the reads just before a pass; older ones are never needed. + if (entry.reads.length > 40) entry.reads.splice(0, entry.reads.length - 40); + } + if (watched.size > 0 && now - spansReadAt >= pace.spanSpacingMs) { + spansReadAt = now; + const earliest = Math.min(...[...watched.values()].map((entry) => entry.since)); + const read = yield* tolerant(reads.passes(earliest - 60_000)); + if (read !== undefined) { + passes = read; + passReads += 1; + } + for (const [account, entry] of watched) { + // Only a pass with room after the source was first seen can have skipped it. + const candidate = passes + .filter(({ span }) => hasRoom(span) && spanStart(span) > entry.since - 60_000) + .toSorted(byStart)[0]; + if (candidate === undefined) continue; + const servedReadAt = yield* Clock.currentTimeMillis; + const served = yield* tolerant(reads.served(account, entry.since - 60_000)); + if (served === undefined) continue; + // The server times of the two reads made last before each candidate pass (by this + // process's clock), each read once. + const before = new Set( + passes + .filter( + ({ span, traceId }) => + hasRoom(span) && + spanStart(span) > entry.since - 60_000 && + !served.some((reconcile) => reconcile.traceId === traceId), + ) + .flatMap(({ span }) => + entry.reads.filter((read) => read.at < spanStart(span)).slice(-2), + ), + ); + for (const read of [...before].slice(-4)) + if (read.server === undefined) { + const span = yield* tolerant(reads.readSpan(read.trace)); + if (span !== undefined) + read.server = { + start: spanStart(span), + end: spanEnd(span), + instance: span.tags[instanceTag], + }; + } + const skipped = skippingPass({ + passes, + served, + reads: entry.reads.flatMap((read) => (read.server === undefined ? [] : [read.server])), + servedReadAt, + deliveryMs: pace.deliveryMs, + servers: pace.servers, + }); + if (skipped !== undefined) + return yield* Effect.die( + new Error( + `${what} did not settle: the webhook source sweep skipped ${entry.label}: its pass at ${skipped.span.startTime} (trace ${skipped.traceId}) took ${sweepTaken(skipped.span)} of its ${sweepLimit(skipped.span)} after a read showed the source pending, reconciled nothing of it, and it is still pending: ${report(current.shown, pending.shown)}`, + ), + ); + } + } + if ( + input.deadline !== undefined && + (yield* Clock.currentTimeMillis) >= input.deadline - pace.reportMarginMs + ) + return yield* Effect.die( + new Error( + `${what} did not settle before the scenario's time-out: ${report(current.shown, pending.shown)}`, + ), + ); + yield* Effect.sleep(pace.readSpacingMs); + } + }); diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index 2a52f83d96..22bcfbc9fa 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -114,6 +114,28 @@ const plan = (scenarios: { readonly [_ in Name]: typeof Tes scenarios; /** Scenario names and applicability used by both test declarations and test selection. */ +/** + * The engine call faults the workflow engine calls scenario's servers run with + * (`EXECUTOR_TEST_WORKFLOW_ENGINE_FAULTS`, `engine-faults.ts`): each sleep of `napOnce` fails once, + * every call of `nap-never` fails, `count-once`'s write runs before its answer fails once, + * `waited-out` fails only after the engine waited, four times, the three `*-once` sleeps get the + * engine's own answer once, and the three `*-until` sleepUntils get an answer once after their time. Each names a step only that scenario's app uses, so managed Cloud, + * whose one Worker serves every scenario, can carry the list for all of them. + */ +export const engineCallFaults = [ + { call: "sleep", step: "nap-once", failures: 1, when: "before", answer: "internal" }, + { call: "until", step: "wake-once", failures: 1, when: "before", answer: "internal" }, + { call: "until", step: "nap-never", failures: 100, when: "before", answer: "internal" }, + { call: "mutation", step: "count-once", failures: 1, when: "after", answer: "internal" }, + { call: "sleep", step: "waited-out", failures: 4, when: "after", answer: "internal" }, + { call: "sleep", step: "paused-once", failures: 1, when: "before", answer: "paused" }, + { call: "sleep", step: "terminated-once", failures: 1, when: "before", answer: "terminated" }, + { call: "sleep", step: "relayed-once", failures: 1, when: "before", answer: "relayed" }, + { call: "until", step: "paused-until", failures: 1, when: "after", answer: "paused" }, + { call: "until", step: "terminated-until", failures: 1, when: "after", answer: "terminated" }, + { call: "until", step: "transient-until", failures: 1, when: "after", answer: "internal" }, +] as const; + export const scenarios = plan({ syncroRest: { fixtures: "actors", @@ -1273,7 +1295,7 @@ export const scenarios = plan({ mcpMediaContent: { fixtures: "actors", file: "mcp-direct-calls.spec.ts", - title: "MCP call and execute return app images and audio as MCP content blocks", + title: "MCP call and execute return app images, audio and files as MCP content blocks", targets: { "self-host": scheduled, cloud: na("Shared MCP path; covered on self-host"), @@ -1395,6 +1417,17 @@ export const scenarios = plan({ local: na("Hosted version API scenario"), }, }, + liveOpenapiHujson: { + fixtures: "actors", + file: "live-openapi-import.spec.ts", + title: + "Live OpenAPI sends and returns HuJSON as written text, while JSON, every +json type and AWS JSON stay JSON", + targets: { + "self-host": scheduled, + cloud: na("Loopback upstream fixture"), + local: na("Hosted version API scenario"), + }, + }, liveOpenapiNaming: { fixtures: "actors", file: "live-openapi-import.spec.ts", @@ -1483,7 +1516,9 @@ export const scenarios = plan({ title: "App cache failures during evaluation keep their safe reason in responses and traces", targets: { "self-host": scheduled, - cloud: na("Uses the shared runtime contract; self-host delivers its traces to Motel."), + cloud: na( + "Uses the shared runtime contract; self-host delivers its traces to its bundled collector.", + ), local: na("Hosted API fixture; the runtime and collector are shared with Local."), }, }, @@ -2827,7 +2862,7 @@ export const scenarios = plan({ title: "SDK batches invocation accounts and finished workflow history", targets: { "self-host": scheduled, - cloud: na("This query budget reads the self-host Motel collector."), + cloud: na("This query budget reads the self-host collector."), local: na("This scenario uses hosted organization routes."), }, }, @@ -2838,7 +2873,7 @@ export const scenarios = plan({ "Tool invocations resolve one committed set of linked accounts without holding a transaction", targets: { "self-host": scheduled, - cloud: na("This scenario reads SQL spans from the self-host Motel collector."), + cloud: na("This scenario reads SQL spans from the self-host collector."), local: na("This scenario uses hosted organization routes."), }, }, @@ -3374,6 +3409,59 @@ export const scenarios = plan({ local: na("ChatGPT app domain verification is for the hosted Cloud domain."), }, }, + appDependencyTarballPrivateRefused: { + fixtures: "actors", + file: "app-dependency-network.spec.ts", + title: + "with private fetch off, a build refuses a dependency whose tarball URL is a private address and never requests it", + serverEnvironment: { EXECUTOR_APPS_ALLOW_PRIVATE_FETCH: "false" }, + targets: { + "self-host": scheduled, + cloud: na( + "Cloud Workers reach only public addresses; the tarball fixture is served on loopback.", + ), + local: na("Local lets builds reach private addresses by design."), + }, + }, + appDependencyTarballPublic: { + fixtures: "actors", + file: "app-dependency-network.spec.ts", + title: + "with private fetch off, a build installs a dependency whose tarball URL is a public address", + serverEnvironment: { EXECUTOR_APPS_ALLOW_PRIVATE_FETCH: "false" }, + targets: { + "self-host": scheduled, + cloud: na("Cloud builds use the same installer; only self-host restricts the build network."), + local: na("Local lets builds reach every address by design."), + }, + }, + appDependencyRegistryOrigin: { + file: "app-dependency-network.spec.ts", + title: + "with private fetch off, a build installs from a private registry's own origin, refuses what it sends to other private addresses, and words failures without its text", + targets: { + "self-host": scheduled, + cloud: na("Cloud Workers reach only public addresses; the registry fixture is on loopback."), + local: na( + "Local lets builds reach every address by design; only self-host starts a scenario-owned server.", + ), + }, + }, + appDependencyTarballPrivateAllowed: { + fixtures: "actors", + file: "app-dependency-network.spec.ts", + title: + "with private fetch on, a build installs tarball URLs on a private address, and a failed download names only the package and status", + targets: { + "self-host": scheduled, + cloud: na( + "Cloud Workers reach only public addresses; the tarball fixture is served on loopback.", + ), + local: na( + "This scenario uses hosted routes; Local shares the same workerd compiler and allows private addresses.", + ), + }, + }, appPackage: { fixtures: "actors", file: "app-package.spec.ts", @@ -4209,7 +4297,7 @@ export const scenarios = plan({ fixtures: "actors", file: "credential-placement.spec.ts", title: - "Executor for Google is offered to providers on Google's endpoints whose scopes it allows, once its credentials are set", + "Executor for Google is offered to providers on Google's endpoints whose scopes it allows, once its credentials are set, and a self-host operator's own Google client still signs in gmail.modify", targets: { "self-host": scheduled, cloud: na( @@ -4308,6 +4396,19 @@ export const scenarios = plan({ local: na("Local products have no operator OAuth clients."), }, }, + credentialManagedHeld: { + fixtures: "actors", + file: "credential-placement.spec.ts", + title: + "the operator's OAuth client keeps renewing grants that hold a scope it may no longer ask for, refuses sign-ins and reconnects that ask for it before the service, and a reconnect on the user's own client keeps that client", + targets: { + "self-host": scheduled, + cloud: na( + "Restarts a runner-owned product with the operator's client; managed Cloud has no per-case operator settings.", + ), + local: na("Local products have no operator OAuth clients."), + }, + }, credentialManagedScopes: { fixtures: "actors", file: "credential-placement.spec.ts", @@ -4334,6 +4435,19 @@ export const scenarios = plan({ local: na("Local products have no operator OAuth clients."), }, }, + credentialManagedGoogleCloud: { + fixtures: "actors", + file: "credential-placement.spec.ts", + title: + "Cloud's own Executor for Google registration refuses before Google a sign-in asking for a scope Google has not approved for it, such as gmail.modify, leading with the user's own client, and still signs in approved scopes", + targets: { + "self-host": na( + "Self-host's operator registers its own Google client, which keeps gmail.modify; the self-host Google scenario covers that.", + ), + cloud: managedCloud, + local: na("Local products have no operator OAuth clients."), + }, + }, credentialManagedCloud: { fixtures: "actors", file: "credential-placement.spec.ts", @@ -4727,6 +4841,57 @@ export const scenarios = plan({ local: na("Hosted deploy routes; local shares the workerd build path."), }, }, + compiledFileLimit: { + fixtures: "actors", + file: "compiled-file-limit.spec.ts", + title: + "Builds refuse an app's own JavaScript or TypeScript file over 2 MB and create no app; a file just under it, a larger JSON import and a package with a larger file build", + targets: { + "self-host": scheduled, + cloud: na( + "Cloud's compiler Worker runs the same compileWorkerApp, and Cloudflare's CPU limit already bounds its compile; self-host covers the shared build.", + ), + local: na("Hosted deploy routes; localCompiledFileLimit covers local through the CLI."), + }, + }, + localCompiledFileLimitImports: { + file: "local-compiled-file-limit.spec.ts", + title: "local refuses a file over the compile limit before resolving any import in it", + targets: { + local: scheduled, + "self-host": na( + "Needs a product started against a registry that holds requests; the shared build is the same.", + ), + cloud: na( + "Needs a product started against a registry that holds requests; the shared build is the same.", + ), + }, + }, + compiledFileLimitResponsive: { + fixtures: "actors", + file: "compiled-file-limit.spec.ts", + title: + "Apps keep answering while a build refuses a 15 MB file: no longer than they wait for one file just under the limit to compile", + targets: { + "self-host": scheduled, + cloud: na( + "Cloud's compiler Worker runs the same compileWorkerApp in its own isolate, so it never holds an app's thread.", + ), + local: na( + "Self-host covers the shared build; local runs it in the same kind of app runtime.", + ), + }, + }, + localCompiledFileLimit: { + file: "local-compiled-file-limit.spec.ts", + title: + "apps CLI deploy reports a file over the compile limit with its path, size and the limit", + targets: { + local: scheduled, + "self-host": na("The CLI's hosted path needs a browser OAuth login; local uses an API key."), + cloud: na("The CLI's hosted path needs a browser OAuth login; local uses an API key."), + }, + }, appOperationFailureDetails: { fixtures: "actors", managementApps: ["owner"], @@ -6567,6 +6732,18 @@ export const scenarios = plan({ local: na("This scenario uses hosted app domains."), }, }, + appUiLongSrcset: { + fixtures: "actors", + file: "app-ui-srcset.spec.ts", + appOrigin: true, + title: + "an app page whose srcset holds a long run of commas is served as fast as a page of its size and rewritten", + targets: { + "self-host": scheduled, + cloud: na("Cloud serves app pages through the same hosted page code that self-host runs."), + local: na("This scenario uses hosted sign-in; Local rewrites pages with the same code."), + }, + }, appUi: { fixtures: "actors", file: "app-ui.spec.ts", @@ -6867,6 +7044,19 @@ export const scenarios = plan({ local: na("Local builds and runs apps through the same workerd compiler as self-host."), }, }, + largeOpenapiAccountsApp: { + fixtures: "actors", + file: "large-openapi-app.spec.ts", + title: + "an OpenAPI app whose resources expand into one another, linked through several accounts, lists each tool once with one branch for every account and its schemas complete", + targets: { + "self-host": scheduled, + // Each target bounds the JSON a cold listing writes, which merging every tool's definitions + // exceeded; the attached run also lists cold within a deployed app Worker's CPU time. + cloud: { status: "scheduled", runtime: "attached" }, + local: na("Local builds and runs apps through the same workerd compiler as self-host."), + }, + }, localExpandableOpenapiTools: { file: "local-openapi-tools.spec.ts", title: @@ -7909,10 +8099,21 @@ export const scenarios = plan({ title: "MCP and API bearer authentication reads its grant in one SQL statement", targets: { "self-host": scheduled, - cloud: na("This scenario reads SQL spans from the self-host Motel collector."), + cloud: na("This scenario reads SQL spans from the self-host collector."), local: na("Local uses its instance credential."), }, }, + searchStatements: { + fixtures: "actors", + file: "search-statements.spec.ts", + title: + "MCP search reads every app's discovery state in statements that do not grow with the apps", + targets: { + "self-host": scheduled, + cloud: na("This scenario reads SQL spans from the self-host collector."), + local: na("Local has no hosted organization policies."), + }, + }, toolCallStatements: { fixtures: "actors", file: "tool-call-statements.spec.ts", @@ -7920,7 +8121,7 @@ export const scenarios = plan({ "An MCP tool call checks its access and links in one statement and its caller in one more", targets: { "self-host": scheduled, - cloud: na("This scenario reads SQL spans from the self-host Motel collector."), + cloud: na("This scenario reads SQL spans from the self-host collector."), local: na("Local has no hosted organization policies."), }, }, @@ -8078,11 +8279,28 @@ export const scenarios = plan({ targets: { "self-host": scheduled, cloud: na( - "Reads SQL spans from the self-host Motel collector; cloud-tool-call-batch.spec.ts covers Cloud's batch.", + "Reads SQL spans from the self-host collector; cloud-tool-call-batch.spec.ts covers Cloud's batch.", ), local: na("Local has no hosted organization policies."), }, }, + cloudMcpConcurrentRecheck: { + fixtures: "actors", + file: "cloud-concurrent-recheck.spec.ts", + title: + "Cloud MCP tool calls made at once with a key whose last use is stale all answer, also the calls that read the key again", + targets: { + // Not run until its contention is certain on a shared database: its round releases the key + // row's lock once two key checks wait on it, and other scenarios' sessions queued on the + // `apikey` table behind the hold were counted as waiters (#4420). The production hang + // count and the `commit-unanswered` and `commit-rollback` pool probe cases cover the fix. + cloud: na( + "Its round's contention is not yet certain on a database other scenarios share; see #4420.", + ), + "self-host": na("Self-host runs a batch under PGlite's lock, not on a multiplexed pool."), + local: na("Local MCP sessions use the local database."), + }, + }, cloudExportHang: { fixtures: "actors", file: "cloud-export-hang.spec.ts", @@ -8640,6 +8858,49 @@ export const scenarios = plan({ local: na("Local uses its instance key."), }, }, + webhookSourceOrder: { + file: "webhook-source-order.spec.ts", + title: + "Pending webhook sources are served first in, first out, and the lease lets one sweep serve a source and loses no link made meanwhile", + targets: { + "self-host": scheduled, + local: na( + "The probe runs webhook-source reconciliation on PGlite in memory; one target covers it.", + ), + cloud: na( + "The probe runs webhook-source reconciliation on PGlite in memory; one target covers it.", + ), + }, + }, + runtimeConnectRepeat: { + file: "runtime-connect-repeat.spec.ts", + title: + "An exchange with the local app runtime that never connected is repeated once, and nothing that may have reached it is", + targets: { + "self-host": scheduled, + local: na("The probe runs the runtime's exchanges against stubs; one target covers it."), + cloud: na("Cloud's apps do not run in a local workerd."), + }, + }, + buildRetainStorage: { + file: "build-retain-storage.spec.ts", + title: + "A build whose framework write storage refused answers try again, concurrent builds share one write, and an app that does not compile still fails", + targets: { + "self-host": scheduled, + local: na("The probe runs the deploy path against a stand-in store; one target covers it."), + cloud: na("The probe runs the SDK's deploy path and Cloud's write retry in one process."), + }, + }, + runtimeHopShape: { + file: "runtime-hop-shape.spec.ts", + title: "A failed exchange with the local app runtime records how far it got, as shapes only", + targets: { + "self-host": scheduled, + local: na("The probe runs the runtime hops' recording against stubs; one target covers it."), + cloud: na("Cloud's apps do not run in a local workerd."), + }, + }, apiKeyStorageOutage: { fixtures: "actors", file: "api-key-storage-outage.spec.ts", @@ -8714,6 +8975,18 @@ export const scenarios = plan({ local: na("Local has no dashboard sessions or organization memberships."), }, }, + workflowEngineCalls: { + fixtures: "actors", + file: "workflow-engine-calls.spec.ts", + title: + "a run's sleep whose engine call failed is called again, a run whose sleep keeps failing records why for its owner, and neither an app's own failure nor a step whose answer was lost is repeated", + serverEnvironment: { EXECUTOR_TEST_WORKFLOW_ENGINE_FAULTS: JSON.stringify(engineCallFaults) }, + targets: { + "self-host": scheduled, + cloud: managedCloud, + local: na("Self-host covers the Node-hosted runtime that local shares."), + }, + }, localWorkflows: { file: "local-workflows.spec.ts", title: "local app workflows execute through the authenticated SDK HTTP surface", @@ -9069,6 +9342,17 @@ export const scenarios = plan({ cloud: scheduled, }, }, + hostedAppAccountListedBrowser: { + fixtures: "actors", + file: "app-accounts-tab.spec.ts", + title: + "an account linked for everyone is named on the app's card for a member who linked none, and its owner's account row lists the app", + targets: { + local: na("Local has one subject; its overview's app links are read in its own scenario."), + "self-host": scheduled, + cloud: scheduled, + }, + }, hostedAppSchedulesBrowser: { fixtures: "actors", file: "app-accounts-tab.spec.ts", @@ -10172,6 +10456,30 @@ export const scenarios = plan({ ), }, }, + telemetryViewer: { + file: "telemetry-viewer.spec.ts", + title: + "The local collector serves the trace viewer: its trace list, a trace, a reload, an unknown trace and the logs filters", + targets: { + local: scheduled, + "self-host": na( + "The image's collector has no address a browser can reach, so it serves no viewer.", + ), + cloud: na("Cloud exports to its own collector, not the bundled one."), + }, + }, + telemetryPartialTrace: { + file: "telemetry-partial-trace.spec.ts", + title: + "A trace the collector answers in part fails a check that reads it and stays marked in evidence", + targets: { + "self-host": scheduled, + local: na( + "The suite's span reader is the same on every target; the self-host run covers it.", + ), + cloud: na("Attached Cloud reads Axiom, which answers no trace in part."), + }, + }, telemetryStoreIdleAtBound: { file: "telemetry-store-bound.spec.ts", title: @@ -11622,10 +11930,10 @@ export const scenarios = plan({ telemetry: { fixtures: "actors", file: "hosted.spec.ts", - title: "real requests reach Motel with correlated server spans", + title: "real requests reach the collector with correlated server spans", targets: { "self-host": scheduled, - cloud: na("Cloud exports to Axiom; this test checks the self-host Motel collector."), + cloud: na("Cloud exports to Axiom; this test checks the self-host collector."), local: na("This test checks hosted organization routes and self-host service spans."), }, }, diff --git a/e2e/test-stage-branch.config.ts b/e2e/test-stage-branch.config.ts new file mode 100644 index 0000000000..17b9f91302 --- /dev/null +++ b/e2e/test-stage-branch.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from "vitest/config"; + +/** Runs the test stage's branch removal against a local stand-in for PlanetScale's API. */ +export default defineConfig({ + test: { include: ["e2e/tests/test-stage-branch.spec.ts"], testTimeout: 90_000 }, +}); diff --git a/e2e/tests/app-account-webhooks.spec.ts b/e2e/tests/app-account-webhooks.spec.ts index 862b038ebc..c2544477e9 100644 --- a/e2e/tests/app-account-webhooks.spec.ts +++ b/e2e/tests/app-account-webhooks.spec.ts @@ -18,6 +18,7 @@ import { } from "../support/app-accounts.ts"; import { appLinksFixture, keyProvider, ok, okBody, refused } from "../support/app-links.ts"; import { + settleOnSweep, settleWebhookSources, settleWebhookSubscriptions, type WebhookSubscriptionRow, @@ -165,27 +166,23 @@ const carriedWebhook = (fixture: Effect.Success - Effect.gen(function* () { - for (;;) { - const rows = yield* subscriptions; - if ( + settleOnSweep( + actors.owner, + path, + `Subscriptions ${status ?? "pinned"} on ${version}`, + Effect.map(subscriptions, (rows) => ({ + done: rows.some( (row) => row.version === version && (status === undefined || row.status === status), - ) && - rows.every((row) => row.status === "disabled" || row.version === version) - ) - return rows; - yield* Effect.sleep("100 millis"); - } - }).pipe( - Effect.timeoutOrElse({ - duration: "30 seconds", - orElse: () => - Effect.die(new Error(`No subscription is ${status ?? "pinned"} on ${version}`)), - }), + ) && rows.every((row) => row.status === "disabled" || row.version === version), + value: rows, + shown: JSON.stringify(rows), + })), ); /** A provider's delivery to a callback URL, signed with `secret`. */ const post = (callbackUrl: string, secret: string) => diff --git a/e2e/tests/app-accounts-tab.spec.ts b/e2e/tests/app-accounts-tab.spec.ts index e788d436f4..08749903f4 100644 --- a/e2e/tests/app-accounts-tab.spec.ts +++ b/e2e/tests/app-accounts-tab.spec.ts @@ -403,6 +403,71 @@ layer(HostedLive, { excludeTestServices: true })("Hosted Accounts tab", (it) => ), ); + it.effect(scenarios.hostedAppAccountListedBrowser.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const browser = yield* Browser, + evidence = yield* Evidence; + const fixture = yield* sharedApp("Team report", source); + const { actors, app } = fixture; + const label = `Team Datadog ${randomUUID().slice(0, 8)}`; + const team = yield* fixture.connect("datadog", label); + expect(yield* fixture.links()).toEqual([["datadog", team]]); + const card = (page: Page) => + page.getByRole("link", { name: `Open ${app.name}`, exact: true }); + + // The member links no account of their own; the app's card names the owner's account + // linked for everyone instead of saying none is linked. + yield* browser.login(actors.member); + yield* browser.use("Open the member's apps", (page) => + page.goto(`/org/${actors.organization.slug}/apps`), + ); + yield* browser.use("The card names the app's account", (page) => + card(page).getByText(label, { exact: true }).waitFor(), + ); + expect( + yield* browser.use("The card says no account is missing", (page) => + card(page) + .getByText(/^(No accounts linked|Needs account)$/) + .count(), + ), + ).toBe(0); + yield* browser.checkpoint("Member's app card with the app's account"); + yield* evidence.attach( + "member-app-card.png", + "image/png", + yield* browser.use("Capture the member's apps", (page) => + page.screenshot({ fullPage: true }), + ), + ); + + // The owner's account row lists the app it is linked to for everyone. + yield* browser.login(actors.owner); + yield* browser.use("Open the owner's accounts", (page) => + page.goto(`/org/${actors.organization.slug}/accounts`), + ); + const row = (page: Page) => page.locator(".inventory-row").filter({ hasText: label }); + yield* browser.use("The account's row links to the app", (page) => + row(page).getByRole("link", { name: app.name }).waitFor(), + ); + expect( + yield* browser.use("The row does not say No apps", (page) => + row(page).getByText("No apps", { exact: true }).count(), + ), + ).toBe(0); + yield* browser.checkpoint("Owner's account row with the app"); + yield* evidence.attach( + "owner-account-row.png", + "image/png", + yield* browser.use("Capture the owner's accounts", (page) => + page.screenshot({ fullPage: true }), + ), + ); + }), + ), + ); + it.effect(scenarios.hostedAppSchedulesBrowser.title, (context) => withHostedCase( context, diff --git a/e2e/tests/app-dependency-network.spec.ts b/e2e/tests/app-dependency-network.spec.ts new file mode 100644 index 0000000000..704d74bda3 --- /dev/null +++ b/e2e/tests/app-dependency-network.spec.ts @@ -0,0 +1,232 @@ +/** + * An app build reaches its registry's own origin over the host's network, and everything else, a + * dependency declared by tarball URL or a tarball or redirect the registry sends elsewhere, over + * the network app code uses: on self-host, public addresses only unless the operator sets + * `EXECUTOR_APPS_ALLOW_PRIVATE_FETCH`. A failed install names the declared package and an HTTP + * status or a fixed reason, never a dependency's name or anything a server sent. + */ +import { expect, layer } from "@effect/vitest"; +import { Effect, Layer, Schema } from "effect"; +import { randomUUID } from "node:crypto"; +import { scenarios } from "../test-plan.ts"; +import { Actors } from "../support/actors.ts"; +import { Api, body, SessionClients, type Session } from "../support/api.ts"; +import { HostedLive, TestLive, withCase, withHostedCase } from "../support/case.ts"; +import { App } from "../support/contracts.ts"; +import { withApps } from "../support/apps-release.ts"; +import { + appPackageFixture, + missingArchiveReason, + privateRegistryFixture, + upstreamMarker, +} from "../support/app-package.ts"; +import { startFreshSelfHost } from "../support/managed-server.ts"; +import { Target } from "../support/platform.ts"; + +const BuildFailed = Schema.Struct({ + _tag: Schema.Literal("VersionBuildFailed"), + stage: Schema.String, + message: Schema.String, +}); +const Organizations = Schema.NonEmptyArray(Schema.Struct({ id: Schema.String })); + +/** An app whose one tool returns `returned`, computed from what it imports from its dependency. */ +const files = (name: string, version: string, imported = "{ value }", returned = "value") => [ + { + path: "package.json", + content: JSON.stringify({ type: "module", dependencies: withApps({ [name]: version }) }), + }, + { + path: "index.ts", + content: `import { defineApp, query, object, router } from "apps"; +import ${imported} from "${name}"; +export default defineApp({}, { tools: router({ value: query({ input: object({}) }, async () => ${returned}) }) });`, + }, +]; + +/** Deploy and call apps as `owner` in the organization at `prefix`. */ +const appsOf = (api: typeof Api.Service, owner: Session, prefix: string) => { + const deploy = (name: string, version: string, imported?: string, returned?: string) => + Effect.gen(function* () { + const deployed = yield* api.request(owner, "POST", `${prefix}/deploy`, { + name: `Dependency ${randomUUID().slice(0, 8)}`, + files: files(name, version, imported, returned), + }); + if (deployed.status === 200) { + const created = yield* body(App, deployed); + yield* Effect.addFinalizer(() => + api.request(owner, "DELETE", `${prefix}/${created.id}`).pipe(Effect.orDie), + ); + } + return deployed; + }); + /** Deploy, then return what the app's tool returns. */ + const installed = (name: string, version: string, imported?: string, returned?: string) => + Effect.gen(function* () { + const deployed = yield* deploy(name, version, imported, returned); + expect(deployed.status, JSON.stringify(deployed.body)).toBe(200); + const app = yield* body(App, deployed); + const called = yield* api.request(owner, "POST", `${prefix}/${app.id}/tools/call`, { + tool: "value", + kind: "query", + input: {}, + }); + expect(called.status, JSON.stringify(called.body)).toBe(200); + return called.body; + }); + /** Deploy, expect the build to fail, and return its message. */ + const failed = (name: string, version: string) => + Effect.gen(function* () { + const deployed = yield* deploy(name, version); + expect(deployed.status, JSON.stringify(deployed.body)).toBe(422); + return (yield* Schema.decodeUnknownEffect(BuildFailed)(deployed.body)).message; + }); + return { installed, failed }; +}; + +const suiteApps = Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors; + return appsOf(api, actors.owner, `/api/organizations/${actors.organization.id}/apps`); +}); + +const refused = "the network policy refused its address"; + +layer(HostedLive, { excludeTestServices: true })("App dependency network", (it) => { + it.effect(scenarios.appDependencyTarballPrivateRefused.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + // The archives listen on loopback. `apps` itself still installs from the suite's + // registry, which also listens on loopback: the registry's origin is not restricted. + const packages = yield* appPackageFixture; + const { failed } = yield* suiteApps; + expect(yield* failed("direct-fixture", packages.direct)).toContain( + `Installing direct-fixture@${packages.direct} failed: Failed to install direct-fixture: ${refused}`, + ); + expect(yield* packages.requests).toEqual({}); + }), + ), + ); + + it.effect(scenarios.appDependencyTarballPublic.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + // npm's own archive host is public; nothing else in the suite is. + const { installed } = yield* suiteApps; + expect( + yield* installed( + "is-number", + "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "isNumber", + "isNumber(5)", + ), + ).toBe(true); + }), + ), + ); + + it.effect(scenarios.appDependencyTarballPrivateAllowed.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const packages = yield* appPackageFixture; + const { installed, failed } = yield* suiteApps; + // The archive and the one it depends on, both on loopback, install. + expect(yield* installed("direct-fixture", packages.direct)).toBe("transitive-package"); + expect(yield* packages.requests).toEqual({ + "/direct-fixture.tgz": 1, + "/transitive-fixture.tgz": 1, + }); + // A missing archive fails the build with its status, not the server's reason or body. + const message = yield* failed("missing-fixture", packages.unused); + expect(message).toContain( + `Installing missing-fixture@${packages.unused} failed: Failed to install missing-fixture: the tarball download answered HTTP 404`, + ); + expect(message).not.toContain(missingArchiveReason); + expect(message).not.toContain("is not served here"); + }), + ), + ); +}); + +layer(TestLive, { excludeTestServices: true })("App dependency registry", (it) => { + it.effect(scenarios.appDependencyRegistryOrigin.title, (context) => + withCase( + context, + Effect.gen(function* () { + const target = yield* Target; + const registry = yield* privateRegistryFixture; + const origin = yield* startFreshSelfHost(target, { + EXECUTOR_NPM_REGISTRY: registry.url, + EXECUTOR_APPS_ALLOW_PRIVATE_FETCH: "false", + }); + const api = yield* Api.pipe( + Effect.provide(Layer.fresh(Api.layer)), + Effect.provide(Layer.fresh(SessionClients.layer)), + Effect.provideService(Target, { ...target, metadata: { ...target.metadata, origin } }), + ); + const owner = yield* api.session(); + expect( + (yield* api.request(owner, "POST", "/api/auth/self-host/setup", { + name: "Registry Owner", + email: "registry-owner@example.test", + password: "Synthetic-registry-password-123!", + organizationName: "Registry lab", + })).status, + ).toBe(200); + const [organization] = yield* body( + Organizations, + yield* api.request(owner, "GET", "/api/auth/organization/list"), + ); + const { installed, failed } = appsOf( + api, + owner, + `/api/organizations/${organization.id}/apps`, + ); + const failure = (name: string) => `Installing ${name}@1.0.0 failed: Failed to install`; + + // The registry's own origin is private and still serves `apps` and its packages. + expect(yield* installed("registry-fixture", "1.0.0")).toBe("registry-package"); + // A tarball its metadata names on another private origin, and one its redirect lands on. + expect(yield* failed("offsite-fixture", "1.0.0")).toContain( + `${failure("offsite-fixture")} offsite-fixture: ${refused}`, + ); + expect(yield* failed("redirected-fixture", "1.0.0")).toContain( + `${failure("redirected-fixture")} redirected-fixture: ${refused}`, + ); + // A tarball URL dependency on another private origin, beneath a package that installs. + expect(yield* failed("nested-fixture", "1.0.0")).toContain( + `${failure("nested-fixture")} a dependency of nested-fixture: ${refused}`, + ); + const requests = yield* registry.requests; + expect(requests["/redirected-fixture.tgz"]).toBe(1); + expect(requests["/nested-fixture/-/nested-fixture-1.0.0.tgz"]).toBe(1); + expect(yield* registry.offsite).toEqual({}); + + // Failures name a fixed reason, never what the registry sent. The registry's own URL is + // written as the build's failure text writes URLs, with its path. + const messages = [ + [ + "malformed-fixture", + `${failure("malformed-fixture")} malformed-fixture: the registry's metadata is not valid JSON`, + ], + [ + "versionless-fixture", + `${failure("versionless-fixture")} versionless-fixture: the registry's metadata names no tarball for the version it resolved`, + ], + [ + "manifest-fixture", + `${failure("manifest-fixture")} a dependency of manifest-fixture: the registry at ${new URL(registry.url).href} has no such package (HTTP 404)`, + ], + ] as const; + for (const [name, expected] of messages) { + const message = yield* failed(name, "1.0.0"); + expect(message).toContain(expected); + expect(message.toLowerCase()).not.toContain(upstreamMarker.toLowerCase()); + } + }), + ), + ); +}); diff --git a/e2e/tests/app-failure-telemetry.spec.ts b/e2e/tests/app-failure-telemetry.spec.ts index 9855ec23ee..a2bdd9db47 100644 --- a/e2e/tests/app-failure-telemetry.spec.ts +++ b/e2e/tests/app-failure-telemetry.spec.ts @@ -737,6 +737,18 @@ layer(HostedLive, { excludeTestServices: true })("App failure telemetry", (it) = for (const { attributes } of appLogs) expect(Object.keys(attributes).filter((key) => !resource.includes(key))).toEqual([]); + // An app's own failure is not a transport failure: no span of the call records the runtime + // session's or route's shape (runtime-hop.ts), on a healthy connection to the runtime. + for (const { spans } of [rejected, malformed]) + for (const { span } of spans) + expect( + Object.keys(span.tags).filter( + (key) => + key.startsWith("executor.runtime.rpc.") || + key.startsWith("executor.runtime.http."), + ), + ).toEqual([]); + if (cloud) { // Cloud's runner records the kind of each failed call, not its text. A cause it does // not recognize is recorded by its shape: the app's Worker rejected the call with an diff --git a/e2e/tests/app-framework-pin.spec.ts b/e2e/tests/app-framework-pin.spec.ts index e46496fbcd..d33cd21b3b 100644 --- a/e2e/tests/app-framework-pin.spec.ts +++ b/e2e/tests/app-framework-pin.spec.ts @@ -583,7 +583,7 @@ const cloudReport = (surface: Surface) => Effect.flatMap((found) => { const fresh = found.data.filter(({ traceId }) => !dispatched.has(traceId)); return fresh.length === 0 - ? Effect.fail(new Error("No dispatch with failed items has reached Motel")) + ? Effect.fail(new Error("No dispatch with failed items has reached the collector")) : Effect.succeed(fresh); }), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), diff --git a/e2e/tests/app-observability.spec.ts b/e2e/tests/app-observability.spec.ts index 73d8ca3e57..b1c15bbaad 100644 --- a/e2e/tests/app-observability.spec.ts +++ b/e2e/tests/app-observability.spec.ts @@ -591,7 +591,7 @@ layer(HostedLive, { excludeTestServices: true })("App observability", (it) => { to: previous.traceId, parent: previous.spanId, }); - // Motel's read API omits native links. The outgoing OTLP assertion above + // The collector's read API omits native links. The outgoing OTLP assertion above // covers managed targets; the deployed Axiom adapter also exposes stored links. if (target.metadata.target === "cloud" && target.metadata.mode === "attached") { const deliveredRetry = yield* telemetry.query(retrySpan.traceId).pipe( diff --git a/e2e/tests/app-ui-srcset.spec.ts b/e2e/tests/app-ui-srcset.spec.ts new file mode 100644 index 0000000000..ccc0448664 --- /dev/null +++ b/e2e/tests/app-ui-srcset.spec.ts @@ -0,0 +1,121 @@ +/** + * Every view of an app page rewrites its `srcset` candidates to the version's asset root. A long + * run of commas inside one candidate costs no more than its length, so the page answers as fast + * as a page of the same size whose commas are in an attribute nothing rewrites. + */ +import { expect, layer } from "@effect/vitest"; +import { Clock, Effect } from "effect"; +import { randomUUID } from "node:crypto"; +import { scenarios } from "../test-plan.ts"; +import { Actors } from "../support/actors.ts"; +import { Api, body } from "../support/api.ts"; +import { Browser } from "../support/browser.ts"; +import { openPrivateApp, waitForAppUrl } from "../support/app-pages.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { App } from "../support/contracts.ts"; +import { appsManifest } from "../support/apps-release.ts"; + +/** + * Commas inside one candidate URL. Rewriting them took time in the square of their count, about + * 4 seconds a view at this count on a 16-vCPU machine, against tens of milliseconds for the same + * page served without a rewrite. + */ +const commas = ",".repeat(100_000); +/** + * How many times slower than the baseline the page may be. Both pages carry the same bytes + * through the same parse, so a linear rewrite keeps them alike; a quadratic one is a hundred times + * slower. + */ +const slowerThanBaseline = 4; +/** Reads of each page, alternated; the fastest of each is compared, so one stalled read is not. */ +const reads = 3; + +const page = (image: string) => [ + { + path: "index.ts", + content: `import { defineApp, query, object, router } from "apps"; +export default defineApp({}, { tools: router({ ready: query({ input: object({}) }, async () => true) }) });`, + }, + { + path: "ui/index.html", + content: `Responsive images
${image}

Loading

`, + }, + { + path: "ui/main.ts", + content: `document.querySelector('[role="status"]').textContent = "Ready";`, + }, + { + path: "ui/public/mark.svg", + content: + '', + }, + appsManifest, +]; + +layer(HostedLive, { excludeTestServices: true })("App page srcset", (it) => { + it.effect(scenarios.appUiLongSrcset.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors, + browser = yield* Browser; + const prefix = `/api/organizations/${actors.organization.id}`; + yield* browser.login(actors.owner); + /** Deploy a page and sign in to its origin on a public file, so the page is unread. */ + const deploy = (name: string, image: string) => + Effect.gen(function* () { + const deployed = yield* api.request(actors.owner, "POST", `${prefix}/apps/deploy`, { + name: `${name} ${randomUUID().slice(0, 8)}`, + files: page(image), + }); + expect(deployed.status, JSON.stringify(deployed.body)).toBe(200); + const app = yield* body(App, deployed); + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", `${prefix}/apps/${app.id}`).pipe(Effect.orDie), + ); + const url = yield* waitForAppUrl(actors.owner, `${prefix}/apps/${app.id}/ui`); + yield* openPrivateApp(`${url}/mark.svg`); + return url; + }); + const long = yield* deploy( + "Long srcset", + `Responsive mark`, + ); + const baseline = yield* deploy("Baseline", `a${commas}b`); + const read = (url: string) => + Effect.gen(function* () { + const started = yield* Clock.currentTimeMillis; + const response = yield* browser.use("Read an app page", (page) => + page + .context() + .request.get(`${url}/`, { headers: { accept: "text/html" }, maxRedirects: 0 }) + .then((response) => + response.text().then((text) => ({ status: response.status(), text })), + ), + ); + expect(response.status).toBe(200); + return { ...response, ms: (yield* Clock.currentTimeMillis) - started }; + }); + const times = { long: [] as number[], baseline: [] as number[] }; + let text = ""; + for (let round = 0; round < reads; round++) { + times.baseline.push((yield* read(baseline)).ms); + const longRead = yield* read(long); + times.long.push(longRead.ms); + text = longRead.text; + } + const fastest = { long: Math.min(...times.long), baseline: Math.min(...times.baseline) }; + expect( + fastest.long, + `the long srcset page's fastest read, against the baseline's ${fastest.baseline} ms (${JSON.stringify(times)})`, + ).toBeLessThanOrEqual(Math.max(fastest.baseline, 1) * slowerThanBaseline); + // Both candidates point at the version's assets; the commas stay inside the first URL. + const version = /srcset="\/_executor\/assets\/([^/"]+)\//.exec(text)?.[1]; + expect(version).toBeDefined(); + const root = `/_executor/assets/${version}/`; + expect(text).toContain(`srcset="${root}a${commas}b 1x, ${root}mark.svg 2x"`); + }), + ), + ); +}); diff --git a/e2e/tests/apps-job.spec.ts b/e2e/tests/apps-job.spec.ts index 645ffafe87..721dd7153b 100644 --- a/e2e/tests/apps-job.spec.ts +++ b/e2e/tests/apps-job.spec.ts @@ -326,6 +326,24 @@ layer(NodeServices.layer)("deploy waits for the apps job", (it) => { }).pipe(Effect.scoped), ); + it.effect("a job of a called workflow named like apps is not the deploy's apps job", () => + Effect.gen(function* () { + const api = yield* jobsApi([ + { + pages: [ + { + total: 1, + jobs: [{ id: 1, name: "x / apps", status: "completed", conclusion: "success" }], + }, + ], + }, + ]); + const result = yield* wait(api.url(), { limit: 2, attempt: 5, poll: 0.5 }); + expect(result.exitCode).toBe(1); + expect(result.log).toContain(`(run ${run} lists no apps job yet); not deploying.`); + }).pipe(Effect.scoped), + ); + it.effect("an apps job listed on the second page is found there", () => Effect.gen(function* () { const others = Array.from({ length: 100 }, (_, index) => ({ diff --git a/e2e/tests/apps-published.spec.ts b/e2e/tests/apps-published.spec.ts index 78509f7af3..7f61d77c9e 100644 --- a/e2e/tests/apps-published.spec.ts +++ b/e2e/tests/apps-published.spec.ts @@ -87,6 +87,10 @@ const registry = ( readonly ownMissing?: number; /** The archive's own URL serves these bytes instead of the origin's. */ readonly ownBytes?: Uint8Array; + /** The tarball URL the version's metadata advertises, instead of the registry's own. */ + readonly tarball?: string; + /** The archive's own URL answers this redirect instead of an archive. */ + readonly redirect?: string; } = {}, ) => { let requests = 0; @@ -113,12 +117,20 @@ const registry = ( response.end( JSON.stringify({ version, - dist: { tarball: `http://127.0.0.1:${port}${tarball}`, integrity }, + dist: { + tarball: options.tarball ?? `http://127.0.0.1:${port}${tarball}`, + integrity, + }, }), ); return; } if (url.pathname === tarball) { + if (options.redirect !== undefined) { + response.writeHead(302, { location: options.redirect }); + response.end(); + return; + } firstArchiveRequest ??= Date.now(); const own = url.search === ""; if (own) ownRequests += 1; @@ -161,6 +173,33 @@ const registry = ( ); }; +/** + * A second origin that serves `bytes` at every path and counts requests. The gate must never + * ask it for an SDK the builder registry advertises elsewhere. + */ +const otherOrigin = (bytes: Uint8Array) => { + let requests = 0; + const server = createServer((_request, response) => { + requests += 1; + response.writeHead(200, { "content-type": "application/octet-stream" }); + response.end(bytes); + }); + return Effect.acquireRelease( + Effect.callback((resume) => { + server.listen(0, "127.0.0.1", () => resume(Effect.void)); + }), + () => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + }), + ).pipe( + Effect.as({ + url: () => `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + requests: () => requests, + }), + ); +}; + /** * Run the check with `flag` against `url` and return its exit code and combined output. The * release check passes no flag; the production deploy's flags refuse another registry. @@ -217,6 +256,31 @@ layer(NodeServices.layer)("apps release check", (it) => { }).pipe(Effect.scoped), ); + it.effect("self-host gate refuses an SDK tarball or redirect on another origin", () => + Effect.gen(function* () { + const { version, local } = yield* archives; + const integrity = integrityOf(local); + const outside = yield* otherOrigin(local); + for (const options of [ + { tarball: `${outside.url()}/apps-${version}.tgz` }, + { redirect: `${outside.url()}/apps-${version}.tgz` }, + ] as const) { + const npm = yield* registry(version, integrity, [{ status: 200, bytes: local }], options); + const run = yield* check(npm.url(), "", {}, true); + expect(run.exitCode, run.log).toBe(1); + expect(run.log).toContain("outside the builder registry's origin"); + expect(run.log).toContain(npm.url()); + expect(run.log).toContain(outside.url()); + } + // The gate refuses before it asks the other origin for the archive. + expect(outside.requests()).toBe(0); + const same = yield* registry(version, integrity, [{ status: 200, bytes: local }]); + const pass = yield* check(same.url(), "", {}, true); + expect(pass.exitCode, pass.log).toBe(0); + expect(pass.log).toContain(`Builder registry serves matching apps@${version}`); + }).pipe(Effect.scoped), + ); + it.effect("self-host gate refuses missing archives and corrupt registry integrity", () => Effect.gen(function* () { const { version, local, different } = yield* archives; diff --git a/e2e/tests/bearer-auth-statements.spec.ts b/e2e/tests/bearer-auth-statements.spec.ts index 25514f5212..a86c11a73f 100644 --- a/e2e/tests/bearer-auth-statements.spec.ts +++ b/e2e/tests/bearer-auth-statements.spec.ts @@ -48,7 +48,9 @@ layer(HostedLive, { excludeTestServices: true })("Bearer authentication", (it) = (entry) => entry.span.tags["http.response.status_code"] === "200", ); if (roots.length === 0 || !complete) - return Effect.fail(new Error("The completed server trace must reach Motel")); + return Effect.fail( + new Error("The completed server trace must reach the collector"), + ); const byId = new Map(result.data.map(({ span }) => [span.spanId, span])); const under = (root: string, spanId: string | null) => { const visited = new Set(); diff --git a/e2e/tests/build-retain-storage.spec.ts b/e2e/tests/build-retain-storage.spec.ts new file mode 100644 index 0000000000..f4e1c4c773 --- /dev/null +++ b/e2e/tests/build-retain-storage.spec.ts @@ -0,0 +1,267 @@ +/** + * A deploy whose build cannot be kept because storage refused the write answers "try again", not + * "your app is wrong". On 17 Nov a Cloud stage's 17 concurrent builds wrote the same framework + * object within 2 s; R2 takes one write a second per key and refused one (`r2.10058`), and that + * deploy answered 422, failing deployed run #1093 before any scenario ran. + * + * Through the SDK's own deploy path (packages/sdk/scripts/build-retain-fixture.ts: the real + * `appRuntime` and retain stage, a compiler stub, and a store that refuses a key's writes faster + * than one a second): + * - 17 concurrent deploys sharing a framework all deploy; the framework is written once and every + * refused write is answered by the object already being there; + * - a framework write refused every time ends with `StorageError` (500, "try again"), naming + * storage, not `VersionBuildFailed`; + * - an app whose own code does not compile still gets `VersionBuildFailed` (422), built once; + * - so does a UI file whose name no storage key can hold (a control character), and a key the + * store refuses as invalid (a 400, as R2's 10020 for a key over 1,024 bytes): the app's own + * failures at the retain stage answer 422 once, with no look-again and no repeat. + * + * And through Cloud's blob store write (apps/hosted/cloud/scripts/r2-write-fixture.ts): R2's + * rate limit (10058), internal error (10001) and temporary unavailability (10043) are repeated and + * the write lands; a failed condition (10031) and an invalid key (10020) are not repeated, and + * the invalid key's failure keeps R2's 400; a persistent refusal stops after three attempts. And + * 17 builds keeping their shared framework at once through that write and the retain stage's + * look-again, against a put refusing a key's writes faster than one a second, are all kept. + * + * And the spans the writes leave: a deploy's version source and initial source writes each have a + * `storage.blob.put` span with the object's kind, its size in bytes and its outcome, and never its + * key; Cloud's write records how many attempts (each a whole upload) it took. + */ +import { expect, layer } from "@effect/vitest"; +import { Effect, Schema } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { scenarios } from "../test-plan.ts"; +import { TestLive, withCase } from "../support/case.ts"; +import { Evidence } from "../support/evidence.ts"; + +const Deploy = Schema.Struct({ + outcome: Schema.Literals(["deployed", "failed"]), + tag: Schema.optional(Schema.String), + stage: Schema.optional(Schema.String), + message: Schema.optional(Schema.String), +}); +const PutSpan = Schema.Struct({ + kind: Schema.String, + size: Schema.Number, + outcome: Schema.String, + ended: Schema.Boolean, + keys: Schema.Array(Schema.String), +}); +const Retain = Schema.Struct({ + writes: Schema.optional(Schema.Array(PutSpan)), + sourceBytes: Schema.optional(Schema.Number), + deploys: Schema.Array(Deploy), + builds: Schema.Number, + framework: Schema.Struct({ + stored: Schema.Number, + refused: Schema.Number, + existsAnswers: Schema.Array(Schema.Boolean), + }), + frameworkObjects: Schema.Number, + elapsedMs: Schema.Number, +}); +const Write = Schema.Struct({ + attempts: Schema.Number, + outcome: Schema.String, + elapsedMs: Schema.Number, +}); +const Writes = Schema.Struct({ + "rate-limited": Write, + internal: Write, + unavailable: Write, + precondition: Write, + persistent: Write, + "invalid-key": Write, + statuses: Schema.Record(Schema.String, Schema.NullOr(Schema.Number)), + attempts: Schema.Record(Schema.String, Schema.NullOr(Schema.Number)), + framework: Schema.Struct({ + retained: Schema.Number, + failed: Schema.Array(Schema.String), + frameworkObjects: Schema.Number, + attempts: Schema.Number, + refused: Schema.Number, + stored: Schema.Number, + elapsedMs: Schema.Number, + }), +}); + +/** Runs a fixture and decodes the JSON line it ends with; its output is kept as evidence. */ +const fixture = >( + script: string, + args: ReadonlyArray, + schema: S, + evidenceName: string, +) => + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner, + evidence = yield* Evidence; + const output = yield* processes.string( + ChildProcess.make("node", [script, ...args], { + env: { PATH: process.env.PATH ?? "", HOME: process.env.HOME ?? "" }, + extendEnv: false, + }), + ); + const result = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(schema))( + output.trim().split("\n").at(-1) ?? "", + ); + yield* evidence.json(evidenceName, result); + return result as S["Type"]; + }); + +const retain = (name: string) => + fixture( + "packages/sdk/scripts/build-retain-fixture.ts", + ["--case", name], + Retain, + `build-retain-${name}.json`, + ); + +layer(TestLive, { excludeTestServices: true })("Build retain storage", (it) => { + it.effect(scenarios.buildRetainStorage.title, (context) => + withCase( + context, + Effect.gen(function* () { + const evidence = yield* Evidence; + yield* evidence.step( + "17 concurrent deploys sharing a framework all deploy, and the framework is written once", + Effect.gen(function* () { + const concurrent = yield* retain("concurrent"); + expect(concurrent.deploys).toHaveLength(17); + expect(concurrent.deploys.filter((deploy) => deploy.outcome !== "deployed")).toEqual( + [], + ); + expect(concurrent.builds).toBe(17); + expect(concurrent.frameworkObjects).toBe(1); + expect(concurrent.framework.stored).toBe(1); + // Every refused write was answered by the object another build stored. + expect(concurrent.framework.refused).toBeGreaterThan(0); + expect(concurrent.framework.existsAnswers.filter(Boolean)).toHaveLength( + concurrent.framework.refused, + ); + }), + ); + yield* evidence.step( + "a framework write refused every time answers StorageError, naming storage, not the app", + Effect.gen(function* () { + const refused = yield* retain("refused"); + expect(refused.deploys).toEqual([ + { + outcome: "failed", + tag: "StorageError", + message: "Executor could not read or write its saved data.", + }, + ]); + expect(refused.frameworkObjects).toBe(0); + // The look before the write, the refused write, then four looks for the object (at once, + // then after 1, 2 and 4 s), none finding it: about 7 s, then the write's own failure. + expect(refused.framework.refused).toBe(1); + expect(refused.framework.existsAnswers).toEqual([false, false, false, false, false]); + expect(refused.elapsedMs).toBeGreaterThan(6_000); + expect(refused.elapsedMs).toBeLessThan(12_000); + }), + ); + yield* evidence.step( + "an app whose own code does not compile still answers VersionBuildFailed, built once", + Effect.gen(function* () { + const source = yield* retain("source"); + expect(source.deploys).toEqual([ + expect.objectContaining({ + outcome: "failed", + tag: "VersionBuildFailed", + stage: "compile", + }), + ]); + expect(source.builds).toBe(1); + expect(source.framework.stored + source.framework.refused).toBe(0); + }), + ); + yield* evidence.step( + "a UI file name no key can hold, and a key the store refuses as invalid, answer 422 once", + Effect.gen(function* () { + for (const name of ["ui-name", "invalid-key"]) { + const own = yield* retain(name); + expect(own.deploys).toEqual([ + { + outcome: "failed", + tag: "VersionBuildFailed", + stage: "retain", + message: "App build failed at the retain stage.", + }, + ]); + expect(own.builds).toBe(1); + // Only the look before the write: no look-again after the refusal. + expect(own.framework.existsAnswers).toEqual([false]); + } + }), + ); + yield* evidence.step( + "a deploy's source writes each have a span with their kind, size and outcome, not their key", + Effect.gen(function* () { + const traced = yield* retain("spans"); + expect(traced.deploys).toEqual([{ outcome: "deployed" }]); + const keys = ["storage.blob.kind", "storage.blob.outcome", "storage.blob.size"]; + expect(traced.writes).toEqual([ + { + kind: "version-source", + size: traced.sourceBytes, + outcome: "stored", + ended: true, + keys, + }, + { + kind: "initial-source", + size: traced.sourceBytes, + outcome: "stored", + ended: true, + keys, + }, + ]); + }), + ); + // One run of Cloud's write fixture: the write retry alone, and with the look-again. + const writes = yield* fixture( + "apps/hosted/cloud/scripts/r2-write-fixture.ts", + [], + Writes, + "r2-write.json", + ); + yield* evidence.step( + "Cloud's blob store repeats R2's rate limit and transient failures, and nothing else", + Effect.gen(function* () { + for (const repeated of [writes["rate-limited"], writes.internal, writes.unavailable]) + expect(repeated).toMatchObject({ attempts: 2, outcome: "stored" }); + expect(writes.precondition).toMatchObject({ attempts: 1, outcome: "failed" }); + expect(writes["invalid-key"]).toMatchObject({ attempts: 1, outcome: "failed" }); + expect(writes.persistent).toMatchObject({ attempts: 3, outcome: "failed" }); + expect(writes.statuses).toEqual({ + "r2.10020": 400, + "r2.10058": null, + "r2.10001": null, + }); + }), + ); + yield* evidence.step( + "Cloud's write records on its span how many attempts it took", + Effect.sync(() => + expect(writes.attempts).toEqual({ + "stored-at-once": 1, + "rate-limited-once": 2, + persistent: 3, + }), + ), + ); + yield* evidence.step( + "17 builds keep a shared framework through Cloud's write retry and the look-again together", + Effect.gen(function* () { + const { framework } = writes; + expect(framework).toMatchObject({ retained: 17, failed: [], frameworkObjects: 1 }); + // R2 takes one write a second to the key: a write or two land, the rest are refused. + expect(framework.refused).toBeGreaterThan(0); + expect(framework.stored).toBeGreaterThanOrEqual(1); + expect(framework.stored).toBeLessThan(5); + }), + ); + }), + ), + ); +}); diff --git a/e2e/tests/ci-repeat.spec.ts b/e2e/tests/ci-repeat.spec.ts index 148bc55484..9f5a42e7a3 100644 --- a/e2e/tests/ci-repeat.spec.ts +++ b/e2e/tests/ci-repeat.spec.ts @@ -10,6 +10,14 @@ import { expect, layer } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { Effect, FileSystem, Path, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { + keyedBase, + payloadFields, + pullRequestReads, + recordKey, + workerGrowthBlock, +} from "../ci-key.ts"; +import { gateEnvironment } from "../support/gate-environment.ts"; /** * A `gh` that answers from files beside it: this run (`run`), the head's runs (`runs`), each run's @@ -41,8 +49,10 @@ if [ -n "$filter" ]; then exec jq -r "$filter" "$file"; else exec cat "$file"; f const self = 200; const head = "0193aee740e16d2b2f15a305e71455381d4667a5"; const base = "6c1f2a9d0b8e4c7f3a5d9e1b2c4f6a8d0e2b4c6f"; -/** What this run's select recorded: its pull request, then a hash of its merge tree and description. */ +/** What this run's select recorded: its pull request, then a key of what it tests (e2e/ci-key.ts). */ const tested = "ci-tested-pr1-3d0f6c2b8a4e1f9d7c5b3a1e0f2d4c6b8a9e7f1d"; +/** What this run's select read for the size gate, as the record the site job uploads. */ +const sized = "ci-sized-pr1-7a1c3e5f9b2d4f6a8c0e2b4d6f8a0c2e4b6d8f0a"; interface Recorded { readonly id: number; @@ -54,7 +64,9 @@ interface Recorded { * this run's record, and the record select uploaded before the pull request was part of it. */ readonly artifacts?: ReadonlyArray; + readonly attempt?: number; readonly select?: string | null; + readonly site?: string | null; readonly check?: string | null; readonly e2e?: string | null; } @@ -67,12 +79,14 @@ const runOf = (input: Recorded) => ({ event: "pull_request", head_sha: input.sha ?? head, workflow_id: 7, + run_attempt: input.attempt ?? 1, }); -const jobsOf = (input: Pick) => ({ +const jobsOf = (input: Pick) => ({ jobs: [ { name: "supersede", conclusion: "success" }, { name: "checks / select", conclusion: input.select === undefined ? "success" : input.select }, + { name: "checks / site", conclusion: input.site === undefined ? "success" : input.site }, { name: "checks / check", conclusion: input.check === undefined ? "success" : input.check }, { name: "checks / e2e", conclusion: input.e2e === undefined ? "success" : input.e2e }, ], @@ -135,6 +149,7 @@ const repeat = (input: { readonly runs: ReadonlyArray; readonly attempt?: number; readonly fail?: boolean; + readonly sized?: string; }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -149,7 +164,9 @@ const repeat = (input: { }; for (const run of all) { answers[`artifacts-${run.id}`] = { - artifacts: (run.artifacts ?? [`ci-tested-${base}`, tested]).map((name) => ({ name })), + artifacts: (run.artifacts ?? [`ci-tested-${base}`, tested, sized]).map((name) => ({ + name, + })), }; answers[`jobs-${run.id}`] = jobsOf(run); } @@ -160,6 +177,7 @@ const repeat = (input: { CI_HEAD_SHA: head, CI_BASE_SHA: base, CI_TESTED: tested, + CI_SIZED: input.sized ?? sized, GITHUB_REPOSITORY: "owner/repository", GITHUB_RUN_ID: String(self), GITHUB_RUN_ATTEMPT: String(input.attempt ?? 1), @@ -201,22 +219,31 @@ const checks = Effect.gen(function* () { return yield* fs.readFileString(".github/workflows/checks.yml"); }); -/** Runs the record step of `select` in a repository whose HEAD holds `tree`, for `pull`. */ +/** + * Runs the record step of `select` in a repository whose HEAD holds `file`, for `pull`, with the + * keys the select step wrote. + */ const record = (input: { readonly pull: number; - readonly description: string; readonly file: string; + readonly base?: string; + /** The pull request's base branch: main by default. */ + readonly baseRef?: string; + readonly selection?: string; + readonly workerGrowth?: string; + /** The size gate's baseline (scripts/ci/worker-baseline.ts's `id`). */ + readonly baseline?: string; }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const script = scriptOf(jobOf(yield* checks, "select"), "Record what this run tests"); - const { directory, bin } = yield* stub({ - pull: { number: input.pull, body: input.description }, - }); + const { directory, bin } = yield* stub({}); const repository = path.join(directory, "checkout"); yield* fs.makeDirectory(repository); yield* fs.writeFileString(path.join(repository, "file"), input.file); + // The step runs e2e/ci-key.ts from the checkout. + yield* fs.symlink(path.resolve("e2e"), path.join(repository, "e2e")); const git = (...args: ReadonlyArray) => spawn( "git", @@ -240,12 +267,19 @@ const record = (input: { PULL: String(input.pull), ACTION: "synchronize", HEAD_SHA: head, - BASE_SHA: base, + BASE_SHA: input.base ?? base, + BASE_REF: input.baseRef ?? "main", + DEFAULT_BRANCH: "main", + SELECTION: input.selection ?? "a".repeat(64), + WORKER_GROWTH: input.workerGrowth ?? "b".repeat(64), + BASELINE: input.baseline ?? `main@${base}`, }, repository, ); expect(result.exitCode, result.log).toBe(0); - return { name: outputsOf(yield* fs.readFileString(output)).name ?? "", log: result.log }; + const tree = (yield* git("rev-parse", "HEAD^{tree}")).log.trim(); + const written = outputsOf(yield* fs.readFileString(output)); + return { name: written.name ?? "", sized: written.sized ?? "", log: result.log, tree }; }).pipe(Effect.scoped); /** Runs `check`'s or `e2e`'s report step, as a repeat of run 199 whose jobs are now `now`. */ @@ -259,16 +293,31 @@ const report = (input: { input.job === "check" ? "Require every static check to pass" : "Require every selected E2E job to pass"; - const script = scriptOf(jobOf(yield* checks, input.job), step); + const text = yield* checks; + const job = jobOf(text, input.job); + const script = scriptOf(job, step); const { bin } = yield* stub({ "jobs-199": jobsOf(input.now) }); + // A repeat's needs: its own select, and every other job skipped, as a repeat skips them. + const needed = job + .match(/^ {4}needs:\s*\[([^\]]*)\]/m)![1]! + .split(",") + .map((name) => name.trim()) + .filter((name) => name.length > 0); + const needs = Object.fromEntries( + needed.map((name) => [ + name, + { + result: name === "select" ? (input.ownSelect ?? "success") : "skipped", + outputs: name === "select" ? { "repeat-of": "199" } : {}, + }, + ]), + ); return yield* spawn("bash", ["-e", "-c", script], bin, { + ...gateEnvironment(text, input.job, step, needs), GITHUB_REPOSITORY: "owner/repository", GITHUB_RUN_ID: String(self), - GH_TOKEN: "synthetic-token", - REPEAT_OF: "199", // What the select of an older version passed on: the result when select ran. REPEAT_RESULT: "failure", - NEEDS: JSON.stringify({ select: { result: input.ownSelect ?? "success", outputs: {} } }), }); }).pipe(Effect.scoped); @@ -281,19 +330,132 @@ layer(NodeServices.layer)("CI repeat", (it) => { expect(passed.exitCode, passed.log).toBe(0); expect(passed.outputs).toEqual({ of: "199", check: "success", e2e: "success" }); expect(passed.log).toContain( - "This run repeats https://github.com/owner/repository/actions/runs/199, which tested the same pull request, merge tree and description: check success, e2e success.", + "This run repeats https://github.com/owner/repository/actions/runs/199, which tested the same pull request, merge tree, base and selection: check success, e2e success.", ); expect(passed.calls.find((call) => call.includes("/workflows/"))).toContain( `head_sha=${head}`, ); - // A failure of a required check is repeated as it was: a repeat never turns red green. + // A failure is not handed on: it is tested again (see "only a pass is handed on"). const failed = yield* repeat({ runs: [{ id: 199, conclusion: "failure", e2e: "failure" }], }); - expect(failed.outputs).toEqual({ of: "199", check: "success", e2e: "failure" }); + expect(failed.outputs).toEqual({}); }), ); + it.effect("only a pass is handed on: a failed, cancelled or unfinished run is tested again", () => + Effect.gen(function* () { + const cases = { + // 61f7d027c, 16 Nov: the hosted emulator timed out in one of two runs on one head. + "a failed e2e": { id: 199, conclusion: "failure", e2e: "failure" }, + "a failed check": { id: 199, conclusion: "failure", check: "failure" }, + "a cancelled run": { id: 199, conclusion: "cancelled" }, + "a run that has not finished": { id: 199, status: "in_progress", conclusion: null }, + } satisfies Record; + for (const [name, run] of Object.entries(cases)) { + const result = yield* repeat({ runs: [run] }); + expect(result.exitCode, `${name}: ${result.log}`).toBe(0); + expect(result.outputs, name).toEqual({}); + expect(result.log, name).toContain("tests in full"); + } + const failed = yield* repeat({ + runs: [{ id: 199, conclusion: "failure", e2e: "failure" }], + }); + expect(failed.log).toContain( + "but its checks failed (check: success, e2e: failure), and only a pass is handed on: a failure is tested again; this run tests in full.", + ); + // A pass still is. + expect((yield* repeat({ runs: [{ id: 199 }] })).outputs.of).toBe("199"); + }), + ); + + /** + * Two recorded runs of one head, as each select logged its key's parts. #4575's two runs at + * fe692429f (16 Nov, 07:42Z) came from one stack push: the lower layer moved from db8e4293f to + * b574b0acc, which the head already held, so the merge tree and selection were the same. #4574, + * a pull request into main, ran on main at 9cbea0adc and later at 82eda7c59. + */ + const recorded = { + upper: { + baseRef: "ci/worker-baseline", + runs: [ + { id: 43825677200, base: "db8e4293ff1e423b597d479f20fc1830ea70191b" }, + { id: 43825680125, base: "b574b0acc36d4a64e3f6947ebec32b94e1dbae32" }, + ], + tree: "f10586856cc6168620d95ea2cbbd6fe9f7ea2d04", + selection: "37f62186dfaffccc8d3c2965b2d9846bf4475edf2232461e7269cc7863706740", + workerGrowth: "5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb", + }, + intoMain: { + baseRef: "main", + runs: [ + { id: 43797130175, base: "9cbea0adc2607de7d6fbabcaa35171d7171b0de8" }, + { id: 43817608300, base: "82eda7c5980da377625cab1c06eea354f744f10c" }, + ], + // Held to one tree, so the base is the only difference. + tree: "c855daf2f83193a10fb116cb79139d405ef65138", + selection: "4accd01b1c0cf0fad749bc11b7e476b915136e964a55266c9c9e2d7afc44c0d1", + workerGrowth: "5b4e0070baf99cd56167f27d96265a3e8e840ecb8cfd4e5a3dea48d9d318c7cb", + }, + }; + + it.effect( + "a stack push that moves the lower layer under an unchanged tree gives the upper's two runs one key", + () => + Effect.gen(function* () { + const { baseRef, runs, tree, selection, workerGrowth } = recorded.upper; + // Main's key before this change: the base commit in every key, so two keys. + const before = runs.map((run) => recordKey(tree, run.base, selection, workerGrowth)); + expect(new Set(before).size).toBe(2); + // Now: an upper layer's key holds no base commit, so one key. + const now = runs.map((run) => + recordKey(tree, keyedBase(baseRef, "main", run.base), selection, workerGrowth), + ); + expect(new Set(now).size).toBe(1); + // And so does the workflow's own record step, for both the tested and the sized record. + const [first, second] = yield* Effect.forEach(runs, (run) => + record({ pull: 4575, file: "the same tree", base: run.base, baseRef }), + ); + expect(first!.name).toBe(second!.name); + expect(first!.sized).toBe(second!.sized); + // The record itself still names each run's real base, which a skip layer below reads. + expect(first!.log).toContain(`base ${runs[0]!.base}`); + }), + ); + + it.effect( + "a pull request into main whose base moved, or a run that selects or compares otherwise, keeps its own key", + () => + Effect.gen(function* () { + const { baseRef, runs, tree, selection, workerGrowth } = recorded.intoMain; + const keys = runs.map((run) => + recordKey(tree, keyedBase(baseRef, "main", run.base), selection, workerGrowth), + ); + expect(new Set(keys).size).toBe(2); + const [first, second] = yield* Effect.forEach(runs, (run) => + record({ pull: 4574, file: "the same tree", base: run.base, baseRef }), + ); + expect(first!.name).not.toBe(second!.name); + expect(first!.sized).not.toBe(second!.sized); + // An upper layer whose base change altered the selection, or its size baseline, keeps two. + const upper = recorded.upper; + const [selected, other] = yield* Effect.forEach([upper.selection, "c".repeat(64)], (each) => + record({ pull: 4575, file: "the same tree", baseRef: upper.baseRef, selection: each }), + ); + expect(selected!.name).not.toBe(other!.name); + const [compared, elsewhere] = yield* Effect.forEach( + ["effect-apps@" + "5".repeat(40), "main@" + "6".repeat(40)], + (baseline) => + record({ pull: 4575, file: "the same tree", baseRef: upper.baseRef, baseline }), + ); + expect(compared!.name).toBe(elsewhere!.name); + expect(compared!.sized).not.toBe(elsewhere!.sized); + // Retargeted onto main, the same head is keyed with its base again. + expect(keyedBase("main", "main", runs[0]!.base)).toBe(runs[0]!.base); + expect(keyedBase(upper.baseRef, "main", runs[0]!.base)).toBe("stacked"); + }), + ); + it.effect("the newest finished run of the same record is the one repeated", () => Effect.gen(function* () { const result = yield* repeat({ @@ -310,8 +472,7 @@ layer(NodeServices.layer)("CI repeat", (it) => { it.effect("a run that did not test the same thing, or did not decide, tests in full", () => Effect.gen(function* () { const cases = { - // Another pull request opened from the same branch onto the same base: its description - // asked for other suites, or carried a deployed-stage line this one lacks. + // Another pull request opened from the same branch onto the same base. "another pull request": [ { id: 199, @@ -321,11 +482,17 @@ layer(NodeServices.layer)("CI repeat", (it) => { ], }, ], - // Another merge tree or description: the record's hash differs. + // Another merge tree, base or selection: the record's key differs. "another record": [ { id: 199, artifacts: ["ci-tested-pr1-0000000000000000000000000000000000000000"] }, ], "no record": [{ id: 199, artifacts: [] }], + // A record whose name only starts or ends like this run's is another record. + "a longer record name": [{ id: 199, artifacts: [`${tested}0`, sized] }], + "a shorter record name": [{ id: 199, artifacts: [tested.slice(0, -1), sized] }], + // A run cancelled after its required checks passed: its other jobs may not have run, and + // a repeat would report it green. + "a cancelled run whose required checks passed": [{ id: 199, conclusion: "cancelled" }], "still running": [{ id: 199, status: "in_progress", conclusion: null }], cancelled: [{ id: 199, conclusion: "cancelled", check: "cancelled", e2e: "cancelled" }], "another head": [{ id: 199, sha: "2ef6ecc4a0c9b09dbb766143499dfefb515f1347" }], @@ -363,24 +530,145 @@ layer(NodeServices.layer)("CI repeat", (it) => { }), ); - it.effect("the record names the pull request and hashes the merge tree and description", () => - Effect.gen(function* () { - const first = yield* record({ pull: 1, description: "```e2e\nnone\n```", file: "one" }); - expect(first.name).toMatch(/^ci-tested-pr1-[0-9a-f]{40}$/); - expect(first.log).toContain("pull request 1"); - // The same pull request, tree and description give the same record. - expect((yield* record({ pull: 1, description: "```e2e\nnone\n```", file: "one" })).name).toBe( - first.name, - ); - // Any one of them changed gives another. - const others = [ - yield* record({ pull: 99, description: "```e2e\nnone\n```", file: "one" }), - yield* record({ pull: 1, description: "```e2e\ngroups.spec.ts\n```", file: "one" }), - yield* record({ pull: 1, description: "```e2e\nnone\n```", file: "two" }), - ]; - for (const other of others) expect(other.name).not.toBe(first.name); - expect(others[0]!.name).toMatch(/^ci-tested-pr99-/); - }), + it.effect( + "the record names the pull request and keys its merge tree, base and what select chose and read", + () => + Effect.gen(function* () { + const first = yield* record({ pull: 1, file: "one" }); + expect(first.name).toBe( + `ci-tested-pr1-${recordKey(first.tree, base, "a".repeat(64), "b".repeat(64))}`, + ); + expect(first.sized).toBe( + `ci-sized-pr1-${recordKey(first.tree, base, "sized", "b".repeat(64), `main@${base}`)}`, + ); + expect(first.log).toContain("pull request 1"); + // The same inputs give the same record. + expect((yield* record({ pull: 1, file: "one" })).name).toBe(first.name); + // Any one of them changed gives another. + const others = { + pull: yield* record({ pull: 99, file: "one" }), + tree: yield* record({ pull: 1, file: "two" }), + base: yield* record({ pull: 1, file: "one", base: "1".repeat(40) }), + selection: yield* record({ pull: 1, file: "one", selection: "c".repeat(64) }), + workerGrowth: yield* record({ pull: 1, file: "one", workerGrowth: "d".repeat(64) }), + }; + for (const [name, other] of Object.entries(others)) + expect(other.name, name).not.toBe(first.name); + expect(others.pull.name).toMatch(/^ci-tested-pr99-/); + // The size gate's record follows only what it reads. + expect(others.selection.sized).toBe(first.sized); + expect(others.workerGrowth.sized).not.toBe(first.sized); + // Compared with another baseline, the size gate's record differs, so a run checked against + // effect-apps is never reported for one that would be checked against main. + const branch = yield* record({ + pull: 1, + file: "one", + baseline: `effect-apps@${"5".repeat(40)}`, + }); + expect(branch.name).toBe(first.name); + expect(branch.sized).not.toBe(first.sized); + }), + ); + + it.effect( + "a run is repeated only from a first attempt whose size gate read what this run's select read", + () => + Effect.gen(function* () { + const cases = { + // Its jobs may have read a later description than its select did. + "a run that was run again": { runs: [{ id: 199, attempt: 2 }] }, + // Its size gate decided on other worker-growth blocks. + "another size-gate read": { + runs: [ + { + id: 199, + artifacts: [tested, "ci-sized-pr1-0000000000000000000000000000000000000000"], + }, + ], + }, + "no size-gate record": { runs: [{ id: 199, artifacts: [tested] }] }, + "this run recorded no size-gate read": { runs: [{ id: 199 }], sized: "" }, + } satisfies Record; sized?: string }>; + for (const [name, input] of Object.entries(cases)) { + const result = yield* repeat(input); + expect(result.exitCode, `${name}: ${result.log}`).toBe(0); + expect(result.outputs, name).toEqual({}); + expect(result.log, name).toContain("tests in full"); + } + // A run whose size gate did not run (a repeat itself, which checked it) needs no record. + const repeated = yield* repeat({ + runs: [{ id: 199, site: "skipped", artifacts: [tested] }], + }); + expect(repeated.outputs.of).toBe("199"); + }), + ); + + it.effect( + "everything a run reads from the pull request is in the key or listed with why it need not be", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const text = yield* checks; + const selection = yield* fs.readFileString("e2e/ci-selection.ts"); + const listed = (file: string, read: string) => + pullRequestReads.some( + ({ where }) => + where.startsWith(`${file}: `) && read.startsWith(where.slice(file.length + 2)), + ); + // Every read the selection makes of the pull request and main. + const routes = [...selection.matchAll(/`repos\/\$\{[^}]+\}\/([^`?]+)/g)].map( + (match) => match[1]!, + ); + expect(routes.length).toBeGreaterThan(5); + expect(routes.filter((route) => !listed("e2e/ci-selection.ts", route))).toEqual([]); + // Every gh pr command the selection runs. + expect(selection).toMatch(/"pr",\s*"list"/); + for (const [command] of selection.matchAll(/"pr",\s*"[a-z]+"/g)) + expect(listed("e2e/ci-selection.ts", command.replace(/,\s*/, ", ")), command).toBe(true); + // Every step of the workflows that reads the pull request through the API or gh pr. + const workflows = yield* fs.readDirectory(".github/workflows"); + for (const name of workflows) { + const workflow = yield* fs.readFileString(`.github/workflows/${name}`); + for (const match of workflow.matchAll( + /^ {6}- name: (.+)\n(?:(?! {6}- ).*\n)*?.*(?:pulls\/\$|gh pr (?:view|list))/gm, + )) + expect(listed(`.github/workflows/${name}`, match[1]!), `${name}: ${match[1]}`).toBe( + true, + ); + // Every field of the event payload's pull request a workflow uses is listed. + for (const [, field] of workflow.matchAll( + /github\.event\.pull_request\.([a-z_]+(?:\.[a-z_]+)*)/g, + )) + expect(Object.keys(payloadFields), `${name}: ${field}`).toContain(field); + } + // Nothing reads the whole event payload, which holds every field of the pull request. + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const grep = yield* processes.spawn( + ChildProcess.make("git", [ + "grep", + "-l", + "-E", + "GITHUB_EVENT_PATH|github[.]event_path", + "--", + ".github", + "e2e", + "scripts", + "apps/hosted/cloud/scripts", + ]), + ); + const payloadReaders = (yield* grep.stdout.pipe(Stream.decodeText(), Stream.mkString)) + .split("\n") + .filter( + (file) => + file !== "" && file !== "e2e/ci-key.ts" && file !== "e2e/tests/ci-repeat.spec.ts", + ); + expect(payloadReaders).toEqual([]); + expect(text).toContain(" - name: Read the pull request's description\n"); + // The size gate and the key read worker-growth blocks with one expression. + expect(yield* fs.readFileString("apps/hosted/cloud/scripts/worker-sizes.ts")).toContain( + `/${workerGrowthBlock.source}/${workerGrowthBlock.flags}`, + ); + }), ); it.effect("a repeat's check and e2e read the repeated run's result as it is now", () => @@ -411,13 +699,29 @@ layer(NodeServices.layer)("CI repeat", (it) => { expect(select).toContain(" repeat-of: ${{ steps.repeat.outputs.of }}\n"); expect(select).toContain(" name: ${{ steps.tested.outputs.name }}\n"); expect(select).toContain(" CI_TESTED: ${{ steps.tested.outputs.name }}\n"); + expect(select).toContain(" CI_SIZED: ${{ steps.tested.outputs.sized }}\n"); expect(select).toContain(" run: node e2e/ci-repeat.ts\n"); // A re-run of the select job replaces its record rather than failing on the name. expect(select).toContain(" overwrite: true\n"); - // The selection does not run, so every E2E job's selection output is empty. - expect(select).toContain( - " - name: Choose the E2E scenarios\n if: steps.repeat.outputs.of == ''\n", - ); + // The selection runs first: what it chose is in the record the repeat check compares. + const order = [ + "Choose the E2E scenarios", + "Record what this run tests", + "Find a finished run", + ].map((step) => select.indexOf(` - name: ${step}`)); + expect(order.every((index) => index > -1)).toBe(true); + expect(order).toEqual([...order].sort((a, b) => a - b)); + // A repeat hands no selection on: every output select writes is empty for it. Skip also + // waits for the check of the layer above (worker-sizes.spec.ts). + const outputs = [...select.matchAll(/^ {6}([a-z-]+): \$\{\{ (.*) \}\}$/gm)]; + expect(outputs.length).toBeGreaterThan(10); + for (const [, name, value] of outputs) + if (value!.includes("steps.select.outputs")) + expect(value, name).toBe( + `steps.repeat.outputs.of == ''${name === "skip" ? " && steps.above.outcome == 'success'" : ""} && steps.select.outputs.${name} || ''`, + ); + // The site job records what its size gate read. + expect(jobOf(text, "site")).toContain(" - name: Record what the size gate reads\n"); for (const job of [ "static", "site", @@ -456,19 +760,26 @@ layer(NodeServices.layer)("CI repeat", (it) => { const script = scriptOf(job, "Require every static check to pass"); const { bin } = yield* stub({}); const run = (results: Record, collector: string | null = "true") => - spawn("bash", ["-e", "-c", script], bin, { - REPEAT_OF: "", - NEEDS: JSON.stringify( - Object.fromEntries( - needs.map((name) => [ - name, - { - result: results[name] ?? "success", - outputs: name === "select" && collector !== null ? { collector } : {}, - }, - ]), + Effect.gen(function* () { + return yield* spawn( + "bash", + ["-e", "-c", script], + bin, + gateEnvironment( + yield* checks, + "check", + "Require every static check to pass", + Object.fromEntries( + needs.map((name) => [ + name, + { + result: results[name] ?? "success", + outputs: name === "select" && collector !== null ? { collector } : {}, + }, + ]), + ), ), - ), + ); }); expect((yield* run({})).exitCode).toBe(0); // A job that failed, was cancelled, or was skipped by mistake fails the check. Select says diff --git a/e2e/tests/ci-selection.spec.ts b/e2e/tests/ci-selection.spec.ts index da84988d7b..b42ef4d3e4 100644 --- a/e2e/tests/ci-selection.spec.ts +++ b/e2e/tests/ci-selection.spec.ts @@ -9,7 +9,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { Effect, FileSystem, Path, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; -import { collectorChanged, collectorInputs } from "../collector-inputs.ts"; +import { collectorChanged, collectorInputs, collectorScenarios } from "../collector-inputs.ts"; import dockerReleaseConfig from "../docker-release.config.ts"; import { lanesFor, releaseSources, releaseSuiteConfigs } from "../release-lanes.ts"; import { @@ -19,7 +19,17 @@ import { packagedEngineVariable, } from "../packaged-engine.ts"; import { closureOf } from "../save-path.ts"; -import { scenarios } from "../test-plan.ts"; +import { + matchesPath, + mcpSearchChanged, + mcpSearchPaths, + mcpSearchScenarios, + mcpSearchTitles, +} from "../mcp-search-guard.ts"; +import { selectionKey, workerGrowthKey } from "../ci-key.ts"; +import { gateEnvironment, stepEnvironmentBlock } from "../support/gate-environment.ts"; +import { fileGuards } from "../file-guards.ts"; +import { scenarios, scenariosForSuite } from "../test-plan.ts"; const jobs = [ "local", @@ -103,6 +113,8 @@ const select = (input: { readonly stackedAbove?: string | ReadonlyArray; readonly wait?: number; readonly root?: string; + /** More of the job's environment. */ + readonly env?: Readonly>; }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -160,6 +172,7 @@ const select = (input: { GITHUB_RUN_ID: "123456", GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: summary, + ...input.env, }, extendEnv: true, }), @@ -169,16 +182,20 @@ const select = (input: { [child.all.pipe(Stream.decodeText(), Stream.mkString), child.exitCode], { concurrency: "unbounded" }, ); - const outputs = Object.fromEntries( + const written = Object.fromEntries( (yield* fs.readFileString(output)) .split("\n") .filter((line) => line.length > 0) .map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)]), ); + // What a later run compares to tell whether it would test the same thing (e2e/ci-key.ts), + // apart from the outputs the jobs read. + const { selection, "worker-growth": workerGrowth, ...outputs } = written; return { exitCode, log, outputs, + keys: { selection, workerGrowth }, summary: yield* fs.readFileString(summary), }; }).pipe(Effect.scoped); @@ -341,6 +358,15 @@ const workflowJob = (name: string) => /** The select outputs that start each E2E job, as its `if` in checks.yml reads them. */ const startedBy = { + // The run's one collector build, for every job below that starts or packages it. + collector: [ + "local-shards", + "self-host-shards", + "self-host-inventory", + "self-host-catalog", + "cloud-runners", + "release-local", + ], "e2e-local": ["local-shards"], "e2e-self-host": ["self-host-shards"], "e2e-self-host-scale": ["self-host-inventory", "self-host-catalog"], @@ -352,6 +378,37 @@ const startedBy = { type E2eJob = keyof typeof startedBy; const e2eJobs = Object.keys(startedBy) as ReadonlyArray; +/** + * The cases that run the selection as many real processes at once. On an idle host they take 10 to + * 20 seconds; at a load of 15 on this host's 16 cores, up to 23; at the load of 22 where one failed + * in review, past the 30-second default. Their limit is several times their idle time, so a busy + * host slows them and does not fail them; every other case keeps the default. + */ +const heavyCaseTimeout = 120_000; + +/** The `select` outputs whose matrix fields the size budget measures (each entry's fields). */ +const matrixOutputs = ["local-shards", "self-host-shards", "cloud-runners"]; + +/** + * Each read of a `select` output in a workflow's code (comments removed) that could put the + * selection's size into a process: anywhere but a job or step condition, a matrix the budget + * measures, a comparison (`== '…'`, `!= '…'`, a short value), or a direct read of an output in + * `measured`. Quoted or inside a larger expression, in an `env:` value or a `run:` body alike. + */ +const unmeasuredReads = (code: string, measured: ReadonlyArray) => + code.split("\n").flatMap((line, index) => { + if (/^\s*(?:- )?if: /.test(line)) return []; + return [...line.matchAll(/needs\.select\.outputs\.([\w-]+)(\s*[!=]=\s*'[^']*')?/g)].flatMap( + (match) => { + const name = match[1]!; + if (match[2] !== undefined) return []; + if (matrixOutputs.includes(name) && /^\s*include: \$\{\{ fromJSON\(/.test(line)) return []; + if (measured.includes(name)) return []; + return [`line ${index + 1} reads ${name}: ${line.trim()}`]; + }, + ); + }); + /** Whether select's outputs start an E2E job. */ const chosen = (outputs: Readonly>, job: E2eJob) => startedBy[job].some((name) => (outputs[name] ?? "") !== ""); @@ -368,13 +425,21 @@ const gate = ( const processes = yield* ChildProcessSpawner.ChildProcessSpawner; const { script } = yield* workflowJob("e2e"); expect(script).toBeDefined(); + const fs = yield* FileSystem.FileSystem; const needs = { select, ...Object.fromEntries(e2eJobs.map((job) => [job, { result: results[job], outputs: {} }])), }; + // What GitHub passes the step: its env block evaluated against these needs. + const env = gateEnvironment( + yield* fs.readFileString(".github/workflows/checks.yml"), + "e2e", + "Require every selected E2E job to pass", + needs, + ); const child = yield* processes.spawn( ChildProcess.make("bash", ["-e", "-c", script!], { - env: { NEEDS: JSON.stringify(needs) }, + env: { ...env, REPEAT_OF: "" }, extendEnv: true, }), ); @@ -388,8 +453,8 @@ const gate = ( /** This suite's own config, rewritten only in a fixture tree. */ const ownConfig = "e2e/ci-selection.config.ts"; -/** Files `fileGuards` in e2e/ci-selection.ts watches; main lists each in its directory. */ -const guardedFiles = ["packages/sdk/src/implementation/app-protocols.ts"]; +/** Files `fileGuards` (e2e/file-guards.ts) watches; main lists each in its directory. */ +const guardedFiles = Object.keys(fileGuards); /** * Main's listings of the guarded files' directories, with each file's blob ID in this checkout * unless `changed` gives another. @@ -403,7 +468,86 @@ const guardedListings = (blobs: ReadonlyArray, changed: Record { + const start = workflow.indexOf(`\n ${job}:\n`); + const line = workflow + .slice(start) + .split("\n") + .find((text) => text.startsWith(" runs-on: ${{ ")); + if (start === -1 || line === undefined) throw new Error(`No runs-on for ${job}`); + const expression = line.slice(" runs-on: ${{ ".length, -" }}".length); + const values: Record = { + "vars.CI_RUNNER_MODE": run.mode, + "github.event_name": run.event, + "github.event.pull_request.head.repo.full_name": run.fork + ? "someone/executor-next" + : "UsefulSoftwareCo/executor-next", + "github.repository": "UsefulSoftwareCo/executor-next", + "needs.select.outputs.full": run.full, + }; + const code = expression.replace(/[a-z][\w.-]*\.[\w.-]+/giu, (name) => { + if (!(name in values)) throw new Error(`runs-on names ${name}, which this does not know`); + return JSON.stringify(values[name]); + }); + if (/[^"'\w\s.()=!&|/-]/u.test(code.replaceAll(/"[^"]*"|'[^']*'/gu, ""))) + throw new Error(`runs-on is not a plain expression: ${expression}`); + // oxlint-disable-next-line no-new-func -- a GitHub expression of literals and comparisons only + return String(new Function(`return (${code.replaceAll("'", '"')});`)()); +}; + layer(NodeServices.layer)("CI E2E selection", (it) => { + it.effect( + "self-host runs whole shards on 8 vCPU and every other run on 16, and the desktop mode is untouched", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const workflow = yield* fs.readFileString(".github/workflows/checks.yml"); + const label = (run: Partial[2]>) => + runnerLabel(workflow, "e2e-self-host", { + mode: "", + event: "pull_request", + fork: false, + full: "false", + ...run, + }); + // A whole run (a push to main, an `all` block): 4 workers a shard, 8 vCPU. + expect(label({ event: "push", full: "true" })).toBe("blacksmith-8vcpu-ubuntu-2404"); + expect(label({ full: "true" })).toBe("blacksmith-8vcpu-ubuntu-2404"); + // A narrowed or partial run: 8 workers, 16 vCPU. + expect(label({ full: "false" })).toBe("blacksmith-16vcpu-ubuntu-2404"); + expect(label({ event: "push", full: "false" })).toBe("blacksmith-16vcpu-ubuntu-2404"); + // The desktop mode takes this repository's runs whatever their size, and never a fork's. + expect(label({ mode: "desktop", full: "true" })).toBe("executor-ci-desktop"); + expect(label({ mode: "desktop", full: "false" })).toBe("executor-ci-desktop"); + expect(label({ mode: "desktop", event: "push", full: "true" })).toBe("executor-ci-desktop"); + expect(label({ mode: "desktop", fork: true, full: "true" })).toBe( + "blacksmith-8vcpu-ubuntu-2404", + ); + expect(label({ mode: "desktop", fork: true, full: "false" })).toBe( + "blacksmith-16vcpu-ubuntu-2404", + ); + // WORKERS reads the same output: whole runs run 4 workers a shard. + expect(workflow).toContain( + "WORKERS: ${{ needs.select.outputs.full == 'true' && '4' || '8' }}", + ); + }), + ); + it.effect("a push to main runs every job's full pattern", () => Effect.gen(function* () { const run = yield* select({}); @@ -573,7 +717,9 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { }); for (const [index, run] of runs.entries()) { expect(run.exitCode, names[index]).toBe(1); - expect(run.log, names[index]).toContain(`not spec files in e2e/tests/: "${names[index]}"`); + expect(run.log, names[index]).toContain( + `- line 1 of the e2e block, \`${names[index]}\`: no spec file has this name. The nearest names are `, + ); expect(run.outputs, names[index]).toEqual({}); } }), @@ -582,21 +728,29 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { it.effect("a spec file another config runs fails and names what runs it", () => Effect.gen(function* () { const cases = [ - ["billing.spec.ts", "e2e/billing.config.ts: bun run e2e:billing. No workflow runs it."], + [ + "billing.spec.ts", + "billing.spec.ts runs only by hand, with e2e/billing.config.ts (bun run e2e:billing); no workflow runs it, so naming it here runs nothing. Remove billing.spec.ts from the e2e block.", + ], + // A release lane runs it, when the change reaches that lane: not on every pull request. [ "docker-release.spec.ts", - "e2e/docker-release.config.ts in Checks (.github/workflows/checks.yml) and Executor releases (.github/workflows/release-artifacts.yml)", + "docker-release.spec.ts runs with e2e/docker-release.config.ts in Checks (.github/workflows/checks.yml) and Executor releases (.github/workflows/release-artifacts.yml), not in these jobs, so naming it here runs nothing. Remove docker-release.spec.ts from the e2e block.", ], // Planned, but its scenarios run only against the release archive. [ "local-bootstrap.spec.ts", - "e2e/local-bootstrap.config.ts in Checks (.github/workflows/checks.yml) and Executor releases (.github/workflows/release-artifacts.yml)", + "local-bootstrap.spec.ts runs with e2e/local-bootstrap.config.ts in Checks (.github/workflows/checks.yml) and Executor releases (.github/workflows/release-artifacts.yml), not in these jobs, so naming it here runs nothing. Remove local-bootstrap.spec.ts from the e2e block.", ], + // The tooling job runs it on every pull request. [ "ci-selection.spec.ts", - "e2e/ci-selection.config.ts in Checks (.github/workflows/checks.yml). Locally: bun run e2e:ci-selection.", + "ci-selection.spec.ts runs with e2e/ci-selection.config.ts in Checks on every pull request anyway, so the e2e block cannot select it. Remove ci-selection.spec.ts from the e2e block. Locally: bun run e2e:ci-selection.", + ], + [ + "claude-mcp.spec.ts", + "these jobs exclude all of claude-mcp.spec.ts's scenarios (see the job patterns in e2e/ci-selection.ts), so naming it here runs nothing. Remove claude-mcp.spec.ts from the e2e block.", ], - ["claude-mcp.spec.ts", "these jobs exclude all of its scenarios"], ] as const; const runs = yield* Effect.forEach( cases, @@ -606,9 +760,8 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { for (const [index, run] of runs.entries()) { const [file, where] = cases[index]!; expect(run.exitCode, file).toBe(1); - expect(run.log, file).toContain(`- ${file}`); - expect(run.log, file).toContain(where); - expect(run.log, file).toContain("Remove them from the e2e block."); + expect(run.log, file).toContain(`- line 2 of the e2e block, \`${file}\`: ${where}`); + expect(run.log, file).toContain("gh run rerun 123456."); expect(run.outputs, file).toEqual({}); } }), @@ -630,7 +783,7 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { expect(run.outputs["self-host"]).toMatch(/^\^\(\?:/); expect(run.summary).toContain("Spec files: groups.spec.ts\n"); expect(run.summary).toContain( - "- billing.spec.ts runs only by hand, with e2e/billing.config.ts", + "- billing.spec.ts: billing.spec.ts runs only by hand, with e2e/billing.config.ts (bun run e2e:billing); no workflow runs it, so these jobs do not run it.", ); }), ); @@ -761,7 +914,7 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { const run = yield* select({ body: block("deployment-links.spec.ts") }); expect(run.exitCode).toBe(1); expect(run.log).toContain( - "- deployment-links.spec.ts: only Cloud tests on main runs its scenarios, on a deployed stage after merge.", + "- line 1 of the e2e block, `deployment-links.spec.ts`: only the deployed stage runs deployment-links.spec.ts's scenarios, in Cloud tests on main after merge, so naming it here runs nothing. Remove deployment-links.spec.ts from the e2e block. To run its scenarios before merging, run them on a deployed stage: bun run e2e:deployed --database planetscale --test-name '^(?:Cloud product links follow the deployment origin)$'", ); }), ); @@ -789,7 +942,7 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { expect(changed.summary).toContain("Spec files: deployment-links.spec.ts\n"); expect(other.exitCode).toBe(1); expect(other.log).toContain( - "- deployment-links.spec.ts: only Cloud tests on main runs its scenarios, on a deployed stage after merge.", + "- line 1 of the e2e block, `deployment-links.spec.ts`: only the deployed stage runs deployment-links.spec.ts's scenarios, in Cloud tests on main after merge, so naming it here runs nothing. Remove deployment-links.spec.ts from the e2e block. To run its scenarios before merging, run them on a deployed stage: bun run e2e:deployed --database planetscale --test-name '^(?:Cloud product links follow the deployment origin)$'", ); expect(other.outputs).toEqual({}); }), @@ -995,9 +1148,17 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { expect(refused.log).toContain( `bun run e2e:deployed --database planetscale --test-name '^(?:${title})$'`, ); + // The exact line to add, which the check then accepts as written. + const line = `Verified on a deployed stage: bun run e2e:deployed --database planetscale --test-name '^(?:${title})$' passed`; expect(refused.log).toContain( - '"Verified on a deployed stage: "', + `When that command passes, add this line to the description, on a line of its own, then run this workflow again:\n${line}\n`, ); + const added = yield* select({ + body: `${block("none")}\n${line}\n`, + changed, + main: main(without), + }); + expect(added.exitCode, added.log).toBe(0); // A statement of how it was verified lets it through; a bare heading does not. expect(stated.exitCode, stated.log).toBe(0); expect(emptyStatement.exitCode).toBe(1); @@ -1011,6 +1172,7 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { "Could not read main's e2e/test-plan.ts, so whether this pull request adds scenarios to the deployed suite was not checked.", ); }), + heavyCaseTimeout, ); it.effect( @@ -1147,7 +1309,7 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { expect(main.exitCode, main.log).toBe(0); expect(named.exitCode).toBe(1); expect(named.log).toContain( - `- ${included} runs with ${ownConfig} in Checks (.github/workflows/checks.yml).`, + `- line 1 of the e2e block, \`${included}\`: ${included} runs with ${ownConfig} in Checks on every pull request anyway, so the e2e block cannot select it.`, ); expect(changed.exitCode).toBe(1); expect(changed.log).toContain( @@ -1365,6 +1527,788 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { }), ); + /** #4456's changed files, and #4457's tests layer above it with its e2e block. */ + const searchBatch = { + compared: [ + "apps/hosted/server/src/implementation/access.ts", + "apps/hosted/server/src/implementation/mcp.ts", + "apps/hosted/server/src/implementation/resource-lifecycle.ts", + "apps/hosted/server/src/implementation/resource-policy.ts", + "apps/hosted/server/src/implementation/tools.ts", + "apps/local/server/src/implementation/app-accounts.ts", + "notes/cloud-latency.md", + "packages/sdk/src/contracts/executor.ts", + "packages/sdk/src/contracts/tools.ts", + "packages/sdk/src/implementation/create.ts", + "packages/sdk/src/implementation/declarations.ts", + "packages/sdk/src/implementation/listings.ts", + "packages/sdk/src/implementation/oauth.ts", + "packages/sdk/src/implementation/tool-search-store.ts", + "packages/sdk/src/implementation/tool-search.ts", + "packages/sdk/src/implementation/tools.ts", + "e2e/test-plan.ts", + "e2e/tests/search-statements.spec.ts", + ], + changed: ["e2e/test-plan.ts", "e2e/tests/search-statements.spec.ts"], + body: block( + "search-statements.spec.ts", + "kept-listings.spec.ts", + "mcp-catalog.spec.ts", + "mcp-connections.spec.ts", + "mcp-execute-failures.spec.ts", + "mcp-execute-reach.spec.ts", + "mcp-server.spec.ts", + "mcp-account-choice.spec.ts", + "app-routers.spec.ts", + "app-accounts-tab.spec.ts", + "bearer-auth-statements.spec.ts", + "tool-call-statements.spec.ts", + "tool-index-redeploy.spec.ts", + ), + }; + /** Every title a run's job patterns select on `target`, read from the patterns as exact titles. */ + const selectsTitle = (outputs: Readonly>, job: string, title: string) => + (outputs[job] ?? "") !== "" && new RegExp(outputs[job]!).test(title); + + it.effect( + "the search batch that reddened main selects the Cloud scenario it broke, from its changed files", + () => + Effect.gen(function* () { + const run = yield* select(searchBatch); + expect(run.exitCode, run.log).toBe(0); + const broken = scenarios.cloudMcpDiscoveryListings.title; + expect(broken).toBe( + "Cloud MCP session objects discover the other apps of an execute while one app's listing waits on a lock", + ); + expect(selectsTitle(run.outputs, "cloud-product", broken)).toBe(true); + expect(run.summary).toContain( + "(e2e/mcp-search-guard.ts), so these Cloud scenarios run too", + ); + }), + ); + + /** The job each guarded scenario runs in, as e2e/ci-selection.ts's job patterns place it. */ + const guardedJobs: Readonly> = { + cloudMcpDiscoveryListings: "cloud-product", + cloudMcpParallelCalls: "cloud-product", + cloudMcpAbandonedStatement: "cloud-product", + cloudMcpObjectPool: "cloud-product", + cloudMcpSessionRouting: "cloud-product", + cloudMcpSessionRestore: "cloud-product", + // Its 120-second idle wait would set the length of any other job. + cloudMcpObjectConnections: "cloud-connections", + cloudMcpObjectConnectRetry: "cloud", + mcpBackgroundListingCloud: "cloud", + mcpStoppedListingCloud: "cloud", + mcpStoppedRenewalCloud: "cloud", + cloudMcpStreamLifetime: "cloud", + // It counts every request in the session object isolate. + cloudMcpSessionTiming: "cloud-isolate", + // It holds a lock on the accounts table. + cloudMcpCallBatch: "cloud-batch", + }; + + it.effect( + "the MCP search guard adds its Cloud scenarios to the block, in their own jobs, once, and never to a skipped layer", + () => + Effect.gen(function* () { + const guarded = "packages/sdk/src/implementation/tools.ts"; + const runs = yield* Effect.all( + { + guarded: select({ body: block("mcp-catalog.spec.ts"), compared: [guarded] }), + unguarded: select({ body: block("mcp-catalog.spec.ts"), compared: ["notes/x.md"] }), + none: select({ body: block("none"), compared: [guarded] }), + named: select({ + body: block("cloud-database-placement.spec.ts"), + compared: [guarded], + }), + skip: select({ body: block("skip"), stackedAbove: "7", compared: [guarded] }), + }, + { concurrency: 4 }, + ); + for (const [name, run] of Object.entries(runs)) + expect(run.exitCode, `${name}: ${run.log}`).toBe(0); + const plan = scenariosForSuite("all", "managed"); + // Each guarded title runs in exactly one Cloud job, the one whose pattern holds it. + for (const title of mcpSearchTitles) { + const holding = cloudJobs.filter((job) => selectsTitle(runs.guarded.outputs, job, title)); + expect(holding, title).toHaveLength(1); + expect(selectsTitle(runs.none.outputs, holding[0]!, title), title).toBe(true); + expect(selectsTitle(runs.unguarded.outputs, holding[0]!, title), title).toBe(false); + } + // Nothing the block selects is removed, and no other target gains a scenario. + for (const job of jobs.filter((job) => !job.startsWith("cloud"))) + expect(runs.guarded.outputs[job], job).toBe(runs.unguarded.outputs[job]); + // The block's own Cloud scenarios outside the table still run beside the guarded ones. + const named = plan.filter( + (scenario) => + scenario.file === "cloud-database-placement.spec.ts" && + scenario.targets.cloud.status === "scheduled" && + !mcpSearchTitles.has(scenario.title), + ); + expect(named.length).toBeGreaterThan(2); + for (const scenario of named) + expect( + cloudJobs.some((job) => selectsTitle(runs.named.outputs, job, scenario.title)), + scenario.title, + ).toBe(true); + // Each guarded title lands in the job checks.yml's reasons give it, and no other. + expect(Object.keys(guardedJobs).sort()).toEqual([...mcpSearchScenarios].sort()); + for (const [key, job] of Object.entries(guardedJobs)) { + const title = scenarios[key as keyof typeof scenarios].title; + expect( + cloudJobs.filter((each) => selectsTitle(runs.none.outputs, each, title)), + `${key} runs only in ${job}`, + ).toEqual([job]); + } + // A title the block already selects is in its job's pattern once. + const broken = scenarios.cloudMcpDiscoveryListings.title; + const pattern = runs.named.outputs["cloud-product"]!; + expect(pattern.split(broken.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).length - 1).toBe(1); + // A skipped layer still skips; the layer above carries the run. + expect(runs.skip.outputs.skip).toBe("true"); + expect(runs.skip.outputs["cloud-runners"] ?? "").toBe(""); + }), + ); + + it.effect("the MCP search guard's table names files that exist and scenarios Cloud runs", () => + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* processes.spawn(ChildProcess.make("git", ["ls-files"])); + const tracked = (yield* child.stdout.pipe(Stream.decodeText(), Stream.mkString)) + .split("\n") + .filter((file) => file !== ""); + expect(tracked.length).toBeGreaterThan(1000); + expect( + mcpSearchPaths.filter((entry) => !tracked.some((file) => matchesPath(entry, file))), + "paths in e2e/mcp-search-guard.ts that match no file", + ).toEqual([]); + // A path entry's forms, and #4456's files. + expect(matchesPath("packages/mcp/", "packages/mcp/src/server.ts")).toBe(true); + expect( + matchesPath( + "packages/sdk/src/implementation/tool-search*.ts", + "packages/sdk/src/implementation/tool-search-store.ts", + ), + ).toBe(true); + expect( + matchesPath( + "packages/sdk/src/implementation/tool-search*.ts", + "packages/sdk/src/implementation/tool-search/x.ts", + ), + ).toBe(false); + expect(mcpSearchChanged(["packages/sdk/src/implementation/tools.ts"])).toBe(true); + expect(mcpSearchChanged(["packages/sdk/src/implementation/tools-index.ts"])).toBe(false); + + const cloud = scenariosForSuite("all", "managed").filter( + (scenario) => scenario.targets.cloud.status === "scheduled", + ); + const scheduled = new Set(cloud.map((scenario) => scenario.title)); + expect( + mcpSearchScenarios.filter((key) => !scheduled.has(scenarios[key].title)), + "scenarios in e2e/mcp-search-guard.ts that Cloud does not run", + ).toEqual([]); + // A new scenario of the family is guarded from the day it is written: every Cloud scenario + // in a cloud-mcp-session-* spec file, and every MCP session scenario of + // cloud-database-placement.spec.ts, read by the phrase its titles share. + const family = cloud.filter( + (scenario) => + /^cloud-mcp-session-[\w-]+\.spec\.ts$/.test(scenario.file) || + (scenario.file === "cloud-database-placement.spec.ts" && + scenario.title.includes("MCP session")), + ); + expect(family.length).toBeGreaterThan(8); + expect( + family.filter((scenario) => !mcpSearchTitles.has(scenario.title)).map((s) => s.title), + "Cloud MCP session scenarios not guarded: add each one's key to mcpSearchScenarios in e2e/mcp-search-guard.ts", + ).toEqual([]); + }), + ); + + /** + * The six pull request heads of 14 Nov whose second run tested again only because the description + * was edited between the two runs' reads: each head's e2e block before and after, as the authors + * wrote it, with the prose around it standing in for theirs. #4365's blocks also named a spec file + * the plan no longer has; it is left out of both versions. + */ + const fourteenth = (() => { + const harness = "all: changes the e2e harness, the CI workflows and the lockfile"; + const runtime = [ + "local-app-runtime-restart.spec.ts", + "local-workflow-engine-steps.spec.ts", + "local-workflow-engine-memory.spec.ts", + "local-registry-credentials.spec.ts", + "tool-call-held-check.spec.ts", + "app-failure-details.spec.ts", + "app-failure-telemetry.spec.ts", + ]; + const limit = [ + "compiled-file-limit.spec.ts", + "local-compiled-file-limit.spec.ts", + "app-failure-details.spec.ts", + "app-source-imports.spec.ts", + "local-apps-cli.spec.ts", + ]; + const described = (prose: string, lines: ReadonlyArray) => + `${prose}\n\n${block(...lines)}`; + return { + // The description gained a results table. + af220a946: [ + described("Summary.", [harness]), + described("Summary.\n\n| Run | Result |\n| --- | --- |\n| 1 | passed |", [harness]), + ], + // A line about a step's skip flag changed. + b6df4276c: [ + described("- The step fails when its outputs are missing.", [harness]), + described("- `--skip collector:build` neither runs the step nor touches its outputs.", [ + harness, + ]), + ], + "5d4189980": [ + described("Notes.", [...runtime, "mcp-protocol-versions.spec.ts"]), + described("Notes, edited.", [...runtime, "mcp-protocol-versions.spec.ts"]), + ], + // The block gained a spec file: the suites changed. + d2e5026fb: [ + described("Notes.", runtime), + described("Notes.", [...runtime, "mcp-protocol-versions.spec.ts"]), + ], + // Rows about a deploy and skipped packages changed. + "7efe1d9e0": [ + described( + "| Check removed | deploy 200 instead of 422 |\n| Packages skipped | fails |", + limit, + ), + described( + "| Check removed (re-run) | deploy 200 instead of 422 |\n\nThe packages skipped row no longer applies.", + limit, + ), + ], + "348c22613": [described("A first draft.", ["none"]), described("A second draft.", ["none"])], + } as const; + })(); + + it.effect( + "the repeat key is what the selection chose: an edit that leaves it keeps the key, one that changes it does not", + () => + Effect.gen(function* () { + const changed = ["e2e/tests/groups.spec.ts"]; + const growth = "```worker-growth\napi +60,000: the SAML library\n```"; + const runs = yield* Effect.all( + { + plain: select({ body: `Summary.\n\n${block("groups.spec.ts")}`, changed }), + prose: select({ + body: `Another summary.\n\n${block("groups.spec.ts")}\n\nResults.`, + changed, + }), + // A deployed-stage line only decides whether select passes; with no scenario new to + // the deployed suite, it changes nothing that runs. + deployedLine: select({ + body: `${block("groups.spec.ts")}\nVerified on a deployed stage: bun run e2e:deployed passed\n`, + changed, + }), + spec: select({ body: block("groups.spec.ts", "invitation-roles.spec.ts"), changed }), + none: select({ body: block("none"), changed }), + growth: select({ body: `${block("groups.spec.ts")}\n\n${growth}`, changed }), + otherGrowth: select({ + body: `${block("groups.spec.ts")}\n\n${growth.replace("60,000", "90,000")}`, + changed, + }), + skip: select({ body: block("skip"), stackedAbove: "7", changed }), + push: select({ changed }), + }, + { concurrency: 4 }, + ); + for (const [name, run] of Object.entries(runs)) { + expect(run.exitCode, `${name}: ${run.log}`).toBe(0); + expect(run.keys.selection, name).toMatch(/^[0-9a-f]{64}$/); + } + const { plain } = runs; + // The key changes with every output the jobs are given: each line select wrote, changed + // alone, changes it. The lines are those select wrote, apart from the two keys. + for (const [name, run] of Object.entries(runs)) { + const lines = Object.entries(run.outputs).map(([key, value]) => `${key}=${value}`); + expect(lines.length, name).toBeGreaterThan(0); + for (const [index, line] of lines.entries()) + expect( + selectionKey(lines.with(index, `${line}0`).join("\n")), + `${name}: ${line.slice(0, line.indexOf("="))}`, + ).not.toBe(run.keys.selection); + } + // Every output a job of checks.yml reads is one select writes in one of these runs, and so + // one the key covers. + const fs = yield* FileSystem.FileSystem; + const read = new Set( + [ + ...(yield* fs.readFileString(".github/workflows/checks.yml")).matchAll( + /steps\.select\.outputs\.([a-z-]+)/g, + ), + ].map((match) => match[1]!), + ); + const written = new Set([ + ...Object.values(runs).flatMap((run) => Object.keys(run.outputs)), + "selection", + "worker-growth", + ]); + expect(read.size).toBeGreaterThan(10); + expect([...read].filter((name) => !written.has(name))).toEqual([]); + expect(runs.growth.keys.workerGrowth).toBe( + workerGrowthKey(`${block("groups.spec.ts")}\n\n${growth}`), + ); + // Same selection, same blocks for the size gate: the same key. none selects what naming + // groups.spec.ts did, because a changed spec file runs anyway. + for (const name of ["prose", "deployedLine", "none"] as const) + expect(runs[name].keys, name).toEqual(plain.keys); + // Another selection: another selection key, the size gate's unchanged. + for (const name of ["spec", "skip"] as const) { + expect(runs[name].keys.selection, name).not.toBe(plain.keys.selection); + expect(runs[name].keys.workerGrowth, name).toBe(plain.keys.workerGrowth); + } + // Another worker-growth block: the same selection, another size-gate key. + expect(runs.growth.keys.selection).toBe(plain.keys.selection); + expect(runs.growth.keys.workerGrowth).not.toBe(plain.keys.workerGrowth); + expect(runs.otherGrowth.keys.workerGrowth).not.toBe(runs.growth.keys.workerGrowth); + // A push has no description, so no size-gate key, and records nothing. + expect(runs.push.keys.workerGrowth).toBeUndefined(); + }), + heavyCaseTimeout, + ); + + it.effect( + "the six heads of 14 Nov that tested twice: five now match, d2e5026fb still tests again", + () => + Effect.gen(function* () { + const keys = yield* Effect.forEach( + Object.entries(fourteenth), + ([head, [before, after]]) => + Effect.all([select({ body: before }), select({ body: after })]).pipe( + Effect.map(([first, second]) => { + expect(first.exitCode, `${head}: ${first.log}`).toBe(0); + expect(second.exitCode, `${head}: ${second.log}`).toBe(0); + return [ + head, + first.keys.selection === second.keys.selection && + first.keys.workerGrowth === second.keys.workerGrowth, + ] as const; + }), + ), + { concurrency: 3 }, + ); + expect(Object.fromEntries(keys)).toEqual({ + af220a946: true, + b6df4276c: true, + "5d4189980": true, + d2e5026fb: false, + "7efe1d9e0": true, + "348c22613": true, + }); + }), + heavyCaseTimeout, + ); + + /** The blocks that failed select on 13–14 Nov, each with the edit its message must name. */ + const failedBlocks = { + // #4439: a file name that is not a spec file. + notASpec: { + body: block("search-statements.spec.ts", "search-rejected-accounts.spec.ts"), + says: "- line 2 of the e2e block, `search-rejected-accounts.spec.ts`: no spec file has this name. The nearest names are ", + }, + // #4377: a spec file Checks runs on every pull request. + checksOnly: { + body: block("mcp-server.spec.ts", "effect-mcp-session-bound.spec.ts"), + says: "- line 2 of the e2e block, `effect-mcp-session-bound.spec.ts`: effect-mcp-session-bound.spec.ts runs with e2e/effect-mcp.config.ts in Checks on every pull request anyway, so the e2e block cannot select it. Remove effect-mcp-session-bound.spec.ts from the e2e block.", + }, + // #4369: a spec file only the deployed stage runs. + deployedOnly: { + body: block("groups.spec.ts", "root-entry-loading.spec.ts"), + says: "- line 2 of the e2e block, `root-entry-loading.spec.ts`: only the deployed stage runs root-entry-loading.spec.ts's scenarios, in Cloud tests on main after merge, so naming it here runs nothing. Remove root-entry-loading.spec.ts from the e2e block. To run its scenarios before merging, run them on a deployed stage: bun run e2e:deployed --database planetscale --test-name '^(?:", + }, + // #4393: skip before the layer above was open. + lateLayer: { + body: block("skip"), + says: "The e2e block says skip (line 1 of the e2e block, `skip`), which only a lower layer of a stack may say: this run waited 0 seconds for the layer above, an open pull request whose base is this branch, ", + }, + } as const; + + it.effect("each block that failed select on 13–14 Nov is told the line and the edit", () => + Effect.gen(function* () { + for (const [name, { body, says }] of Object.entries(failedBlocks)) { + const run = yield* select({ body }); + expect(run.exitCode, name).toBe(1); + expect(run.log, name).toContain(says); + expect(run.outputs, name).toEqual({}); + } + // #4439's nearest names include the spec file its stack had added under another name. + const near = yield* select({ body: block("search-statement.spec.ts") }); + expect(near.log).toContain("The nearest names are search-statements.spec.ts, "); + }), + ); + + it.effect( + "e2e:check-description gives select's verdict on a draft, failing or passing, from the checkout", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const directory = yield* fs.makeTempDirectoryScoped({ + prefix: "executor-check-description-", + }); + const branch = (yield* processes + .spawn(ChildProcess.make("git", ["rev-parse", "--abbrev-ref", "HEAD"])) + .pipe( + Effect.flatMap((child) => child.stdout.pipe(Stream.decodeText(), Stream.mkString)), + Effect.scoped, + )).trim(); + // HEAD as main and as the base: nothing changed, as in select's runs below. + const check = (body: string) => + Effect.gen(function* () { + const file = path.join(directory, "description.md"); + yield* fs.writeFileString(file, body); + const child = yield* processes.spawn( + ChildProcess.make( + "node", + ["e2e/check-description.ts", "--file", file, "--main", "HEAD", "--base", "HEAD"], + { extendEnv: true }, + ), + ); + const [log, exitCode] = yield* Effect.all( + [child.all.pipe(Stream.decodeText(), Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + return { log, exitCode: Number(exitCode) }; + }).pipe(Effect.scoped); + /** The verdict a run prints: its failure message, or the selection's job table. */ + const verdict = (log: string) => + ( + /SelectionFailed: ([\s\S]*?)\n {4}at /.exec(log)?.[1] ?? + log.slice(log.indexOf("| Job | Scenarios | Runners |")) + ) + // Only the advice a CI run can follow differs. + .replaceAll( + "CI waited 0 seconds for the description. Once it is fixed, rerun the whole workflow: gh run rerun 123456", + "Edit the description and check it again", + ) + .replaceAll(`base is this branch, ${branch},`, "base is this branch, layer,") + .replaceAll(`with base ${branch},`, "with base layer,"); + // A skip is checked once the layer above is open (the next case), so it is left out here. + const drafts = { + ...Object.fromEntries( + Object.entries(failedBlocks) + .filter(([name]) => name !== "lateLayer") + .map(([name, { body }]) => [name, body]), + ), + passes: block("groups.spec.ts"), + }; + for (const [name, body] of Object.entries(drafts)) { + const [local, ci] = [yield* check(body), yield* select({ body })]; + expect(local.exitCode, `${name}: ${local.log}`).toBe(ci.exitCode); + expect(verdict(local.log), name).toBe(verdict(ci.log)); + expect(verdict(local.log).length, name).toBeGreaterThan(40); + } + }), + heavyCaseTimeout, + ); + + it.effect( + "e2e:check-description passes a skip it cannot check yet, gives no CI advice, checks a pull request only at its head, and lists 300 files as GitHub does", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "executor-check-pr-" }); + const run = ( + command: string, + args: ReadonlyArray, + env: Record = {}, + cwd?: string, + ) => + Effect.gen(function* () { + const child = yield* processes.spawn( + ChildProcess.make(command, args, { + env, + extendEnv: true, + ...(cwd === undefined ? {} : { cwd }), + }), + ); + const [log, exitCode] = yield* Effect.all( + [child.all.pipe(Stream.decodeText(), Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + return { log, exitCode: Number(exitCode) }; + }).pipe(Effect.scoped); + const head = (yield* run("git", ["rev-parse", "HEAD"])).log.trim(); + const file = path.join(directory, "description.md"); + const local = (args: ReadonlyArray, env: Record = {}) => + run("node", ["e2e/check-description.ts", ...args, "--main", "HEAD"], env); + + // A skip in a draft: nothing to check it against yet, so it says so and passes. + yield* fs.writeFileString(file, block("skip")); + // Run from inside a CI job, the check clears GITHUB_ACTIONS for the selection it starts. + const skip = yield* local(["--file", file], { GITHUB_ACTIONS: "true" }); + expect(skip.exitCode, skip.log).toBe(0); + expect(skip.log).toContain("A skip cannot be checked before both pull requests exist."); + // A failing draft gets the advice an author can follow locally, not a CI run's. + yield* fs.writeFileString(file, failedBlocks.notASpec.body); + const failed = yield* local(["--file", file]); + expect(failed.exitCode).toBe(1); + expect(failed.log).toContain("Edit the description and check it again."); + expect(failed.log).not.toContain("gh run rerun"); + + // --pr reads the pull request with gh: here a stand-in answering for pull request 7, + // whose layer above is #9, at `oid`. + const bin = path.join(directory, "bin"); + yield* fs.makeDirectory(bin); + const pr = (oid: string) => + Effect.gen(function* () { + yield* fs.writeFileString( + path.join(bin, "view.json"), + JSON.stringify({ + body: block("skip"), + baseRefName: "main", + headRefName: "feature", + headRefOid: oid, + }), + ); + yield* fs.writeFileString( + path.join(bin, "gh"), + // The layer above is based on the pull request's head branch, feature, not this one. + '#!/bin/sh\ncase "$*" in\n "pr view"*) cat "$(dirname "$0")/view.json" ;;\n "pr list"*"--base feature"*) echo 9 ;;\n "pr list"*) ;;\n *) exit 3 ;;\nesac\n', + ); + yield* fs.chmod(path.join(bin, "gh"), 0o755); + return yield* local(["--pr", "7"], { PATH: `${bin}:${process.env.PATH ?? ""}` }); + }); + const elsewhere = yield* pr("0".repeat(40)); + expect(elsewhere.exitCode).toBe(2); + expect(elsewhere.log).toContain( + `This checkout's HEAD is ${head}, but pull request 7's head is ${"0".repeat(40)}, so its changed files would not be the pull request's. Check out that commit first: git fetch origin feature && git checkout --detach ${"0".repeat(40)}`, + ); + const atHead = yield* pr(head); + expect(atHead.exitCode, atHead.log).toBe(0); + expect(atHead.log).toContain("Skipped: a lower stack layer under #9."); + + // The comparison lists at most 300 files, as GitHub's does, so a longer one runs all. + const repository = path.join(directory, "repository"); + yield* fs.makeDirectory(repository); + const git = (...args: ReadonlyArray) => + run( + "git", + ["-c", "user.name=Synthetic", "-c", "user.email=synthetic@example.test", ...args], + {}, + repository, + ); + yield* git("init", "-q"); + yield* fs.writeFileString(path.join(repository, "README.md"), "main\n"); + yield* git("add", "-A"); + yield* git("commit", "-q", "-m", "main"); + const main = (yield* git("rev-parse", "HEAD")).log.trim(); + for (let index = 0; index < 301; index++) + yield* fs.writeFileString( + path.join(repository, `file-${String(index).padStart(3, "0")}.ts`), + "\n", + ); + yield* git("add", "-A"); + yield* git("commit", "-q", "-m", "head"); + const listed = yield* run( + "node", + [ + path.resolve("e2e/check-description-gh.ts"), + "api", + "repos/o/r/compare/main...head", + "--jq", + ".files[] | .filename", + ], + { CHECK_DESCRIPTION_MAIN: main }, + repository, + ); + expect(listed.exitCode, listed.log).toBe(0); + expect(listed.log.trim().split("\n")).toHaveLength(300); + }), + ); + + it.effect( + "the local check's mode cannot run in a workflow: it fails there, and no workflow sets it", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const [inActions, local, ci] = yield* Effect.all( + [ + select({ body: block("skip"), env: { E2E_LOCAL_CHECK: "1", GITHUB_ACTIONS: "true" } }), + select({ body: block("skip"), env: { E2E_LOCAL_CHECK: "1", GITHUB_ACTIONS: "" } }), + select({ body: block("skip"), env: { GITHUB_ACTIONS: "true" } }), + ], + { concurrency: "unbounded" }, + ); + // In a GitHub Actions run it fails at once, before reading the pull request. + expect(inActions.exitCode).toBe(1); + expect(inActions.log).toContain( + "E2E_LOCAL_CHECK is set in a GitHub Actions run. It is the mode of e2e/check-description.ts, which passes a skip that has no layer above; a workflow must never set it.", + ); + expect(inActions.log).not.toContain("Waiting"); + expect(inActions.outputs).toEqual({}); + // Outside one, a lone skip is reported as not checkable yet, and passes. + expect(local.exitCode, local.log).toBe(0); + expect(local.log).toContain("A skip cannot be checked before both pull requests exist."); + expect(local.outputs).toEqual({}); + // A workflow's lone skip still fails after its wait. + expect(ci.exitCode).toBe(1); + expect(ci.log).toContain("which only a lower layer of a stack may say"); + // Nothing under .github/ sets it. + const mentions: Array = []; + const walk = (directory: string): Effect.Effect => + Effect.gen(function* () { + for (const name of yield* fs.readDirectory(directory)) { + const file = `${directory}/${name}`; + if ((yield* fs.stat(file)).type === "Directory") yield* walk(file); + else if ((yield* fs.readFileString(file)).includes("E2E_LOCAL_CHECK")) + mentions.push(file); + } + }); + yield* walk(".github"); + expect(mentions).toEqual([]); + }), + ); + + /** The skill entries of `fileGuards`, each spec file with the jobs its scenarios run in. */ + const skillGuards: Readonly>>>> = { + "packages/app-templates/executor/skills/executor/SKILL.md": { + "framework-discovery.spec.ts": ["self-host"], + "cli-tool-commands.spec.ts": ["local"], + }, + "packages/app-templates/executor/skills/executor/feedback.md": { + "skill-read-telemetry.spec.ts": ["local", "self-host"], + }, + "packages/app-templates/executor/skills/code-mode/SKILL.md": { + "framework-discovery.spec.ts": ["self-host"], + }, + "packages/app-templates/executor/skills/app-authoring/SKILL.md": { + "framework-discovery.spec.ts": ["self-host"], + "cli-tool-commands.spec.ts": ["local"], + "mcp-server.spec.ts": ["self-host"], + "local-skills.spec.ts": ["local"], + }, + "packages/app-templates/executor/skills/app-authoring/ui.md": { + "framework-discovery.spec.ts": ["self-host"], + "cli-tool-commands.spec.ts": ["local"], + "local-skills.spec.ts": ["local"], + }, + "packages/app-templates/src/implementation/executor-intro.gen.ts": { + "cli-tool-commands.spec.ts": ["local"], + "mcp-server.spec.ts": ["self-host"], + }, + }; + + /** Main as this checkout, except that each of `changed` has another blob ID there. */ + const mainChanging = (changed: ReadonlyArray) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const names = yield* fs.readDirectory("patches"); + const hashed = (yield* processes.string( + ChildProcess.make("git", [ + "hash-object", + ...guardedFiles, + ...names.map((name) => `patches/${name}`), + ]), + )) + .trim() + .split("\n"); + return { + "package.json": yield* fs.readFileString("package.json"), + patches: names + .map((name, index) => `${hashed[guardedFiles.length + index]} ${name}`) + .join("\n"), + ...guardedListings( + hashed.slice(0, guardedFiles.length), + Object.fromEntries(changed.map((file) => [file, "0".repeat(40)])), + ), + }; + }); + + it.effect( + "the skill texts #4545 reworded select framework-discovery's scenario, whatever #4546's block named", + () => + Effect.gen(function* () { + const run = yield* select({ + body: block( + "mcp-direct-calls.spec.ts", + "cli-tool-commands.spec.ts", + "cli-grant-choice.spec.ts", + "tool-call-log.spec.ts", + ), + changed: [ + "e2e/test-plan.ts", + "e2e/tests/cli-tool-commands.spec.ts", + "e2e/tests/mcp-direct-calls.spec.ts", + ], + main: yield* mainChanging([ + "packages/app-templates/executor/skills/code-mode/SKILL.md", + "packages/app-templates/executor/skills/executor/SKILL.md", + "packages/app-templates/src/implementation/executor-intro.gen.ts", + ]), + }); + expect(run.exitCode, run.log).toBe(0); + // By the scenario's title in the plan, whatever its wording becomes. + expect(selectsTitle(run.outputs, "self-host", scenarios.frameworkDiscovery.title)).toBe( + true, + ); + expect(run.summary).toContain("A guarded file differs from main, so its guards run: "); + // Without the skill texts changed, the block alone does not select it. + const unguarded = yield* select({ + body: block( + "mcp-direct-calls.spec.ts", + "cli-tool-commands.spec.ts", + "cli-grant-choice.spec.ts", + "tool-call-log.spec.ts", + ), + main: yield* mainChanging([]), + }); + expect( + selectsTitle(unguarded.outputs, "self-host", scenarios.frameworkDiscovery.title), + ).toBe(false); + }), + ); + + it.effect( + "each skill text runs the spec files that read it back, in the jobs a pull request runs them in", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const plan = scenariosForSuite("all", "managed"); + for (const [file, specs] of Object.entries(skillGuards)) { + // The table names exactly these spec files for the file. + expect([...(fileGuards[file] ?? [])].sort(), file).toEqual(Object.keys(specs).sort()); + const run = yield* select({ body: block("none"), main: yield* mainChanging([file]) }); + expect(run.exitCode, `${file}: ${run.log}`).toBe(0); + for (const [spec, expected] of Object.entries(specs)) { + const titles = plan.filter((scenario) => scenario.file === spec); + const ran = jobs.filter((job) => + titles.some((scenario) => selectsTitle(run.outputs, job, scenario.title)), + ); + expect(ran, `${file}: ${spec}`).toEqual(expected); + } + } + // Every guarded file exists, and every guarded spec file has scenarios a pull request runs. + for (const [file, specs] of Object.entries(fileGuards)) { + expect(yield* fs.exists(file), file).toBe(true); + for (const spec of specs) { + expect(yield* fs.exists(`e2e/tests/${spec}`), spec).toBe(true); + expect( + plan.some( + (scenario) => + scenario.file === spec && + (scenario.targets.local.status === "scheduled" || + scenario.targets["self-host"].status === "scheduled"), + ), + `${spec} has no scenario a pull request runs`, + ).toBe(true); + } + } + }), + heavyCaseTimeout, + ); + it.effect("none, all and skip keep their meaning", () => Effect.gen(function* () { const [none, all, bareAll, skip, loneSkip, missing, twice] = yield* Effect.all( @@ -1447,7 +2391,7 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { ); } expect(upper.exitCode).toBe(1); - expect(upper.log).toContain('not spec files in e2e/tests/: "SKIP"'); + expect(upper.log).toContain("- line 1 of the e2e block, `SKIP`: no spec file has this name."); expect(upper.log).not.toContain("Waiting"); expect(named.exitCode, named.log).toBe(0); expect(named.outputs["self-host"]).toMatch(/^\^\(\?:/); @@ -1470,8 +2414,9 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { "CI waited 1 second for the description. Once it is fixed, rerun the whole workflow: gh run rerun 123456.", ); expect(loneSkip.exitCode).toBe(1); - expect(loneSkip.log).toContain("no open pull request builds on this branch"); - expect(loneSkip.log).toContain("gh run rerun 123456."); + expect(loneSkip.log).toContain( + "The e2e block says skip (line 1 of the e2e block, `skip`), which only a lower layer of a stack may say: this run waited 1 second for the layer above, an open pull request whose base is this branch, layer, and there is none. Open the layer above with base layer, then run this workflow again: gh run rerun 123456.", + ); }), ); it.effect("a full run passes when every scheduled scenario ran, passed or failed", () => @@ -1846,10 +2791,10 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { [["packages/telemetry/collector/src/http.rs"], ["true", "true"]], [["packages/telemetry/collector/Cargo.lock"], ["true", "true"]], [["packages/telemetry/scripts/build-collector.ts"], ["true", "true"]], - // The actions that build it run only in the matrix after merge until the cutover wires them - // into these lanes, so they start neither. - [[".github/actions/collector/action.yml"], ["", ""]], - [[".github/actions/rustup/action.yml"], ["", ""]], + // The actions that build it: the release-local jobs build or restore the collector through + // them before packaging it; the image builds its own in its Dockerfile. + [[".github/actions/collector/action.yml"], ["", "true"]], + [[".github/actions/rustup/action.yml"], ["", "true"]], [["apps/docs/public/install.sh"], ["", "true"]], [["scripts/releases/config.ts"], ["true", "true"]], // The lanes' own definition runs both. @@ -2070,7 +3015,8 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { "packages/telemetry/src/isolate.ts", "packages/telemetry/collector-notes.md", "e2e/tests/collector.spec.ts", - "packages/telemetry/scripts/build-motel.ts", + // Another script beside the build script. + "packages/telemetry/scripts/check-notices.ts", ".github/actions/setup/action.yml", ".github/actions/collector-notes.md", ]) @@ -2105,6 +3051,19 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { expect(runs[1]!.summary).toContain( "collector-native: skipped, no collector input changed (e2e/collector-inputs.ts).", ); + // A crate change runs the scenarios that exercise the collector, whatever the block names; + // another change runs none of them. + for (const file of collectorScenarios) { + expect(runs[0]!.summary, file).toContain(file); + expect(runs[1]!.summary, file).not.toContain(file); + } + expect(runs[0]!.summary).toContain( + "A collector input changed (e2e/collector-inputs.ts), so the scenarios that exercise it run", + ); + expect(runs[0]!.outputs["self-host"]).toContain( + "The bundled collector at its size bound keeps its traces", + ); + expect(runs[1]!.summary).toContain("No E2E scenarios"); }), ); @@ -2655,10 +3614,6 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { // Windows and macOS only. "Install Chromium on desktop runners", "Smoke-test the installed CLI", - // The wait for this run's collector job and its one build, which nothing packages yet; - // the lane gets its own collector step when packaging starts using it. - "Wait for the telemetry collector", - "Fetch the telemetry collector", ], ], [ @@ -2681,10 +3636,6 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { "Record the packaged desktop app's digest", // Runs only when the job failed; the lane's own e2e evidence covers CI. "Gather the failed scenarios' evidence", - // The wait for this run's collector job and its one build, which nothing packages yet; - // the lane gets its own collector step when packaging starts using it. - "Wait for the telemetry collector", - "Fetch the telemetry collector", ], ], ] as const; @@ -2775,6 +3726,27 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { ); }), ); + it.effect("the e2e check names a failed collector job as the cause of the jobs that waited", () => + Effect.gen(function* () { + const outputs = { "local-shards": "1/1", "release-local": "true" }; + const results = Object.fromEntries( + e2eJobs.map((job) => [job, chosen(outputs, job) ? "failure" : "skipped"]), + ) as Record; + const failed = yield* gate({ result: "success", outputs }, results); + expect(failed.exitCode).toBe(1); + expect(failed.log).toContain( + "::error::collector (linux-x64): failure. The E2E jobs that wait for its build fail with it; rerun that job first.", + ); + expect(failed.log).toContain("::error::e2e-local: selected, but failure"); + // A collector that succeeded is never named as a cause. + const passed = yield* gate( + { result: "success", outputs }, + { ...results, collector: "success" }, + ); + expect(passed.exitCode).toBe(1); + expect(passed.log).not.toContain("The E2E jobs that wait for its build fail with it"); + }), + ); it.effect( "the e2e check fails when either release-local job of a selected lane fails, is cancelled or is skipped", () => @@ -2787,8 +3759,9 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { ), ...change, }) as Record; - // The lane is two jobs, and selecting it selects both. + // The lane is two jobs, and selecting it selects both, and the collector build they package. expect(e2eJobs.filter((job) => chosen(outputs, job))).toEqual([ + "collector", "release-local-cli", "release-local-desktop", ]); @@ -2805,6 +3778,7 @@ layer(NodeServices.layer)("CI E2E selection", (it) => { { result: "success", outputs: {} }, { ...results(), + collector: "skipped", "release-local-cli": "skipped", "release-local-desktop": "success", }, @@ -3178,4 +4152,221 @@ layer(NodeServices.layer)("Release check after a push to main", (it) => { expect(result.log.split("\n").some((line) => line.startsWith("::"))).toBe(false); }), ); + + /** + * Linux refuses an environment variable or an argument longer than 131,072 bytes (E2BIG, #4583). + * With every scenario selected, each value the selection puts into a process's environment or + * arguments stays under this budget, a quarter of the limit: the largest today is a shard's + * pattern, 7 kB on main and 13 kB on effect-apps. + */ + const selectionValueBudget = 32 * 1024; + + it.effect( + "with every scenario selected, nothing the selection carries reaches an environment variable or argument near its limit", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const workflow = yield* fs.readFileString(".github/workflows/checks.yml"); + const full = yield* select({}); + expect(full.exitCode, full.log).toBe(0); + const outputs = full.outputs; + const values: Array<{ readonly what: string; readonly value: string }> = []; + // The two required gates, as GitHub passes them a full run's needs. + const jobNeeds = (job: string) => + workflow + .slice(workflow.indexOf(`\n ${job}:\n`)) + .match(/^ {4}needs:\s*\[([^\]]*)\]/m)![1]! + .split(",") + .map((name) => name.trim()) + .filter((name) => name.length > 0); + for (const [job, step] of [ + ["check", "Require every static check to pass"], + ["e2e", "Require every selected E2E job to pass"], + ] as const) { + const needs = Object.fromEntries( + jobNeeds(job).map((name) => [ + name, + { result: "success", outputs: name === "select" ? outputs : {} }, + ]), + ); + for (const [name, value] of Object.entries(gateEnvironment(workflow, job, step, needs))) + values.push({ what: `${job}'s ${name}`, value }); + } + // Each matrix entry's fields reach TEST_NAME, `--test-name` and E2E_TEST_NAME. + for (const output of ["local-shards", "self-host-shards", "cloud-runners"]) + for (const entry of JSON.parse(outputs[output] || "[]") as ReadonlyArray< + Record + >) + for (const [field, value] of Object.entries(entry)) + values.push({ what: `${output} ${entry.shard ?? entry.name} ${field}`, value }); + // The select outputs a step reads into a variable directly. + const direct = [ + "collector", + "repeat-of", + "full", + "self-host-inventory", + "self-host-catalog", + ]; + for (const output of direct) values.push({ what: output, value: outputs[output] ?? "" }); + const over = values + .map(({ what, value }) => ({ what, bytes: Buffer.byteLength(value) })) + .filter(({ bytes }) => bytes > selectionValueBudget) + .map( + ({ what, bytes }) => + `${what} is ${bytes.toLocaleString("en-US")} bytes with every scenario selected, over its budget of ${selectionValueBudget.toLocaleString("en-US")}. Linux refuses an environment variable or argument over 131,072 bytes (spawn E2BIG): split it (more shards for its job in e2e/ci-selection.ts) or pass it through a file. Do not shorten scenario titles to fit.`, + ); + expect(over).toEqual([]); + expect(values.length).toBeGreaterThan(20); + // No step gets a select output into its environment other than those measured above, or + // as whether it is set, and no gate gets toJSON(needs): a new path is measured first. + const code = workflow + .split("\n") + .filter((line) => !line.trimStart().startsWith("#")) + .join("\n"); + expect(code).not.toMatch(/toJSON\(needs\b/); + expect(unmeasuredReads(code, direct)).toEqual([]); + // The run's own processes: vitest gets its title pattern from its config and the prepared + // Cloud organizations from a file, not from an argument or a variable. + const suite = yield* fs.readFileString("e2e/sdk/suite.ts"); + expect(suite).not.toContain('"--testNamePattern"'); + expect(suite).not.toContain("E2E_PREPARED_SCENARIOS"); + expect(yield* fs.readFileString("e2e/vitest.config.ts")).toContain( + 'testNamePattern: patternForTarget(target, suite, process.env.E2E_TEST_NAME ?? "", cloudMode),', + ); + }), + ); + + it.effect( + "a gate fails when a job in its needs has no variable, or a variable names no job it needs", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const workflow = yield* fs.readFileString(".github/workflows/checks.yml"); + for (const [job, step] of [ + ["check", "Require every static check to pass"], + ["e2e", "Require every selected E2E job to pass"], + ] as const) { + const text = workflow.slice(workflow.indexOf(`\n ${job}:\n`)); + const needed = text + .match(/^ {4}needs:\s*\[([^\]]*)\]/m)![1]! + .split(",") + .map((name) => name.trim()) + .filter((name) => name.length > 0); + // Statically: one NEED_ variable per needed job, each reading that job's own result. + const block = stepEnvironmentBlock(workflow, job, step); + const variables = [...block.matchAll(/^ {10}NEED_(\w+): (.*)$/gm)].map((match) => ({ + job: match[1]!.replaceAll("_", "-"), + value: match[2]!, + })); + expect(variables.map((variable) => variable.job).toSorted(), job).toEqual( + needed.toSorted(), + ); + for (const variable of variables) + expect(variable.value, job).toBe(`\${{ needs.${variable.job}.result }}`); + expect(block, job).toContain(" RESULTS: ${{ join(needs.*.result, ' ') }}\n"); + // Each SET_ reads whether its own output is set, each OUTPUT_ its own output's value. + const outputs = [...block.matchAll(/^ {10}(SET|OUTPUT)_(\w+): (.*)$/gm)].map((match) => ({ + kind: match[1]!, + name: match[2]!.replaceAll("_", "-"), + value: match[3]!, + })); + for (const output of outputs) + expect(output.value, `${job} ${output.kind}_${output.name}`).toBe( + output.kind === "SET" + ? `\${{ needs.select.outputs.${output.name} != '' }}` + : `\${{ needs.select.outputs.${output.name} }}`, + ); + // Every output the rule reads has its variable: one it names without one would read as + // unset, so its job would count as not selected, and a wrongly skipped job would pass. + const ruleText = text.slice(text.indexOf(` - name: ${step}\n`)); + const rule = ruleText + .slice(ruleText.indexOf(" run: |\n") + " run: |\n".length) + .split("\n") + .filter((line, index, lines) => + lines + .slice(0, index + 1) + .every((each) => each.startsWith(" ") || each === ""), + ) + .join("\n"); + const named = new Set([ + ...[...rule.matchAll(/chosen\(\[([^\]]*)\]\)/g)].flatMap((match) => + [...match[1]!.matchAll(/"([\w-]+)"/g)].map((name) => name[1]!), + ), + ...[...rule.matchAll(/\.select\.outputs\.([\w-]+)/g)].map((match) => match[1]!), + ]); + expect(named.size, job).toBeGreaterThan(0); + expect( + [...named].filter((name) => !outputs.some((output) => output.name === name)), + job, + ).toEqual([]); + // At run time: the same gate, all passing, with one more job in needs than it reads, + // and with one NEED_ variable for a job needs does not list. + const script = text.slice(text.indexOf(` - name: ${step}\n`)); + const body = script + .slice(script.indexOf(" run: |\n") + " run: |\n".length) + .split("\n") + .filter((line, index, lines) => + lines + .slice(0, index + 1) + .every((each) => each.startsWith(" ") || each === ""), + ) + .map((line) => line.slice(10)) + .join("\n"); + const passing = Object.fromEntries( + needed.map((name) => [ + name, + { + result: "success", + outputs: + name === "select" + ? { + collector: "true", + "local-shards": "[x]", + "self-host-shards": "[x]", + "self-host-inventory": "x", + "self-host-catalog": "x", + "cloud-runners": "[x]", + "release-local": "x", + "release-docker": "x", + } + : {}, + }, + ]), + ); + const run = (env: Readonly>) => + Effect.gen(function* () { + const child = yield* processes.spawn( + ChildProcess.make("bash", ["-e", "-c", body], { + env: { ...env, REPEAT_OF: "" }, + extendEnv: true, + }), + ); + const [log, exitCode] = yield* Effect.all( + [child.all.pipe(Stream.decodeText(), Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + return { log, exitCode: Number(exitCode) }; + }).pipe(Effect.scoped); + const all = gateEnvironment(workflow, job, step, passing); + expect((yield* run(all)).exitCode, job).toBe(0); + const added = yield* run( + gateEnvironment(workflow, job, step, { + ...passing, + "a-new-job": { result: "success", outputs: {} }, + }), + ); + expect(added.exitCode, job).toBe(1); + expect(added.log).toContain("every job in needs must have its NEED_ variable here"); + const { [`NEED_${needed.at(-1)!.replaceAll("-", "_")}`]: _dropped, ...missing } = all; + const forgotten = yield* run({ + ...missing, + [`NEED_${needed.at(-1)!.replaceAll("-", "_")}`]: "", + }); + expect(forgotten.exitCode, job).toBe(1); + const stray = yield* run({ ...all, NEED_not_needed: "" }); + expect(stray.exitCode, job).toBe(1); + } + }), + ); }); diff --git a/e2e/tests/cli-tool-commands.spec.ts b/e2e/tests/cli-tool-commands.spec.ts index 9f93a76301..3b5597ffc2 100644 --- a/e2e/tests/cli-tool-commands.spec.ts +++ b/e2e/tests/cli-tool-commands.spec.ts @@ -19,7 +19,15 @@ import { Target } from "../support/platform.ts"; import { serverControl } from "../support/server-control.ts"; import { stdioMcp } from "../support/stdio-mcp.ts"; -/** An account-free app whose `review` waits for approval and whose `echo` answers at once. */ +// A 1x1 PNG, and the head and tail of a PDF, as tools return them. +const png = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNQWPX/PwAFgALJpImt0wAAAABJRU5ErkJggg=="; +const pdf = Buffer.from("%PDF-1.4\n%\u00e2\u00e3\u00cf\u00d3\n%%EOF\n", "latin1"); + +/** + * An account-free app whose `review` waits for approval, whose `echo` answers at once and whose + * `report` returns an image and a PDF as MCP content blocks. + */ const reviewApp = (receipt: string) => [ { path: "index.ts", @@ -30,6 +38,10 @@ export default defineApp({}, async () => ({ tools: router({ review: mutation({ description: "Review after approval", input: object({}), approval: always() }, async () => ({ reviewed: ${JSON.stringify(receipt)} })), echo: query({ description: "Echo a word", input: object({ word: string() }) }, async (_, { word }) => ({ echoed: word })), + report: query({ description: "Return a chart and a report", input: object({}) }, async () => ({ + chart: { type: "image", mimeType: "image/png", data: ${JSON.stringify(png)} }, + report: { type: "resource", resource: { uri: "file:///reports/Q3%20summary.pdf", mimeType: "application/pdf", blob: ${JSON.stringify(pdf.toString("base64"))} } }, + })), }), })); `, @@ -273,6 +285,46 @@ layer(TestLive, { excludeTestServices: true })("CLI tool commands", (it) => { expect( json(Completed, yield* cli.ok(["execute", "--code", "return 'inline'"])).execution.value, ).toBe("inline"); + + // The host sends a result's images and files as MCP content blocks, outside its JSON. + // The CLI writes each to a file, in a new temporary directory unless --files names one, + // and prints its path beside its contentIndex. + const report = pathOf("report"); + const Saved = Schema.Struct({ contentIndex: Schema.Number, path: Schema.String }); + const saved = (stdout: string) => { + const value = json( + Outcome, + JSON.stringify(json(Completed, stdout).execution.value), + ).value; + return Schema.decodeUnknownSync(Schema.Struct({ chart: Saved, report: Saved }))(value); + }; + const temporary = saved(yield* cli.ok(["call", report])); + expect(cli.path.dirname(temporary.chart.path)).toBe( + cli.path.dirname(temporary.report.path), + ); + expect(cli.path.basename(temporary.chart.path)).toBe( + `${temporary.chart.contentIndex}-content.png`, + ); + expect(cli.path.basename(temporary.report.path)).toBe( + `${temporary.report.contentIndex}-Q3-summary.pdf`, + ); + expect(Buffer.from(yield* cli.fs.readFile(temporary.chart.path)).toString("base64")).toBe( + png, + ); + expect(Buffer.from(yield* cli.fs.readFile(temporary.report.path))).toEqual(pdf); + yield* cli.fs.remove(cli.path.dirname(temporary.report.path), { recursive: true }); + const directory = cli.path.join(cli.home, "attachments"); + const chosen = saved( + yield* cli.ok([ + "execute", + "--files", + directory, + "--code", + `return { path: "report", ok: true, value: await ${report}({}) };`, + ]), + ); + expect(cli.path.dirname(chosen.report.path)).toBe(directory); + expect(Buffer.from(yield* cli.fs.readFile(chosen.report.path))).toEqual(pdf); }), ), ); diff --git a/e2e/tests/cloud-app-query-auth.spec.ts b/e2e/tests/cloud-app-query-auth.spec.ts index dcd790db2f..85bf93c9bf 100644 --- a/e2e/tests/cloud-app-query-auth.spec.ts +++ b/e2e/tests/cloud-app-query-auth.spec.ts @@ -123,7 +123,9 @@ export default defineApp({}, async () => ({ tools: router({ result.data.some(({ span }) => span.operationName === "runtime.cloud.query") && result.data.filter(({ span }) => span.operationName === "auth.sql.timing").length >= 3 ? Effect.succeed(result.data) - : Effect.fail(new Error("The completed app query trace has not reached Motel")), + : Effect.fail( + new Error("The completed app query trace has not reached the collector"), + ), ), Effect.retry({ schedule: Schedule.spaced("1 second"), times: 40 }), ); diff --git a/e2e/tests/cloud-concurrent-recheck.spec.ts b/e2e/tests/cloud-concurrent-recheck.spec.ts new file mode 100644 index 0000000000..4710995039 --- /dev/null +++ b/e2e/tests/cloud-concurrent-recheck.spec.ts @@ -0,0 +1,368 @@ +/** + * MCP tool calls made at once in one `execute`, whose key checks contend: a personal access token's + * check records the key's use at most once a minute, so when several calls check a stale key at + * once, those that wait behind the one recording it read the key again, after the batch their + * first check went out in, while that batch's `COMMIT` may still be unanswered. Through Hyperdrive + * a statement written then was never answered, and the call and every call behind it waited for + * the execute's five-minute deadline. Every round must answer within 45 seconds, on a session + * object whose pool is still opening its connections (a new key reaches a new object) and on one + * whose connections are open, and its calls must have read their key again. + * + * The key row's lock is held through the runner's fixture process (`/api-key-use`), which has the + * Cloud's database locally and on a deployed stage alike, for the scenario's own keys only. + */ +import { expect, layer } from "@effect/vitest"; +import { Clock, Effect, Fiber, Schedule, Schema } from "effect"; +import { randomUUID } from "node:crypto"; +import { Actors } from "../support/actors.ts"; +import { Api, body } from "../support/api.ts"; +import { appsManifest } from "../support/apps-release.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { App, Resource } from "../support/contracts.ts"; +import { Evidence, Telemetry } from "../support/evidence.ts"; +import { McpClient } from "../support/mcp-client.ts"; +import { fixtureRequest } from "../sdk/fixtures.ts"; +import { connectionsPath, linkedAccountIds } from "../support/app-accounts.ts"; +import { Target } from "../support/platform.ts"; +import { scenarios } from "../test-plan.ts"; + +const source = `import { defineApp, defineProvider, secrets, object, string, number, query, router } from "apps"; +const service = defineProvider({ name: "Concurrent recheck fixture", auth: { + key: secrets({ label: "API key", fields: object({ token: string() }) }) +} }); +export default defineApp({ user: { accounts: { service } } }, async ctx => ({ + tools: router({ + token: query({ input: object({}), output: string() }, async () => ctx.user.accounts.service.fields.token), + slow: query({ input: object({ ms: number() }), output: string() }, async (_ctx, { ms }) => { + await new Promise((resolve) => setTimeout(resolve, ms)); + return ctx.user.accounts.service.fields.token; + }), + }), +}));`; + +const Token = Schema.Struct({ key: Schema.RedactedFromValue(Schema.String), id: Schema.String }); +const Execution = Schema.Struct({ + status: Schema.Literal("completed"), + execution: Schema.Struct({ + ok: Schema.Boolean, + value: Schema.optional(Schema.Unknown), + error: Schema.optional(Schema.Unknown), + }), +}); + +/** Calls each `execute` makes at once, as the production programs that hung did. */ +const parallel = 6; +/** + * Key checks waiting on the hold that make it certain one is a tool call's. One request checks its + * key in one statement at a time, so a request's own checks are at most one waiter; two at once + * include a tool call's check, which the commit then makes contended (`bearer-access.ts`). + */ +const waitersForCertainty = 2; +/** Rounds, each with the key's last use cleared first. */ +const rounds = 6; + +layer(HostedLive, { excludeTestServices: true })("Cloud concurrent recheck", (it) => { + it.effect( + scenarios.cloudMcpConcurrentRecheck.title, + (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors, + mcp = yield* McpClient, + evidence = yield* Evidence, + telemetry = yield* Telemetry; + const prefix = `/api/organizations/${actors.organization.id}`; + const pat = yield* body( + Token, + yield* api.request(actors.owner, "POST", "/api/auth/api-key/create", { + name: "Concurrent recheck", + }), + ); + yield* Effect.addFinalizer(() => + api + .request(actors.owner, "POST", "/api/auth/api-key/delete", { keyId: pat.id }) + .pipe(Effect.orDie), + ); + const deployed = yield* api.request(actors.owner, "POST", `${prefix}/apps/deploy`, { + name: `Concurrent recheck ${randomUUID().slice(0, 8)}`, + files: [{ path: "index.ts", content: source }, appsManifest], + }); + expect(deployed.status).toBe(200); + const app = yield* body(App, deployed); + const path = `${prefix}/apps/${app.id}`; + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", path).pipe(Effect.orDie), + ); + const pending = yield* body( + Resource, + yield* api.request(actors.owner, "POST", connectionsPath(path, "user", "service"), {}), + ); + const account = (yield* body( + Resource, + yield* api.request(actors.owner, "POST", `${prefix}/connections/${pending.id}/submit`, { + method: "key", + label: "Synthetic", + fields: { token: "synthetic-recheck" }, + }), + )).id; + yield* Effect.addFinalizer(() => + api + .request(actors.owner, "DELETE", `${prefix}/accounts/${account}`) + .pipe(Effect.ignore), + ); + yield* linkedAccountIds(actors.owner, path).pipe( + Effect.filterOrFail((linked) => JSON.stringify(linked) === JSON.stringify([account])), + Effect.retry({ schedule: Schedule.spaced("100 millis"), times: 300 }), + ); + + const client = yield* mcp.connect(pat.key, "concurrent-recheck", { + organization: actors.organization.id, + }); + const tool = `tools[${JSON.stringify(app.slug)}].token`; + type Client = typeof client; + const executeOn = (client: Client, operation: string, code: string) => + client + .use(operation, (client, signal) => + client.callTool({ name: "execute", arguments: { code } }, undefined, { + signal, + timeout: 290_000, + }), + ) + .pipe( + Effect.flatMap((result) => + Schema.decodeUnknownEffect(Execution)(result.structuredContent), + ), + Effect.map(({ execution }) => execution), + ); + const execute = (operation: string, code: string) => executeOn(client, operation, code); + /** A session of its own, whose object's pool has no connection open yet. */ + // Warm the session: its first call reads the links in a round of its own. + expect(yield* execute("Warm the session", `return await ${tool}({})`)).toEqual({ + ok: true, + value: "synthetic-recheck", + }); + + /** + * What of each tool call of an execute's trace has ended, grouped by the call's span ID + * so that a call still running (its own span not ended) is seen too: its batch, the key + * reads under its recheck, and the call itself. + */ + const callsOf = (traceId: string, settled: boolean) => + telemetry.query(traceId).pipe( + Effect.flatMap((trace) => { + const byId = new Map(trace.data.map(({ span }) => [span.spanId, span])); + const calls = new Map< + string, + { + ended: boolean; + batches: number; + keyReads: number; + keyReadsMs: number[]; + } + >(); + const of = (id: string) => { + const found = calls.get(id) ?? { + ended: false, + batches: 0, + keyReads: 0, + keyReadsMs: [], + }; + calls.set(id, found); + return found; + }; + for (const { span } of trace.data) { + if (span.operationName === "mcp.tool.call") of(span.spanId).ended = true; + // A batch's span is the call's child; the recheck's statements are under its + // `auth.authenticate`, whose parent is the call. + if (span.operationName === "sql.pipeline" && span.parentSpanId !== null) + of(span.parentSpanId).batches++; + if ( + span.operationName === "sql.execute" && + String(span.tags["db.query.text"] ?? "").includes("from apikey where key") + ) { + const auth = byId.get(span.parentSpanId ?? ""); + if (auth?.operationName === "auth.authenticate" && auth.parentSpanId !== null) { + const call = of(auth.parentSpanId); + call.keyReads++; + call.keyReadsMs.push(Math.round(span.durationMs)); + } + } + } + const found = [...calls.values()].filter((call) => call.batches > 0 || call.ended); + return !settled || found.filter((call) => call.ended).length === parallel + 1 + ? Effect.succeed(found) + : Effect.fail(new Error(`${found.length} tool calls have reached the collector`)); + }), + Effect.retry({ schedule: Schedule.spaced("1 second"), times: 40 }), + ); + // The key row's lock is held through the runner's fixture process, which has the + // stage's database, locally and on a deployed stage alike; only the scenario's own keys. + const target = yield* Target; + if (target.fixtures === undefined || target.scenarioId === undefined) + return yield* Effect.die(new Error("The scenario needs the runner's fixture control")); + const fixtures = target.fixtures, + scenarioId = target.scenarioId; + const keyUse = ( + key: string, + action: "age" | "hold" | "release" | "blocked", + seconds?: number, + ) => + fixtureRequest(fixtures, "/api-key-use", { + id: scenarioId, + key, + action, + ...(seconds === undefined ? {} : { seconds }), + }).pipe(Effect.orDie); + /** The sessions waiting on the scenario's hold of the key row, directly or in its queue. */ + const blocked = (key: string) => + keyUse(key, "blocked").pipe( + Effect.flatMap( + Schema.decodeUnknownEffect( + Schema.Struct({ keyChecks: Schema.Number, others: Schema.Array(Schema.String) }), + ), + ), + Effect.orDie, + ); + const outcomes: Array> = []; + yield* evidence.step( + `${parallel} concurrent calls whose key checks contend all answer, on cold and warm pools`, + Effect.gen(function* () { + let roundKey = pat; + let roundClient = client; + for (let round = 0; round < rounds; round++) { + // Even rounds are cold: a new key reaches a new session object (an object is its + // bearer's), whose pool opens its connections while the calls arrive. Odd rounds + // reuse that session a few seconds later, its connections open. Each round's first + // call warms the session's call state, so the calls after it take the warm path. + const cold = round % 2 === 0; + if (cold) { + const created = yield* body( + Token, + yield* api.request(actors.owner, "POST", "/api/auth/api-key/create", { + name: `Concurrent recheck ${round}`, + }), + ); + yield* Effect.addFinalizer(() => + api + .request(actors.owner, "POST", "/api/auth/api-key/delete", { + keyId: created.id, + }) + .pipe(Effect.ignore), + ); + roundKey = created; + roundClient = yield* mcp.connect(created.key, `concurrent-recheck-${round}`, { + organization: actors.organization.id, + }); + } + const started = yield* Clock.currentTimeMillis; + // Used 52 seconds ago: the request's own checks and the round's first call find the + // use recorded, so they write nothing and pass the lock below; the calls made ten + // seconds later find it stale, and their checks wait for the lock to record it. + yield* keyUse(roundKey.id, "age", 52); + // The hold is taken before the calls and always released here, whatever fails + // meanwhile: a lock left held would stall the key's deletion until the fixture ends. + const { fiber, waiting, waiters } = yield* Effect.scoped( + Effect.gen(function* () { + yield* Effect.acquireRelease(keyUse(roundKey.id, "hold"), () => + keyUse(roundKey.id, "release").pipe(Effect.ignore), + ); + const fiber = yield* Effect.forkChild( + Effect.exit( + executeOn( + roundClient, + `Round ${round}: ${parallel} calls at once, ${cold ? "cold" : "warm"} pool`, + `await ${tool.replace(".token", ".slow")}({ ms: 10000 }); +return await Promise.all(Array.from({ length: ${parallel} }, () => ${tool}({})));`, + ).pipe(Effect.timeout("45 seconds")), + ), + ); + // Release only once tool calls' key checks wait on the hold. A check updates + // the key's last use only when its snapshot finds it stale; one that does waits + // for the hold, and when the hold commits a fresh use the check finds the row + // changed, so it is contended and its call reads the key again. + const seen = yield* blocked(roundKey.id).pipe( + Effect.filterOrFail((found) => found.keyChecks >= waitersForCertainty), + Effect.retry({ schedule: Schedule.spaced("50 millis"), times: 600 }), + Effect.option, + ); + const waiting = seen._tag === "Some" ? seen.value : yield* blocked(roundKey.id); + return { fiber, waiting: waiting.keyChecks, waiters: waiting.others }; + }), + ); + const result = yield* Fiber.join(fiber); + const ms = (yield* Clock.currentTimeMillis) - started; + const request = (yield* evidence.requests) + .filter((entry) => entry.path === "/mcp") + .at(-1); + const answered = result._tag === "Success"; + const calls = + request === undefined + ? undefined + : yield* Effect.option( + callsOf(request.traceId, answered).pipe( + // A call that did not end has no span of its own: record what of it ended. + Effect.catch(() => callsOf(request.traceId, false)), + ), + ); + const objectId = + request === undefined + ? undefined + : yield* telemetry.query(request.traceId).pipe( + Effect.map((trace) => + trace.data + .map(({ span }) => span.tags["executor.mcp.object_id"]) + .find((id) => id !== undefined), + ), + Effect.orElseSucceed(() => undefined), + ); + outcomes.push({ + round, + objectId, + cold, + ms, + waiting, + waiters, + calls: calls?._tag === "Some" ? calls.value : "unavailable", + reads: + calls?._tag === "Some" + ? calls.value.map((call) => call.keyReads) + : "unavailable", + outcome: + result._tag === "Success" ? JSON.stringify(result.value) : String(result.cause), + }); + yield* evidence.json("concurrent-recheck-rounds.json", outcomes); + } + expect( + outcomes + .filter((entry) => Number(entry.waiting) < waitersForCertainty) + .map((entry) => entry.round), + `Rounds whose set-up failed: fewer than ${waitersForCertainty} key checks waited on the key row's lock at once within 30 seconds`, + ).toEqual([]); + expect( + outcomes + .filter((entry) => String(entry.outcome).includes("TimeoutError")) + .map((entry) => entry.round), + "Rounds whose execute did not answer within 45 seconds", + ).toEqual([]); + expect( + outcomes + .filter( + (entry) => + typeof entry.reads !== "object" || + !(entry.reads as number[]).some((n) => n >= 2), + ) + .map((entry) => entry.round), + "Rounds in which no call read its key again", + ).toEqual([]); + // A round can also fail fast: a key check that is contended twice, or reads its + // key under the scenario's lock, answers 503 or 401. That is not this fault and is + // not judged here; each round's outcome is in the evidence. + }), + ); + }).pipe(Effect.provide(McpClient.layer)), + ), + { timeout: 600_000 }, + ); +}); diff --git a/e2e/tests/cloud-database-placement.spec.ts b/e2e/tests/cloud-database-placement.spec.ts index 7b88126171..682a6bbe9c 100644 --- a/e2e/tests/cloud-database-placement.spec.ts +++ b/e2e/tests/cloud-database-placement.spec.ts @@ -57,7 +57,7 @@ const ancestry = (spans: Spans, spanId: string) => { /** * A trace, once the span `select` finds and its whole ancestry up to the trace's root have reached - * Motel. A parent ends after its children, so it leaves in the same export or a later one, and a + * the collector. A parent ends after its children, so it leaves in the same export or a later one, and a * refused export is sent again after later ones. */ const deliveredAncestry = (traceId: string, select: (span: Spans[number]["span"]) => boolean) => @@ -66,11 +66,11 @@ const deliveredAncestry = (traceId: string, select: (span: Spans[number]["span"] Effect.flatMap((trace) => { const found = trace.data.find(({ span }) => select(span)); const chain = found === undefined ? [] : ancestry(trace.data, found.span.spanId); - // Motel answers a span whose parent has not arrived with a placeholder root in its place. + // The collector answers a span whose parent has not arrived with a placeholder root in its place. return chain.at(-1)?.parentSpanId === null && !chain.some((span) => span.operationName.startsWith("[missing parent")) ? Effect.succeed({ trace: trace.data, chain }) - : Effect.fail(new Error("The span or its ancestors have not all reached Motel")); + : Effect.fail(new Error("The span or its ancestors have not all reached the collector")); }), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), ); @@ -223,7 +223,7 @@ const toolCallStatements = Effect.gen(function* () { .filter(({ runtime }) => !runtime); return found.some(({ text }) => text.includes("from apikey where key")) ? Effect.succeed(found) - : Effect.fail(new Error("The tool call's statements have not reached Motel")); + : Effect.fail(new Error("The tool call's statements have not reached the collector")); }), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), ); @@ -232,7 +232,7 @@ const toolCallStatements = Effect.gen(function* () { const objectConnections = (owner: string, options: Parameters[1] = {}) => Effect.map(objectDatabase(owner, options), ({ connections }) => connections); -/** The session object behind the last MCP request, once its spans have reached Motel. */ +/** The session object behind the last MCP request, once its spans have reached the collector. */ const lastSessionObject = Effect.gen(function* () { const evidence = yield* Evidence, telemetry = yield* Telemetry; @@ -244,7 +244,7 @@ const lastSessionObject = Effect.gen(function* () { .map(({ span }) => span.tags["executor.mcp.object_id"]) .find((id) => id !== undefined); return id === undefined - ? Effect.fail(new Error("The session object's spans have not reached Motel")) + ? Effect.fail(new Error("The session object's spans have not reached the collector")) : Effect.succeed(`mcp ${String(id)}`); }), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 40 }), @@ -376,7 +376,7 @@ layer(HostedLive, { excludeTestServices: true })("Cloud database placement", (it ), Effect.flatMap((found) => found[0] === undefined - ? Effect.fail(new Error("No dispatched provisioning job has reached Motel")) + ? Effect.fail(new Error("No dispatched provisioning job has reached the collector")) : Effect.succeed(found[0]), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), @@ -475,7 +475,7 @@ return out;`, expect(completed.execution.value).toEqual(["0", "1", "2", "3", "4", "5"]); return (yield* evidence.requests).slice(seen).map((request) => request.traceId); }); - // The session object's own spans, once each one a check reads has reached Motel. A + // The session object's own spans, once each one a check reads has reached the collector. A // request's spans leave the object in several exports, one each second while it runs and // one when it ends, and a refused export is sent again later, so they arrive in any order. type Counts = Record<"operations" | "databases" | "connects" | "windows", number>; @@ -515,7 +515,7 @@ return out;`, }) : Effect.fail( new Error( - `The session's spans have not reached Motel: ${JSON.stringify(counts)}`, + `The session's spans have not reached the collector: ${JSON.stringify(counts)}`, ), ); }), @@ -617,7 +617,7 @@ return out;`, .request(actors.owner, "POST", "/api/auth/api-key/delete", { keyId: key.id }) .pipe(Effect.orDie), ); - // The session object's own spans for some requests, once `ready` finds them in Motel. + // The session object's own spans for some requests, once `ready` finds them in the collector. const objectSpans = ( traces: ReadonlyArray, ready: (spans: ReadonlyArray) => boolean, @@ -633,7 +633,7 @@ return out;`, Effect.flatMap((spans) => ready(spans) ? Effect.succeed(spans) - : Effect.fail(new Error("The session's spans have not reached Motel")), + : Effect.fail(new Error("The session's spans have not reached the collector")), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 40 }), ); @@ -780,7 +780,9 @@ return out;`, ), Effect.flatMap((found) => found[0] === undefined - ? Effect.fail(new Error("No cron wake of the coordinator has reached Motel")) + ? Effect.fail( + new Error("No cron wake of the coordinator has reached the collector"), + ) : Effect.succeed(found[0]), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), @@ -825,7 +827,7 @@ return out;`, yield* evidence.json("scheduled-runs.json", runs); expect(runs.map((run) => run.status)).toEqual(["succeeded", "succeeded", "succeeded"]); - // Motel matches operation names by prefix, so `schedule.retire` would also find + // The collector matches operation names by the start of their words, so `schedule.retire` would also find // `schedule.retired`. const exactly = (operation: string, found: typeof SpanQuery.Type) => found.data.filter(({ span }) => span.operationName === operation); @@ -863,7 +865,7 @@ return out;`, found.retirements.length === 0 || found.runs.length < 3 || found.dispatches.length === 0 - ? Effect.fail(new Error("The coordinator's spans have not reached Motel")) + ? Effect.fail(new Error("The coordinator's spans have not reached the collector")) : Effect.succeed(found), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), @@ -919,7 +921,9 @@ return out;`, Effect.map((found) => found.filter((beat) => !known.has(beat.trace))), Effect.flatMap((found) => found.length < 2 - ? Effect.fail(new Error(`${found.length} of 2 heartbeats have reached Motel`)) + ? Effect.fail( + new Error(`${found.length} of 2 heartbeats have reached the collector`), + ) : Effect.succeed(found), ), Effect.retry({ schedule: Schedule.spaced("1 second"), times: 150 }), @@ -1490,6 +1494,26 @@ return listed;`; }, }); + // A leaf that reads again once its first statement answers, as a bearer recheck whose key + // was contended does, writes nothing while its batch's `COMMIT` is unanswered: through + // Hyperdrive such a statement is never answered, and the call waits until its deadline. + // The read goes out, and answers, once the `COMMIT` has: also when a statement inside the + // batch failed, so its `COMMIT` is answered as a rollback. + const later = yield* step( + "A batch leaf's later statement waits for its batch's COMMIT to answer before it is written", + "commit-unanswered", + ); + expect(later).toEqual({ leaves: ["ok", "ok", "ok"], behindCommit: 0, later: "answered" }); + const rolledBack = yield* step( + "A batch leaf's later statement waits for a COMMIT answered as a rollback, then is written", + "commit-rollback", + ); + expect(rolledBack).toEqual({ + leaves: ["ok", "failed", "failed"], + behindCommit: 0, + later: "answered", + }); + // A connect attempt that reaches its timeout, and one whose caller gives up while it // waits, close their sockets: a server that accepts and never answers sees each closed. const abandoned = yield* step( diff --git a/e2e/tests/cloud-export-hang.spec.ts b/e2e/tests/cloud-export-hang.spec.ts index d6d75d49d8..887dcb4022 100644 --- a/e2e/tests/cloud-export-hang.spec.ts +++ b/e2e/tests/cloud-export-hang.spec.ts @@ -270,7 +270,7 @@ layer(HostedLive, { excludeTestServices: true })("Cloud export hang", (it) => { yield* telemetry.query(trace).pipe( Effect.filterOrFail( (found) => found.data.some(({ span }) => span.operationName === "mcp.tool.call"), - () => new Error("The program's spans have not reached Motel"), + () => new Error("The program's spans have not reached the collector"), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), ); @@ -286,8 +286,8 @@ layer(HostedLive, { excludeTestServices: true })("Cloud export hang", (it) => { // An export's retries, the waits Retry-After asks for included, end within the time the // runtime keeps an event's work after its response (`exportBudget` in the telemetry // package): the spans of a program whose exports could not get out at its close still - // arrive within it, measured from the program's answer, when its event's export starts. Motel - // is read every 500 ms, so a span is seen up to that late. + // arrive within it, measured from the program's answer, when its event's export starts. The + // collector is read every 500 ms, so a span is seen up to that late. const exportBudgetMs = 27_000; const arrives = (label: string, trouble: Effect.Effect) => Effect.gen(function* () { @@ -301,7 +301,7 @@ layer(HostedLive, { excludeTestServices: true })("Cloud export hang", (it) => { yield* telemetry.query(run.trace).pipe( Effect.filterOrFail( (found) => found.data.some(({ span }) => span.operationName === "mcp.tool.call"), - () => new Error("The program's spans have not reached Motel"), + () => new Error("The program's spans have not reached the collector"), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 80 }), ); @@ -374,7 +374,7 @@ layer(HostedLive, { excludeTestServices: true })("Cloud export hang", (it) => { ), Effect.filterOrFail( (records) => records.length >= held.length, - () => new Error("The held calls' close reports have not reached Motel"), + () => new Error("The held calls' close reports have not reached the collector"), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), ); diff --git a/e2e/tests/cloud-job-outcomes.spec.ts b/e2e/tests/cloud-job-outcomes.spec.ts index 67d23abb0b..da0ed06102 100644 --- a/e2e/tests/cloud-job-outcomes.spec.ts +++ b/e2e/tests/cloud-job-outcomes.spec.ts @@ -68,7 +68,9 @@ const dispatched = ( }); const entry = found.data.find(({ traceId }) => !before.has(traceId)); if (entry === undefined) - return yield* Effect.fail(new Error(`No ${outcome} ${job} dispatch has reached Motel`)); + return yield* Effect.fail( + new Error(`No ${outcome} ${job} dispatch has reached the collector`), + ); const trace = yield* telemetry.query(entry.traceId); const dispatch = trace.data.find( ({ span }) => span.operationName === "job.dispatch" && span.tags["executor.job"] === job, diff --git a/e2e/tests/cloud-mcp-session-restore.spec.ts b/e2e/tests/cloud-mcp-session-restore.spec.ts index 0b8501e0d7..3b14bef28b 100644 --- a/e2e/tests/cloud-mcp-session-restore.spec.ts +++ b/e2e/tests/cloud-mcp-session-restore.spec.ts @@ -230,7 +230,7 @@ layer(HostedLive, { excludeTestServices: true })("Cloud MCP session restore", (i span.tags["http.response.status_code"] !== undefined, ) ? Effect.succeed(all) - : Effect.fail(new Error("The request's spans have not reached Motel")), + : Effect.fail(new Error("The request's spans have not reached the collector")), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 40 }), Effect.tap((all) => evidence.json(`trace-${traceId}.json`, all)), diff --git a/e2e/tests/cloud-mcp-session-timing.spec.ts b/e2e/tests/cloud-mcp-session-timing.spec.ts index 52063fe409..a847a8f6cd 100644 --- a/e2e/tests/cloud-mcp-session-timing.spec.ts +++ b/e2e/tests/cloud-mcp-session-timing.spec.ts @@ -205,7 +205,7 @@ layer(HostedLive, { excludeTestServices: true })("Cloud MCP session timing", (it Effect.flatMap((traces) => forwarded(traces, method)), Effect.flatMap((found) => found.length < count - ? Effect.fail(new Error("The requests' spans have not reached Motel")) + ? Effect.fail(new Error("The requests' spans have not reached the collector")) : Effect.succeed(found), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: attempts }), @@ -289,7 +289,7 @@ layer(HostedLive, { excludeTestServices: true })("Cloud MCP session timing", (it Effect.flatMap((spans) => spans.some((span) => span.operationName === "mcp.tool.call") ? Effect.succeed(spans) - : Effect.fail(new Error("The call's tool span has not reached Motel")), + : Effect.fail(new Error("The call's tool span has not reached the collector")), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 40 }), ); @@ -441,7 +441,9 @@ layer(HostedLive, { excludeTestServices: true })("Cloud MCP session timing", (it Effect.flatMap((result) => result.data.some(({ span }) => span.operationName === "mcp.session.request") ? Effect.void - : Effect.fail(new Error("The abandoned call's object span has not reached Motel")), + : Effect.fail( + new Error("The abandoned call's object span has not reached the collector"), + ), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 40 }), ); @@ -524,7 +526,7 @@ layer(HostedLive, { excludeTestServices: true })("Cloud MCP session timing", (it ), Effect.flatMap((span) => span === undefined - ? Effect.fail(new Error("The stream's forward has not reached Motel")) + ? Effect.fail(new Error("The stream's forward has not reached the collector")) : Effect.succeed(span), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), diff --git a/e2e/tests/cloud-mcp-stream-lifetime.spec.ts b/e2e/tests/cloud-mcp-stream-lifetime.spec.ts index f7fcbe5f69..00f1ca044e 100644 --- a/e2e/tests/cloud-mcp-stream-lifetime.spec.ts +++ b/e2e/tests/cloud-mcp-stream-lifetime.spec.ts @@ -130,7 +130,7 @@ layer(HostedLive, { excludeTestServices: true })("Cloud MCP stream lifetime", (i }; return found.forward && found.request && found[phase] ? Effect.succeed(found) - : Effect.fail(new Error("The request's spans have not reached Motel")); + : Effect.fail(new Error("The request's spans have not reached the collector")); }), Effect.retry({ schedule: Schedule.spaced("1 second"), times: 75 }), ); diff --git a/e2e/tests/cloud-postgres-volumes.spec.ts b/e2e/tests/cloud-postgres-volumes.spec.ts new file mode 100644 index 0000000000..13aba3e0b2 --- /dev/null +++ b/e2e/tests/cloud-postgres-volumes.spec.ts @@ -0,0 +1,216 @@ +/** + * A Cloud run removes another run's database volume only when that run is certainly gone + * (`volumeVerdict` in `e2e/support/cloud-postgres.ts`). Several runs share one Docker daemon, so + * every doubt keeps the volume. The verdict cases supply the owner's start time; the last cases + * read real processes' start times. + */ +import { expect, layer } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { Effect, Option } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { + processStarted, + volumeLabels, + volumeVerdict, + type SweptVolume, + type VolumeOwner, +} from "../support/cloud-postgres.ts"; + +const here: VolumeOwner = { + boot: "0f8b4d1e-2c3a-4b5d-8e6f-7a8b9c0d1e2f", + namespace: "linux/0f8b4d1e-2c3a-4b5d-8e6f-7a8b9c0d1e2f/pid:[4026531836]/time:[4026531834]", + pid: 1000, + started: "500", +}; +const now = Date.parse("2026-11-15T12:00:00Z"); +const hourAgo = now - 60 * 60_000; + +/** A volume another run in this namespace made an hour ago, unmounted, owned by pid 4242. */ +const volume = (change: Partial = {}): SweptVolume => ({ + name: "executor-e2e-0123456789abcdef-data", + labels: { + [volumeLabels.role]: "cloud-postgres-data", + [volumeLabels.boot]: here.boot, + [volumeLabels.namespace]: here.namespace, + [volumeLabels.pid]: "4242", + [volumeLabels.started]: "777", + }, + createdAt: hourAgo, + mountedBy: 0, + ...change, +}); + +/** pid 4242 as the operating system reports it; every other pid is gone. */ +const owner = + (answer: Option.Option | undefined) => + (pid: number): Effect.Effect | undefined> => + Effect.succeed(pid === 4242 ? answer : Option.none()); + +const verdict = (subject: SweptVolume, answer: Option.Option | undefined) => + volumeVerdict(subject, here, now, owner(answer)); + +const without = (label: string) => { + const labels: Record = { ...volume().labels }; + delete labels[label]; + return volume({ labels }); +}; + +layer(NodeServices.layer, { excludeTestServices: true })("Cloud test database volumes", (it) => { + it.effect("a volume whose owner runs, with the recorded start time, is kept", () => + Effect.gen(function* () { + expect(yield* verdict(volume(), Option.some("777"))).toEqual({ + remove: false, + why: "owner runs", + }); + }), + ); + + it.effect("a volume whose owner has exited is removed", () => + Effect.gen(function* () { + expect(yield* verdict(volume(), Option.none())).toEqual({ + remove: true, + why: "owner exited", + }); + }), + ); + + it.effect("a volume from another boot is removed, whatever its pid names now", () => + Effect.gen(function* () { + const old = volume({ + labels: { + ...volume().labels, + [volumeLabels.boot]: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee", + [volumeLabels.namespace]: + "linux/aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee/pid:[4026531836]/time:[4026531834]", + }, + }); + expect(yield* verdict(old, Option.some("777"))).toEqual({ + remove: true, + why: "another boot", + }); + }), + ); + + it.effect("a volume whose pid now names a process with another start time is removed", () => + Effect.gen(function* () { + expect(yield* verdict(volume(), Option.some("91234"))).toEqual({ + remove: true, + why: "owner's pid reused", + }); + }), + ); + + it.effect( + "a volume a stopped or running container mounts is kept, even with its owner gone", + () => + Effect.gen(function* () { + expect(yield* verdict(volume({ mountedBy: 1 }), Option.none())).toEqual({ + remove: false, + why: "mounted", + }); + const old = volume({ + mountedBy: 1, + labels: { + ...volume().labels, + [volumeLabels.boot]: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee", + }, + }); + expect(yield* verdict(old, Option.none())).toEqual({ remove: false, why: "mounted" }); + }), + ); + + it.effect("a volume missing any owner label, or with a malformed one, is kept", () => + Effect.gen(function* () { + for (const label of [ + volumeLabels.boot, + volumeLabels.namespace, + volumeLabels.pid, + volumeLabels.started, + ]) + expect(yield* verdict(without(label), Option.none()), label).toEqual({ + remove: false, + why: "incomplete labels", + }); + for (const [label, value] of [ + [volumeLabels.pid, "0"], + [volumeLabels.pid, "42x"], + [volumeLabels.boot, "not-a-boot"], + [volumeLabels.started, ""], + ] as const) + expect( + yield* verdict(volume({ labels: { ...volume().labels, [label]: value } }), Option.none()), + `${label}=${value}`, + ).toEqual({ remove: false, why: "incomplete labels" }); + }), + ); + + it.effect("a volume this harness did not label is kept", () => + Effect.gen(function* () { + // An anonymous volume, a hand-run database's, the development database's. + for (const labels of [ + { "com.docker.volume.anonymous": "" }, + {}, + { "com.docker.compose.project": "executor" }, + { ...volume().labels, [volumeLabels.role]: "cloud-postgres" }, + ]) + expect(yield* verdict(volume({ labels }), Option.none())).toEqual({ + remove: false, + why: "not ours", + }); + }), + ); + + it.effect("a volume made in the last five minutes is kept, before its container mounts it", () => + Effect.gen(function* () { + expect(yield* verdict(volume({ createdAt: now - 60_000 }), Option.none())).toEqual({ + remove: false, + why: "young", + }); + }), + ); + + it.effect("a volume from another process namespace of this boot is kept", () => + Effect.gen(function* () { + const other = volume({ + labels: { + ...volume().labels, + [volumeLabels.namespace]: `linux/${here.boot}/pid:[4026532999]/time:[4026531834]`, + }, + }); + expect(yield* verdict(other, Option.none())).toEqual({ + remove: false, + why: "another process namespace", + }); + }), + ); + + it.effect("a volume whose owner's start time cannot be read is kept", () => + Effect.gen(function* () { + expect(yield* verdict(volume(), undefined)).toEqual({ remove: false, why: "owner unknown" }); + }), + ); + + it.effect.skipIf(process.platform !== "linux" && process.platform !== "darwin")( + "a real process's start time reads the same twice, and as none once it has exited", + () => + Effect.gen(function* () { + const own = yield* processStarted(process.pid); + expect(own).toEqual(yield* processStarted(process.pid)); + expect(own !== undefined && Option.isSome(own)).toBe(true); + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* processes.spawn( + ChildProcess.make(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + stdout: "ignore", + stderr: "ignore", + }), + ); + const pid = Number(child.pid); + const started = yield* processStarted(pid); + expect(started !== undefined && Option.isSome(started)).toBe(true); + expect(started).not.toEqual(own); + yield* child.kill(); + yield* child.exitCode.pipe(Effect.ignore); + expect(yield* processStarted(pid)).toEqual(Option.none()); + }).pipe(Effect.scoped), + ); +}); diff --git a/e2e/tests/cloud-telemetry-idle-flush.spec.ts b/e2e/tests/cloud-telemetry-idle-flush.spec.ts index 356f59f6ef..caf35a1696 100644 --- a/e2e/tests/cloud-telemetry-idle-flush.spec.ts +++ b/e2e/tests/cloud-telemetry-idle-flush.spec.ts @@ -75,7 +75,7 @@ const session = (name: string) => return { raw }; }); -/** A trace's span named `name`, once Motel has it. Reading Motel sends the server nothing. */ +/** A trace's span named `name`, once the collector has it. Reading it sends the server nothing. */ const delivered = (traceId: string, name: string, attempts: number) => Effect.gen(function* () { const telemetry = yield* Telemetry; @@ -84,7 +84,7 @@ const delivered = (traceId: string, name: string, attempts: number) => Effect.flatMap((spans) => { const found = spans.find((span) => span.operationName === name); return found === undefined - ? Effect.fail(new Error(`${name} has not reached Motel`)) + ? Effect.fail(new Error(`${name} has not reached the collector`)) : Effect.succeed(found); }), Effect.retry({ schedule: Schedule.spaced("250 millis"), times: attempts }), diff --git a/e2e/tests/cloud-tool-call-batch.spec.ts b/e2e/tests/cloud-tool-call-batch.spec.ts index c7950229ba..7ec707e57e 100644 --- a/e2e/tests/cloud-tool-call-batch.spec.ts +++ b/e2e/tests/cloud-tool-call-batch.spec.ts @@ -258,7 +258,9 @@ layer(HostedLive, { excludeTestServices: true })("Cloud tool call batch", (it) = .map(({ span }) => span.tags["executor.mcp.object_id"]) .find((id) => id !== undefined), }) - : Effect.fail(new Error("The tool calls' spans have not all reached Motel")); + : Effect.fail( + new Error("The tool calls' spans have not all reached the collector"), + ); }), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), ); @@ -588,14 +590,14 @@ try { }), Effect.filterOrFail( done, - () => new Error("The slow log writes have not reached Motel"), + () => new Error("The slow log writes have not reached the collector"), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), ); /** * Wait for the app's own spans, which the app runtime forwards from the app's isolate - * once its call ends, to reach Motel in `traces`; return their operations. + * once its call ends, to reach the collector in `traces`; return their operations. */ const appSpans = ( traces: ReadonlyArray, @@ -610,13 +612,13 @@ try { ), Effect.filterOrFail( (names) => needed.every((name) => names.includes(name)), - () => new Error("The app's forwarded spans have not reached Motel"), + () => new Error("The app's forwarded spans have not reached the collector"), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), ); yield* evidence.step( - "A log write that ends long after its call answered still reaches Motel", + "A log write that ends long after its call answered still reaches the collector", Effect.gen(function* () { yield* slowLog("insert"); const started = performance.now(); @@ -632,7 +634,7 @@ try { ); yield* evidence.step( - "A call that waits for input answered by a second request does not wait for its slow log write, which still reaches Motel", + "A call that waits for input answered by a second request does not wait for its slow log write, which still reaches the collector", Effect.gen(function* () { // The call ends, and writes its row, while the second request is served. yield* slowLog("insert"); diff --git a/e2e/tests/collector-build.spec.ts b/e2e/tests/collector-build.spec.ts new file mode 100644 index 0000000000..7c0518c3f5 --- /dev/null +++ b/e2e/tests/collector-build.spec.ts @@ -0,0 +1,913 @@ +/** + * `packages/telemetry/scripts/build-collector.ts build` and its shared store. These cases copy the + * script into a temporary git repository shaped like the crate, put fake `rustup`, `rustc`, + * `cargo`, `cc`, `ld` and `ldd` first on PATH, and give the script its own cache directory, as + * `prepare-cache.spec.ts` fakes e2e:prepare's builds. They read what the script compiled, copied, + * kept and removed. + */ +import { expect, layer } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { Effect, FileSystem, Path, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +const triple = "x86_64-unknown-linux-gnu"; +const output = "packages/telemetry/dist/collector"; + +/** + * The fake tools. `cargo build` writes an executable naming the build's number, and appends to + * `FAKE_STATE/ran`; the rest print fixed versions, as the identity reads them. Files in + * `FAKE_STATE` steer `cargo build`: `slow` makes it take a second, `fail-once` makes the next build + * fail after a second, `die-once` makes the next one kill the build script that ran it, and + * `vanish` leaves a temporary file in the target directory, as rustc does. The script runs its + * tools with only the variables that find them, so the fakes' own paths are written into them + * (`FAKE_STATE`, `FAKE_SYSROOT`, `FAKE_METADATA` are replaced when the repository is made). + */ +const tools: Record = { + rustup: `#!/bin/sh +case "$1 $2" in + "--version "*) echo "rustup 1.29.3 (fake)";; + "toolchain list") echo "1.100.0-${triple}";; + "target list") echo "${triple}";; +esac +`, + rustc: `#!/bin/sh +case "$1" in + -vV) printf 'rustc 1.100.0 (fake)\\nhost: ${triple}\\n';; + --print) echo "FAKE_SYSROOT";; +esac +`, + cargo: `#!/bin/sh +case "$1" in + -vV) echo "cargo 1.100.0 (fake)";; + fetch) echo "proxy=\${HTTPS_PROXY:-unset} lower=\${https_proxy:-unset} cainfo=\${CARGO_HTTP_CAINFO:-unset} cert=\${SSL_CERT_FILE:-unset} make=\${MAKEFLAGS:-unset} cargomake=\${CARGO_MAKEFLAGS:-unset} backtrace=\${RUST_BACKTRACE:-unset} log=\${RUST_LOG:-unset}" >> "FAKE_STATE/fetch-env";; + metadata) cat "FAKE_METADATA";; + build) + echo "proxy=\${HTTPS_PROXY:-unset} lower=\${https_proxy:-unset} cainfo=\${CARGO_HTTP_CAINFO:-unset} cert=\${SSL_CERT_FILE:-unset} make=\${MAKEFLAGS:-unset} cargomake=\${CARGO_MAKEFLAGS:-unset} backtrace=\${RUST_BACKTRACE:-unset} log=\${RUST_LOG:-unset} args=$*" >> "FAKE_STATE/build-env" + target="" + while [ $# -gt 0 ]; do [ "$1" = --target-dir ] && target="$2"; shift; done + if [ -f "FAKE_STATE/fail-once" ]; then rm "FAKE_STATE/fail-once"; sleep 1; exit 1; fi + if [ -f "FAKE_STATE/die-once" ]; then rm "FAKE_STATE/die-once"; sleep 1; kill -9 $PPID; exit 1; fi + started=$(date +%s.%N) + [ -f "FAKE_STATE/slow" ] && sleep 1 + [ -f "FAKE_STATE/vanish" ] && mkdir -p "$target/.temp-archive" && echo a > "$target/.temp-archive/tmp.a" + echo build >> "FAKE_STATE/ran" + echo "$started $(date +%s.%N)" >> "FAKE_STATE/spans" + echo "\${CARGO_BUILD_JOBS:-unset}" >> "FAKE_STATE/jobs" + mkdir -p "$target/${triple}/release" + printf 'collector build %s\\n' "$(wc -l < "FAKE_STATE/ran")" > "$target/${triple}/release/executor-collector";; +esac +`, + cc: `#!/bin/sh +echo "cc (fake) 13" +`, + ld: `#!/bin/sh +echo "GNU ld (fake) 2.42" +`, + ldd: `#!/bin/sh +echo "ldd (fake) 2.39" +`, +}; + +/** A repository with the crate's shape, the real script, the fakes, a cache and a state file. */ +const repository = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "executor-collector-build-" }); + const repo = path.join(root, "repo"); + const crate = path.join(repo, "packages/telemetry/collector"); + yield* fs.makeDirectory(path.join(crate, "src"), { recursive: true }); + yield* fs.writeFileString(path.join(crate, "rust-toolchain.toml"), 'channel = "1.100.0"\n'); + yield* fs.writeFileString(path.join(crate, "Cargo.toml"), '[package]\nname = "fake"\n'); + yield* fs.writeFileString(path.join(crate, "NOTICE"), "notice\n"); + yield* fs.writeFileString(path.join(crate, "src/main.rs"), "fn main() {}\n"); + yield* fs.makeDirectory(path.join(repo, "packages/telemetry/scripts"), { recursive: true }); + yield* fs.copyFile( + "packages/telemetry/scripts/build-collector.ts", + path.join(repo, "packages/telemetry/scripts/build-collector.ts"), + ); + yield* processes.exitCode(ChildProcess.make("git", ["init", "--quiet"], { cwd: repo })); + const bin = path.join(root, "bin"); + yield* fs.makeDirectory(bin); + const sysroot = path.join(root, "sysroot"); + const metadata = path.join(root, "metadata.json"); + const state = path.join(root, "state"); + for (const [name, text] of Object.entries(tools)) { + yield* fs.writeFileString( + path.join(bin, name), + text + .replaceAll("FAKE_STATE", state) + .replaceAll("FAKE_SYSROOT", sysroot) + .replaceAll("FAKE_METADATA", metadata), + ); + yield* fs.chmod(path.join(bin, name), 0o755); + } + yield* fs.makeDirectory(path.join(sysroot, "share/doc/rust"), { recursive: true }); + yield* fs.writeFileString(path.join(sysroot, "share/doc/rust/COPYRIGHT-library.html"), "©\n"); + const id = "path+file:///fake#0.1.0"; + yield* fs.writeFileString( + metadata, + JSON.stringify({ + packages: [ + { + id, + name: "fake", + version: "0.1.0", + manifest_path: path.join(crate, "Cargo.toml"), + license_file: null, + targets: [{ kind: ["bin"] }], + }, + ], + resolve: { root: id, nodes: [{ id, deps: [] }] }, + }), + ); + yield* fs.makeDirectory(state); + yield* fs.makeDirectory(path.join(root, "home")); + return { root, repo, bin, sysroot, metadata, state, cache: path.join(root, "cache") }; +}); +type Repository = Effect.Success; + +/** Runs `build-collector.ts build` in `input.repo`; its exit code, output and the builds it ran. */ +const build = ( + input: Repository, + preload?: string, + repo: string = input.repo, + extra: Record = {}, + flags: ReadonlyArray = [], +) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const ranFile = path.join(input.state, "ran"); + const before = (yield* fs.exists(ranFile)) + ? (yield* fs.readFileString(ranFile)).split("\n").filter(Boolean).length + : 0; + const child = yield* processes.spawn( + ChildProcess.make( + "node", + [ + ...(preload === undefined ? [] : ["--import", preload]), + "packages/telemetry/scripts/build-collector.ts", + "--target", + "linux-x64", + ...flags, + "build", + ], + { + cwd: repo, + env: { + PATH: `${input.bin}:${process.env.PATH ?? ""}`, + HOME: path.join(input.root, "home"), + CARGO_HOME: path.join(input.root, "home/.cargo"), + RUSTUP_HOME: path.join(input.root, "home/.rustup"), + EXECUTOR_COLLECTOR_CACHE: input.cache, + FAKE_STATE: input.state, + FAKE_SYSROOT: input.sysroot, + FAKE_METADATA: input.metadata, + CI: "", + ...extra, + }, + extendEnv: false, + }, + ), + ); + const [log, exitCode] = yield* Effect.all( + [ + child.all.pipe(Stream.decodeText(), Stream.mkString), + // A build killed by a signal has no exit code: -1. + child.exitCode.pipe(Effect.orElseSucceed(() => -1)), + ], + { concurrency: "unbounded" }, + ); + const after = (yield* fs.exists(ranFile)) + ? (yield* fs.readFileString(ranFile)).split("\n").filter(Boolean).length + : 0; + return { exitCode, log, compiled: after - before }; + }).pipe(Effect.scoped); + +/** The store's entries for linux-x64, partial copies included. */ +const storeEntries = (input: Repository) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = path.join(input.cache, "store/linux-x64"); + return (yield* fs.exists(directory)) ? (yield* fs.readDirectory(directory)).sort() : []; + }); + +const read = (file: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return yield* fs.readFileString(file); + }); + +/** Copies of `input.repo`, each its own workspace, sharing its cache, fakes and state. */ +const workspaces = (input: Repository, count: number) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const made: string[] = []; + for (let index = 0; index < count; index++) { + const copy = path.join(input.root, `workspace-${index}`); + yield* fs.copy(input.repo, copy); + made.push(copy); + } + return made; + }); + +/** Every `cargo build` the fakes ran, in all workspaces. */ +const compiles = (input: Repository) => + read(`${input.state}/ran`).pipe( + Effect.map((text) => text.split("\n").filter(Boolean).length), + Effect.orElseSucceed(() => 0), + ); + +const locks = (input: Repository) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = `${input.cache}/locks/linux-x64`; + return (yield* fs.exists(directory)) ? yield* fs.readDirectory(directory) : []; + }); + +/** Waits until a build holds the identity's lock. */ +const lockTaken = (input: Repository) => + Effect.gen(function* () { + for (let tries = 0; tries < 500; tries++) { + if ((yield* locks(input)).some((name) => name.endsWith(".lock"))) return; + yield* Effect.sleep("20 millis"); + } + return yield* Effect.die(new Error("no build took the lock within 10 s")); + }); + +const flag = (input: Repository, name: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString(`${input.state}/${name}`, ""); + }); + +// The real clock: the cases that run builds at once wait for one another in real time. +const suite = layer(NodeServices.layer, { excludeTestServices: true }); + +suite("The collector's build and its store", (it) => { + it.effect("a damaged executable is never taken as up to date, in a workspace or the store", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const input = yield* repository; + const first = yield* build(input); + expect(first.exitCode, first.log).toBe(0); + expect(first.compiled).toBe(1); + const [key] = yield* storeEntries(input); + expect(key, first.log).toMatch(/^[0-9a-f]{64}$/u); + const executable = path.join(input.repo, output, "executor-collector"); + const stored = path.join(input.cache, "store/linux-x64", key!, "executor-collector"); + + // The workspace's executable cut short: not "up to date"; the store's whole copy replaces it. + yield* fs.writeFileString(executable, "cut"); + const workspace = yield* build(input); + expect(workspace.exitCode, workspace.log).toBe(0); + expect(workspace.log).not.toContain("is up to date"); + expect(workspace.log).toContain("copied from"); + expect(yield* read(executable)).toBe("collector build 1\n"); + + // The stored executable cut short: removed and built again, never handed to a workspace. + yield* fs.writeFileString(stored, "cut"); + yield* fs.remove(path.join(input.repo, output), { recursive: true }); + const damaged = yield* build(input); + expect(damaged.exitCode, damaged.log).toBe(0); + expect(damaged.log).toContain("Removed the damaged stored build"); + expect(damaged.compiled, damaged.log).toBe(1); + expect(yield* read(executable)).toBe("collector build 2\n"); + expect(yield* read(stored)).toBe("collector build 2\n"); + + // One byte changed, the length kept: the digest, not the size, decides. In the workspace's + // copy first, then in the store's. + yield* fs.writeFileString(executable, "Xollector build 2\n"); + const flipped = yield* build(input); + expect(flipped.exitCode, flipped.log).toBe(0); + expect(flipped.log).not.toContain("is up to date"); + expect(flipped.log).toContain("copied from"); + expect(yield* read(executable)).toBe("collector build 2\n"); + yield* fs.writeFileString(stored, "Xollector build 2\n"); + yield* fs.writeFileString(executable, "Xollector build 2\n"); + const storeFlipped = yield* build(input); + expect(storeFlipped.exitCode, storeFlipped.log).toBe(0); + expect(storeFlipped.log).toContain("Removed the damaged stored build"); + expect(storeFlipped.compiled, storeFlipped.log).toBe(1); + expect(yield* read(executable)).toBe("collector build 3\n"); + expect(yield* read(stored)).toBe("collector build 3\n"); + + // Unchanged since: up to date, nothing compiled. + const again = yield* build(input); + expect(again.log).toContain("is up to date"); + expect(again.compiled).toBe(0); + }).pipe(Effect.scoped), + ); + + it.effect("an abandoned partial copy is removed and counted, a live writer's is kept", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const input = yield* repository; + const store = path.join(input.cache, "store/linux-x64"); + yield* fs.makeDirectory(store, { recursive: true }); + // A copy whose writer has exited (no process has that id) and one whose writer runs: this + // test's own process, the copy fresh. + const dead = path.join(store, `${"0".repeat(64)}.partial-2147483646-0a0b0c0d`); + const live = path.join(store, `${"1".repeat(64)}.partial-${process.pid}-0a0b0c0d`); + for (const partial of [dead, live]) { + yield* fs.makeDirectory(partial); + yield* fs.writeFileString(path.join(partial, "executor-collector"), "partial"); + } + const run = yield* build(input); + expect(run.exitCode, run.log).toBe(0); + expect(yield* fs.exists(dead), "the abandoned copy is removed").toBe(false); + expect(yield* fs.exists(live), "a live writer's copy is kept").toBe(true); + // Partial copies count against the store's bound: 31 old entries, the live partial and the + // new build are 33, so the oldest entry goes. + const old = (Date.now() - 2 * 24 * 60 * 60 * 1000) / 1000; + for (let index = 0; index < 31; index++) { + const entry = path.join(store, index.toString(16).padStart(64, "a")); + yield* fs.makeDirectory(entry); + yield* fs.writeFileString(path.join(entry, "last-used"), ""); + yield* fs.utimes(path.join(entry, "last-used"), old + index, old + index); + } + yield* fs.writeFileString( + path.join(input.repo, "packages/telemetry/collector/src/main.rs"), + "fn main() { }\n", + ); + const next = yield* build(input); + expect(next.exitCode, next.log).toBe(0); + const entries = yield* storeEntries(input); + expect(entries).toHaveLength(32); + expect(entries).toContain(path.basename(live)); + expect(entries).not.toContain("0".padStart(64, "a")); + }).pipe(Effect.scoped), + ); + + it.effect("a copy that fails part way leaves no partial copy behind", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const input = yield* repository; + const first = yield* build(input); + expect(first.exitCode, first.log).toBe(0); + yield* fs.remove(path.join(input.repo, output), { recursive: true }); + // Every copy out of the store fails part way, as a full disk would. + const preload = path.join(input.root, "fail-copy.mjs"); + yield* fs.writeFileString( + preload, + `import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +const copy = fs.cpSync; +fs.cpSync = (from, to, options) => { + if (String(from).includes("/store/")) { copy(from, to, options); throw new Error("ENOSPC: copy cut short"); } + return copy(from, to, options); +}; +syncBuiltinESMExports(); +`, + ); + const failed = yield* build(input, preload); + expect(failed.exitCode).not.toBe(0); + const left = (yield* fs.readDirectory( + path.join(input.repo, "packages/telemetry/dist"), + )).filter((name) => name.includes(".partial-")); + expect(left, "no partial copy is left in the workspace").toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect("an entry marked after pruning listed it is kept: pruning reads it again", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const input = yield* repository; + const first = yield* build(input); + expect(first.exitCode, first.log).toBe(0); + const [key] = yield* storeEntries(input); + const marker = path.join(input.cache, "store/linux-x64", key!, "last-used"); + // Unused for 40 days, past the store's 30. + const old = (Date.now() - 40 * 24 * 60 * 60 * 1000) / 1000; + yield* fs.utimes(marker, old, old); + // Another crate's build publishes and prunes. Just after its pruning first reads the old + // entry's last use, another workspace marks the entry used, before pruning decides. + yield* fs.writeFileString( + path.join(input.repo, "packages/telemetry/collector/src/main.rs"), + "fn main() { }\n", + ); + const preload = path.join(input.root, "mark-after-listing.mjs"); + yield* fs.writeFileString( + preload, + `import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +const stat = fs.statSync; +let listed = false; +fs.statSync = (file, ...rest) => { + const found = stat(file, ...rest); + if (!listed && String(file) === ${JSON.stringify(marker)}) { + listed = true; + const now = new Date(); + fs.utimesSync(file, now, now); + } + return found; +}; +syncBuiltinESMExports(); +`, + ); + const run = yield* build(input, preload); + expect(run.exitCode, run.log).toBe(0); + expect(yield* fs.exists(marker), "the entry marked after the listing is kept").toBe(true); + }).pipe(Effect.scoped), + ); + + it.effect("an old entry being copied is not removed by another build's pruning", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const input = yield* repository; + const first = yield* build(input); + expect(first.exitCode, first.log).toBe(0); + const [key] = yield* storeEntries(input); + const entry = path.join(input.cache, "store/linux-x64", key!); + // Unused for 40 days, past the store's 30. + const old = (Date.now() - 40 * 24 * 60 * 60 * 1000) / 1000; + yield* fs.utimes(path.join(entry, "last-used"), old, old); + yield* fs.remove(path.join(input.repo, output), { recursive: true }); + // A second workspace, whose crate differs, publishes and prunes just as this one starts + // copying the old entry: run it from inside the first copy out of the store. + const second = path.join(input.root, "second"); + yield* fs.copy(input.repo, second); + yield* fs.writeFileString( + path.join(second, "packages/telemetry/collector/src/main.rs"), + "fn main() { }\n", + ); + const preload = path.join(input.root, "prune-during-copy.mjs"); + yield* fs.writeFileString( + preload, + `import fs from "node:fs"; +import { execFileSync } from "node:child_process"; +import { syncBuiltinESMExports } from "node:module"; +const copy = fs.cpSync; +let once = false; +fs.cpSync = (from, to, options) => { + if (!once && String(from).includes("/store/")) { + once = true; + execFileSync("node", ["packages/telemetry/scripts/build-collector.ts", "--target", "linux-x64", "build"], { cwd: ${JSON.stringify(second)}, stdio: "ignore" }); + } + return copy(from, to, options); +}; +syncBuiltinESMExports(); +`, + ); + const run = yield* build(input, preload); + expect(run.exitCode, run.log).toBe(0); + expect(run.log).toContain("copied from"); + expect(yield* fs.exists(entry), "the entry being copied is kept").toBe(true); + expect(yield* read(path.join(input.repo, output, "executor-collector"))).toBe( + "collector build 1\n", + ); + }).pipe(Effect.scoped), + ); + + it.effect("CARGO_BUILD_JOBS reaches cargo and is not part of the identity", () => + Effect.gen(function* () { + const input = yield* repository; + const first = yield* build(input, undefined, input.repo, { CARGO_BUILD_JOBS: "3" }); + expect(first.exitCode, first.log).toBe(0); + expect(yield* read(`${input.state}/jobs`)).toBe("3\n"); + // Another limit, the same identity: up to date, nothing compiled. + const second = yield* build(input, undefined, input.repo, { CARGO_BUILD_JOBS: "5" }); + expect(second.log).toContain("is up to date"); + expect(second.compiled).toBe(0); + }).pipe(Effect.scoped), + ); + + it.effect("proxy and certificate variables reach the fetch alone, outside the identity", () => + Effect.gen(function* () { + const input = yield* repository; + const proxied = { + HTTPS_PROXY: "http://proxy.invalid:3128", + https_proxy: "http://lower.invalid:3128", + CARGO_HTTP_CAINFO: "/etc/fake-ca.pem", + SSL_CERT_FILE: "/etc/fake-ca.pem", + }; + const first = yield* build(input, undefined, input.repo, proxied); + expect(first.exitCode, first.log).toBe(0); + expect(yield* read(`${input.state}/fetch-env`)).toContain( + "proxy=http://proxy.invalid:3128 lower=http://lower.invalid:3128 cainfo=/etc/fake-ca.pem cert=/etc/fake-ca.pem", + ); + expect(yield* read(`${input.state}/build-env`)).toContain( + "proxy=unset lower=unset cainfo=unset cert=unset", + ); + // Another proxy, or none: the same identity, nothing compiled. + for (const env of [{ HTTPS_PROXY: "http://other.invalid:8080" }, {}]) { + const again = yield* build(input, undefined, input.repo, env); + expect(again.log).toContain("is up to date"); + expect(again.compiled).toBe(0); + } + }).pipe(Effect.scoped), + ); + + it.effect("jobserver and diagnostic variables reach neither the fetch nor the compile", () => + Effect.gen(function* () { + const input = yield* repository; + const run = yield* build(input, undefined, input.repo, { + MAKEFLAGS: "-j2 --jobserver-auth=3,4", + CARGO_MAKEFLAGS: "-j2 --jobserver-auth=3,4", + RUST_BACKTRACE: "full", + RUST_LOG: "debug", + }); + expect(run.exitCode, run.log).toBe(0); + const unset = "make=unset cargomake=unset backtrace=unset log=unset"; + expect(yield* read(`${input.state}/fetch-env`)).toContain(unset); + expect(yield* read(`${input.state}/build-env`)).toContain(unset); + }).pipe(Effect.scoped), + ); + + it.effect("--verbose asks cargo and the compiler for more, outside the identity", () => + Effect.gen(function* () { + const input = yield* repository; + // A failed build says how to see more. + yield* flag(input, "fail-once"); + const failed = yield* build(input); + expect(failed.exitCode).not.toBe(0); + expect(failed.log).toContain("run the build again with --verbose"); + const verbose = yield* build(input, undefined, input.repo, {}, ["--verbose"]); + expect(verbose.exitCode, verbose.log).toBe(0); + expect(yield* read(`${input.state}/build-env`)).toMatch( + /backtrace=1 log=unset args=build -vv /u, + ); + const plain = yield* build(input); + expect(plain.log).toContain("is up to date"); + expect(plain.compiled).toBe(0); + }).pipe(Effect.scoped), + ); + + it.effect( + "a store that cannot be written is said loudly, and three builds at once do not queue", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const input = yield* repository; + yield* flag(input, "slow"); + const three = yield* workspaces(input, 3); + // The store alone is read-only: the locks beside it can still be taken. + const store = path.join(input.cache, "store"); + yield* fs.makeDirectory(store, { recursive: true }); + yield* fs.chmod(store, 0o555); + const notes = three.map((repo) => `${repo}.notes`); + const runs = yield* Effect.all( + three.map((repo, index) => + build(input, undefined, repo, { EXECUTOR_PREPARE_NOTES: notes[index]! }), + ), + { concurrency: "unbounded" }, + ).pipe(Effect.ensuring(fs.chmod(store, 0o755).pipe(Effect.orDie))); + for (const run of runs) { + expect(run.exitCode, run.log).toBe(0); + expect(run.log).toContain("cannot be written"); + } + for (const file of notes) + expect(yield* read(file), "the line e2e:prepare repeats").toMatch( + /^The shared collector store .* cannot be written \(Error: EACCES/u, + ); + // Three compiles, each while the others ran: none waited for another's build. + const spans = (yield* read(`${input.state}/spans`)) + .split("\n") + .filter(Boolean) + .map((line) => line.split(" ").map(Number) as [number, number]); + expect(spans).toHaveLength(3); + expect( + Math.max(...spans.map(([start]) => start)), + runs.map((run) => run.log).join("\n"), + ).toBeLessThan(Math.min(...spans.map(([, end]) => end))); + const note = path.join(input.cache, "locks/linux-x64/store-unwritable"); + expect(yield* fs.exists(note), "the note beside the locks").toBe(true); + expect(yield* locks(input)).toEqual(["store-unwritable"]); + // The store repaired: the next build stores, and the note goes with no one removing it. + yield* fs.remove(path.join(three[0]!, output), { recursive: true }); + const repaired = yield* build(input, undefined, three[0]); + expect(repaired.exitCode, repaired.log).toBe(0); + expect(repaired.log).not.toContain("cannot be written"); + expect(yield* storeEntries(input)).toHaveLength(1); + expect(yield* fs.exists(note)).toBe(false); + }).pipe(Effect.scoped), + ); + + it.effect( + "a store with room for the probe but not the build: waiters read the note and do not queue", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const input = yield* repository; + yield* flag(input, "slow"); + const three = yield* workspaces(input, 3); + // Every copy into the store fails as a full disk would; the small probe still fits. + const preload = path.join(input.root, "store-full.mjs"); + yield* fs.writeFileString( + preload, + `import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +const copy = fs.cpSync; +fs.cpSync = (from, to, options) => { + if (String(to).includes("/store/")) throw Object.assign(new Error("ENOSPC: no space left on device"), { code: "ENOSPC" }); + return copy(from, to, options); +}; +syncBuiltinESMExports(); +`, + ); + const [first, ...rest] = three; + const [held, ...waited] = yield* Effect.all( + [ + build(input, preload, first), + ...rest.map((repo) => + lockTaken(input).pipe(Effect.andThen(build(input, preload, repo))), + ), + ], + { concurrency: "unbounded" }, + ); + for (const run of [held!, ...waited]) expect(run.exitCode, run.log).toBe(0); + expect(held!.log).toContain("cannot be written (Error: ENOSPC"); + // Each read the note: while the holder still held the lock, or on taking it after. + for (const run of waited) + expect(run.log).toMatch(/building (here without waiting|without holding the lock)/u); + // The two waiters compile together once the holder has noted the store, not one after another. + const spans = (yield* read(`${input.state}/spans`)) + .split("\n") + .filter(Boolean) + .map((line) => line.split(" ").map(Number) as [number, number]) + .sort((a, b) => a[0] - b[0]); + expect(spans).toHaveLength(3); + const [, second, third] = spans; + expect(Math.max(second![0], third![0])).toBeLessThan(Math.min(second![1], third![1])); + }).pipe(Effect.scoped), + ); + + it.effect( + "a link to nothing counts by its target; editor leftovers do not count, near misses do", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const input = yield* repository; + const src = path.join(input.repo, "packages/telemetry/collector/src"); + // Emacs's lock link for an open file: a link to nothing, its text a user, host and pid. + yield* fs.symlink("someone@host.4242:1700000000", path.join(src, ".#main.rs")); + const first = yield* build(input); + expect(first.exitCode, first.log).toBe(0); + expect(first.compiled).toBe(1); + // Left out: another Emacs lock, a vim swap file, a backup and .DS_Store. + yield* fs.remove(path.join(src, ".#main.rs")); + yield* fs.symlink("someone@host.4343:1700000001", path.join(src, ".#main.rs")); + for (const name of [".main.rs.swp", "main.rs~", ".DS_Store"]) + yield* fs.writeFileString(path.join(src, name), "leftover\n"); + const leftovers = yield* build(input); + expect(leftovers.log, "leftovers are not part of the identity").toContain("is up to date"); + // Counted: a near miss of the swap pattern, and any other link to nothing, by its target. + for (const [change, act] of [ + ["a near miss, main.swp", fs.writeFileString(path.join(src, "main.swp"), "counts\n")], + ["a link to nothing", fs.symlink("nowhere", path.join(src, "dangling"))], + [ + "the same link's new target", + fs + .remove(path.join(src, "dangling")) + .pipe(Effect.andThen(fs.symlink("elsewhere", path.join(src, "dangling")))), + ], + ] as const) { + yield* act; + const run = yield* build(input); + expect(run.exitCode, run.log).toBe(0); + expect(run.compiled, change).toBe(1); + } + // A link to a directory is still refused. + yield* fs.symlink("..", path.join(src, "up")); + const refused = yield* build(input); + expect(refused.exitCode).toBe(1); + expect(refused.log).toContain("neither a file, a directory nor a link to a file"); + }).pipe(Effect.scoped), + ); + + it.effect("a source file git ignores is part of the identity", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const input = yield* repository; + // The repository's root ignores dist/, as executor-next's does; the compiler still reads it. + yield* fs.writeFileString(path.join(input.repo, ".gitignore"), "dist/\n"); + const module = path.join(input.repo, "packages/telemetry/collector/src/dist/mod.rs"); + yield* fs.makeDirectory(path.dirname(module), { recursive: true }); + yield* fs.writeFileString(module, "pub const A: u8 = 1;\n"); + const first = yield* build(input); + expect(first.exitCode, first.log).toBe(0); + expect(first.compiled).toBe(1); + yield* fs.writeFileString(module, "pub const A: u8 = 2;\n"); + const changed = yield* build(input); + expect(changed.exitCode, changed.log).toBe(0); + expect(changed.log).not.toContain("is up to date"); + expect(changed.compiled, "the ignored file's change is compiled").toBe(1); + // The crate's own target/, where cargo test writes, is not. + const tested = path.join(input.repo, "packages/telemetry/collector/target/debug/x"); + yield* fs.makeDirectory(path.dirname(tested), { recursive: true }); + yield* fs.writeFileString(tested, "test output\n"); + const again = yield* build(input); + expect(again.log).toContain("is up to date"); + }).pipe(Effect.scoped), + ); + + it.effect( + "nothing after a whole build fails it: a file gone in the walk, a cache it cannot read", + () => + Effect.gen(function* () { + const path = yield* Path.Path; + const input = yield* repository; + // Another workspace's rustc removes its temporary file between the walk's listing and its stat. + yield* flag(input, "vanish"); + const preload = path.join(input.root, "vanish.mjs"); + yield* (yield* FileSystem.FileSystem).writeFileString( + preload, + `import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +const stat = fs.statSync; +fs.statSync = (file, ...rest) => { + if (String(file).endsWith("/.temp-archive/tmp.a")) { + fs.rmSync(file, { force: true }); + throw Object.assign(new Error("ENOENT: no such file or directory, stat '" + file + "'"), { code: "ENOENT" }); + } + return stat(file, ...rest); +}; +syncBuiltinESMExports(); +`, + ); + const vanished = yield* build(input, preload); + expect(vanished.exitCode, vanished.log).toBe(0); + expect(vanished.log).not.toContain("failed; the build stands"); + expect(yield* read(path.join(input.repo, output, "executor-collector"))).toBe( + "collector build 1\n", + ); + // Pruning cannot even list the cache's target directories: logged, and the build stands. + yield* (yield* FileSystem.FileSystem).writeFileString( + path.join(input.repo, "packages/telemetry/collector/src/main.rs"), + "fn main() { }\n", + ); + const unreadable = path.join(input.root, "unreadable.mjs"); + yield* (yield* FileSystem.FileSystem).writeFileString( + unreadable, + `import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +const list = fs.readdirSync; +fs.readdirSync = (directory, ...rest) => { + if (String(directory) === ${JSON.stringify(path.join(input.cache, "target"))}) + throw Object.assign(new Error("EACCES: permission denied"), { code: "EACCES" }); + return list(directory, ...rest); +}; +syncBuiltinESMExports(); +`, + ); + const failed = yield* build(input, unreadable); + expect(failed.exitCode, failed.log).toBe(0); + expect(failed.log).toContain("failed; the build stands: Error: EACCES"); + expect(yield* read(path.join(input.repo, output, "executor-collector"))).toBe( + "collector build 2\n", + ); + }).pipe(Effect.scoped), + ); + + it.effect("four builds of one identity at once compile once and each copy it", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const input = yield* repository; + yield* flag(input, "slow"); + const four = yield* workspaces(input, 4); + const runs = yield* Effect.all( + four.map((repo) => build(input, undefined, repo)), + { concurrency: "unbounded" }, + ); + for (const run of runs) expect(run.exitCode, run.log).toBe(0); + expect(yield* compiles(input), runs.map((run) => run.log).join("\n")).toBe(1); + expect(runs.filter((run) => run.log.includes("Built executor-collector"))).toHaveLength(1); + const copied = runs.filter((run) => run.log.includes("copied from")); + expect(copied).toHaveLength(3); + for (const run of copied) expect(run.log).toContain("Waiting for process"); + for (const repo of four) + expect(yield* read(path.join(repo, output, "executor-collector"))).toBe( + "collector build 1\n", + ); + expect(yield* locks(input), "every lock is released").toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect("a waiter builds when the lock's holder fails, and takes over when it is killed", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const input = yield* repository; + const [first, second] = yield* workspaces(input, 2); + // The holder's build fails: the waiter finds no stored build and compiles it itself. + yield* flag(input, "fail-once"); + const [failed, waited] = yield* Effect.all( + [ + build(input, undefined, first), + lockTaken(input).pipe(Effect.andThen(build(input, undefined, second))), + ], + { concurrency: "unbounded" }, + ); + expect(failed.exitCode).not.toBe(0); + expect(waited.exitCode, waited.log).toBe(0); + expect(waited.log).toContain("Waiting for process"); + expect(waited.log).toContain("Built executor-collector"); + expect(yield* read(path.join(second!, output, "executor-collector"))).toBe( + "collector build 1\n", + ); + expect(yield* locks(input)).toEqual([]); + + // The holder is killed part way, its lock left behind: the waiter takes it over. + const fs = yield* FileSystem.FileSystem; + for (const repo of [first!, second!]) + yield* fs.writeFileString( + path.join(repo, "packages/telemetry/collector/src/main.rs"), + "fn main() { }\n", + ); + yield* flag(input, "die-once"); + const [killed, tookOver] = yield* Effect.all( + [ + build(input, undefined, first), + lockTaken(input).pipe(Effect.andThen(build(input, undefined, second))), + ], + { concurrency: "unbounded" }, + ); + expect(killed.exitCode).not.toBe(0); + expect(tookOver.exitCode, tookOver.log).toBe(0); + expect(tookOver.log).toMatch(/is stale, its builder, process \d+, has exited/u); + expect(tookOver.log).toContain("Built executor-collector"); + expect(yield* read(path.join(second!, output, "executor-collector"))).toBe( + "collector build 2\n", + ); + expect(yield* locks(input)).toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect( + "a lock whose process id now names another process, or over an hour old, is stale", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const input = yield* repository; + // Learn the identity, then make its lock by hand. + const first = yield* build(input); + expect(first.exitCode, first.log).toBe(0); + const [key] = yield* storeEntries(input); + yield* fs.remove(path.join(input.cache, "store"), { recursive: true }); + yield* fs.remove(path.join(input.repo, output), { recursive: true }); + const lock = path.join(input.cache, "locks/linux-x64", `${key}.lock`); + yield* fs.makeDirectory(path.dirname(lock), { recursive: true }); + // This test's process is running, but it started long after the builder that took the lock. + yield* fs.writeFileString( + lock, + JSON.stringify({ pid: process.pid, started: 0, token: "a" }), + ); + const reused = yield* build(input); + expect(reused.exitCode, reused.log).toBe(0); + expect(reused.log).toContain(`process ${process.pid} is another process now`); + expect(reused.log).toContain("Built executor-collector"); + // The process and its start time match, but the lock was taken two hours ago. + yield* fs.remove(path.join(input.cache, "store"), { recursive: true }); + yield* fs.remove(path.join(input.repo, output), { recursive: true }); + const started = Math.round(Date.now() - process.uptime() * 1000); + yield* fs.writeFileString(lock, JSON.stringify({ pid: process.pid, started, token: "b" })); + const old = (Date.now() - 2 * 60 * 60 * 1000) / 1000; + yield* fs.utimes(lock, old, old); + const aged = yield* build(input); + expect(aged.exitCode, aged.log).toBe(0); + expect(aged.log).toContain("minutes ago"); + expect(aged.log).toContain("Built executor-collector"); + expect(yield* locks(input)).toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect( + "a store this build cannot write is no lock and no failure: it builds and says so", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const input = yield* repository; + for (const name of ["store", "locks"]) { + yield* fs.makeDirectory(path.join(input.cache, name), { recursive: true }); + yield* fs.chmod(path.join(input.cache, name), 0o555); + } + const run = yield* build(input).pipe( + Effect.ensuring( + Effect.forEach(["store", "locks"], (name) => + fs.chmod(path.join(input.cache, name), 0o755), + ).pipe(Effect.orDie), + ), + ); + expect(run.exitCode, run.log).toBe(0); + expect(run.log).toContain("Cannot take the lock"); + expect(run.log).toContain("cannot be written"); + expect(yield* read(path.join(input.repo, output, "executor-collector"))).toBe( + "collector build 1\n", + ); + }).pipe(Effect.scoped), + ); +}); diff --git a/e2e/tests/collector-job.spec.ts b/e2e/tests/collector-job.spec.ts index bdc5e7d14f..d298866dbb 100644 --- a/e2e/tests/collector-job.spec.ts +++ b/e2e/tests/collector-job.spec.ts @@ -96,6 +96,7 @@ const wait = ( target: string, attempt = 1, limits?: { readonly limit: number; readonly attempt: number; readonly poll: number }, + caller?: string, ) => Effect.gen(function* () { const processes = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -107,6 +108,8 @@ const wait = ( GITHUB_RUN_ID: run, GH_TOKEN: "synthetic-token", TARGET: target, + COLLECTOR_USE: "packaging", + ...(caller === undefined ? {} : { CALLER: caller }), GITHUB_RUN_ATTEMPT: String(attempt), ...(limits === undefined ? {} @@ -160,7 +163,7 @@ layer(NodeServices.layer)("packaging waits for its target's collector job", (it) const result = yield* wait(api.url(), "win32-x64"); expect(result.exitCode).toBe(1); expect(result.log).toContain( - `The collector (win32-x64) job ended ${conclusion}, so this run has no win32-x64 collector to package.`, + `The collector (win32-x64) job ended ${conclusion}, so this job has no win32-x64 collector for its packaging; this failure follows from that one. Rerun the collector (win32-x64) job, then this one.`, ); expect(api.requests()).toHaveLength(1); expect(result.seconds).toBeLessThan(10); @@ -202,6 +205,77 @@ layer(NodeServices.layer)("packaging waits for its target's collector job", (it) }).pipe(Effect.scoped), ); + it.effect("a called workflow's collector job is found under its caller's job name", () => + Effect.gen(function* () { + const api = yield* jobsApi([ + { + jobs: [ + { name: "checks / e2e-local (1/1)", ...inProgress }, + { name: "checks / collector (linux-x64)", ...inProgress }, + ], + }, + { jobs: [{ name: "checks / collector (linux-x64)", ...succeeded }] }, + ]); + const result = yield* wait( + api.url(), + "linux-x64", + 1, + { limit: 10, attempt: 2, poll: 1 }, + "checks", + ); + expect(result.exitCode, result.log).toBe(0); + expect(api.requests()).toHaveLength(2); + }).pipe(Effect.scoped), + ); + + it.effect("a job named like the collector's under a caller and on its own stops packaging", () => + Effect.gen(function* () { + const api = yield* jobsApi([ + { + jobs: [ + { name: "collector (linux-x64)", ...succeeded }, + { name: "checks / collector (linux-x64)", ...succeeded }, + ], + }, + ]); + const result = yield* wait(api.url(), "linux-x64", 1, undefined, "checks"); + expect(result.exitCode).toBe(1); + expect(result.log).toContain("cannot tell which to wait for"); + }).pipe(Effect.scoped), + ); + + it.effect( + "another caller's collector job is not this run's, nor one without a named caller", + () => + Effect.gen(function* () { + for (const caller of ["checks", undefined]) { + const api = yield* jobsApi([ + { jobs: [{ name: "other-caller / collector (linux-x64)", ...succeeded }] }, + ]); + const result = yield* wait( + api.url(), + "linux-x64", + 1, + { limit: 2, attempt: 1, poll: 1 }, + caller, + ); + expect(result.exitCode, String(caller)).toBe(1); + expect(result.log).toContain("lists no collector (linux-x64) job yet"); + } + }).pipe(Effect.scoped), + ); + + it.effect("a job whose name only ends like the collector's is not it", () => + Effect.gen(function* () { + const api = yield* jobsApi([ + { jobs: [{ name: "checks /collector (linux-x64)", ...succeeded }] }, + ]); + const result = yield* wait(api.url(), "linux-x64", 1, { limit: 2, attempt: 1, poll: 1 }); + expect(result.exitCode).toBe(1); + expect(result.log).toContain("lists no collector (linux-x64) job yet"); + }).pipe(Effect.scoped), + ); + it.effect("two jobs with the collector job's name on two reads in a row stop packaging", () => Effect.gen(function* () { const api = yield* jobsApi([ diff --git a/e2e/tests/compiled-file-limit.spec.ts b/e2e/tests/compiled-file-limit.spec.ts new file mode 100644 index 0000000000..d8793a7762 --- /dev/null +++ b/e2e/tests/compiled-file-limit.spec.ts @@ -0,0 +1,390 @@ +/** + * A build refuses one of the app's own JavaScript or TypeScript files over 2,000,000 bytes before + * compiling it: esbuild compiles one file in a single stretch on the thread that runs every app's + * code. Files of npm packages the app imports are not limited and compile however large. JSON + * files are outside the limit: the app's own server JSON imports become modules of their own, and + * other JSON goes through esbuild's JSON loader. + */ +import { expect, layer } from "@effect/vitest"; +import { Clock, Effect, Fiber, Schema } from "effect"; +import { randomUUID } from "node:crypto"; +import { scenarios } from "../test-plan.ts"; +import { Actors } from "../support/actors.ts"; +import { Api, body } from "../support/api.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { App, Inventory } from "../support/contracts.ts"; +import { Evidence } from "../support/evidence.ts"; +import { appsManifest, withApps } from "../support/apps-release.ts"; +import { largeModuleCount, largeModulePackage } from "../support/npm-registry.ts"; + +const limit = 2_000_000; +/** An app that imports a package whose one module is 3,000,000 bytes, served by the e2e registry. */ +const largePackageModule = largeModulePackage.name; +const largePackageApp = [ + { + path: "index.ts", + content: `import { defineApp, object, query, router } from "apps"; +import { count } from "${largePackageModule}"; +export default defineApp({}, { + tools: router({ count: query({ input: object({}) }, async () => count) }), +});`, + }, + { + path: "package.json", + content: JSON.stringify({ + dependencies: withApps({ [largePackageModule]: largeModulePackage.version }), + }), + }, +]; + +const BuildFailed = Schema.Struct({ + _tag: Schema.Literal("VersionBuildFailed"), + stage: Schema.String, + location: Schema.optional(Schema.Struct({ file: Schema.String })), + message: Schema.String, +}); + +const head = 'export const filler = "'; +const tail = '";\n'; +/** A module of exactly `bytes` UTF-8 bytes whose `filler` export is `unit` repeated. */ +const module = (bytes: number, unit = "x") => { + const unitBytes = new TextEncoder().encode(unit).length; + const room = bytes - head.length - tail.length; + const content = `${head}${unit.repeat(Math.floor(room / unitBytes))}${"x".repeat(room % unitBytes)}${tail}`; + expect(new TextEncoder().encode(content).length).toBe(bytes); + return content; +}; +/** + * A module of exactly `bytes` ASCII bytes of generated objects, as an app's bundled API description + * is: dense code, which costs the compiler far more per byte than one string. `filler` is the + * number of objects. + */ +const denseModule = (bytes: number) => { + const entry = (at: number) => + ` { id: ${at}, name: "item ${at}", tags: ["a", "b"], nested: { x: ${at}, y: "z" } },\n`; + const open = "export const filler = [\n"; + const close = "].length;\n"; + const entries: string[] = []; + let used = open.length + close.length + 3; + while (used + entry(entries.length).length <= bytes) { + used += entry(entries.length).length; + entries.push(entry(entries.length)); + } + const content = `${open}${entries.join("")}${close}//${" ".repeat(bytes - used)}\n`; + expect(new TextEncoder().encode(content).length).toBe(bytes); + return { content, count: entries.length }; +}; +const dense = denseModule(limit - 1_000); +const utf8 = (text: string) => new TextEncoder().encode(text).length; +/** A module of `filler` followed by a comment that ends the file with `last`. */ +const endingWith = (bytes: number, last: string) => { + const room = bytes - head.length - tail.length - 2 - utf8(last); + const content = `${head}${"x".repeat(room)}${tail}//${last}`; + expect(utf8(content)).toBe(bytes); + return content; +}; +const reader = `import { defineApp, object, query, router } from "apps"; +import { filler } from "./filler.ts"; +export default defineApp({}, { + tools: router({ size: query({ input: object({}) }, async () => filler.length) }), +});`; + +layer(HostedLive, { excludeTestServices: true })("Compiled file limit", (it) => { + it.effect(scenarios.compiledFileLimit.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors, + evidence = yield* Evidence; + const prefix = `/api/organizations/${actors.organization.id}`; + const deploy = (name: string, files: ReadonlyArray<{ path: string; content: string }>) => + api.request(actors.owner, "POST", `${prefix}/apps/deploy`, { name, files }); + const appNames = Effect.gen(function* () { + const inventory = yield* body( + Inventory, + yield* api.request(actors.owner, "GET", `${prefix}/inventory`), + ); + return inventory.apps.map((app) => app.name); + }); + const refused = ( + response: { readonly status: number; readonly body: unknown }, + stage: string, + file: string, + size: number, + ) => + Effect.gen(function* () { + expect(response.status, JSON.stringify(response.body)).toBe(422); + const failed = yield* body(BuildFailed, response); + expect(failed).toMatchObject({ stage, location: { file } }); + expect(failed.message).toContain( + `${file} is ${size.toLocaleString("en-US")} bytes; a JavaScript or TypeScript file the build compiles can be at most 2,000,000 bytes (2 MB).`, + ); + return failed; + }); + + // A module just under the limit builds and runs, beside a larger JSON import, which becomes + // a module of its own and is outside the limit. + const underName = `Under the limit ${randomUUID().slice(0, 8)}`; + const document = `{"items":"${"y".repeat(3_000_000)}"}`; + const started = yield* Clock.currentTimeMillis; + const under = yield* deploy(underName, [ + { + path: "index.ts", + content: `import { defineApp, object, query, router } from "apps"; +import { filler } from "./filler.ts"; +import document from "./document.json"; +export default defineApp({}, { + tools: router({ + size: query({ input: object({}) }, async () => ({ filler, document: document.items.length })), + }), +});`, + }, + { path: "filler.ts", content: dense.content }, + { path: "document.json", content: document }, + appsManifest, + ]); + const finished = yield* Clock.currentTimeMillis; + yield* evidence.json("under-the-limit.json", { + status: under.status, + buildMs: finished - started, + }); + expect(under.status, JSON.stringify(under.body)).toBe(200); + const app = yield* body(App, under); + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", `${prefix}/apps/${app.id}`).pipe(Effect.orDie), + ); + expect( + yield* body( + Schema.Struct({ filler: Schema.Number, document: Schema.Number }), + yield* api.request(actors.owner, "POST", `${prefix}/apps/${app.id}/tools/call`, { + tool: "size", + kind: "query", + input: {}, + }), + ), + ).toEqual({ filler: dense.count, document: 3_000_000 }); + + // Exactly the limit builds, whatever characters reach it. A high surrogate with no low + // one after it, here the file's last unit, is three bytes, as TextEncoder writes it. + const atLimit = { + ascii: module(limit), + twoByte: module(limit, "é"), + threeByte: module(limit, "漢"), + surrogatePairs: module(limit, "🙂"), + loneHighAtEnd: endingWith(limit, "\ud800"), + }; + const accepted: string[] = []; + for (const [kind, content] of Object.entries(atLimit)) { + const name = `At the limit ${kind} ${randomUUID().slice(0, 8)}`; + const deployed = yield* deploy(name, [ + { path: "index.ts", content: reader }, + { path: "filler.ts", content }, + appsManifest, + ]); + expect(deployed.status, `${kind}: ${JSON.stringify(deployed.body)}`).toBe(200); + const atLimitApp = yield* body(App, deployed); + yield* Effect.addFinalizer(() => + api + .request(actors.owner, "DELETE", `${prefix}/apps/${atLimitApp.id}`) + .pipe(Effect.orDie), + ); + accepted.push(name); + } + + // Over the limit is refused before it compiles, and no app is created: one byte over in + // ASCII and in two-byte characters, and text that is not valid UTF-16, which TextEncoder + // and esbuild write with three bytes for each lone surrogate. 350,000 unpaired high + // surrogates, each before a three-byte character, are 700,025 units and 2,100,025 bytes. + const overLimit = { + ascii: module(limit + 1), + twoByte: module(limit + 1, "é"), + unpairedHigh: `${head}${"\ud800漢".repeat(350_000)}";`, + loneLow: endingWith(limit + 1, "\udc00"), + loneHighAtEnd: endingWith(limit + 1, "\ud800"), + }; + expect(utf8(overLimit.unpairedHigh)).toBe(2_100_025); + expect(overLimit.unpairedHigh.length).toBe(700_025); + const refusedNames: string[] = []; + for (const [kind, content] of Object.entries(overLimit)) { + const name = `Over the limit ${kind} ${randomUUID().slice(0, 8)}`; + const deployed = yield* deploy(name, [ + { path: "index.ts", content: reader }, + { path: "filler.ts", content }, + appsManifest, + ]); + yield* evidence.json(`over-the-limit-${kind}.json`, deployed.body); + yield* refused(deployed, "source", "filler.ts", utf8(content)); + refusedNames.push(name); + } + + // A package's file is not limited: an app that imports a 3,000,000-byte module builds and + // its tool answers. + const packageName = `Large package ${randomUUID().slice(0, 8)}`; + const large = yield* deploy(packageName, largePackageApp); + yield* evidence.json("large-package.json", large.body); + expect(large.status, JSON.stringify(large.body)).toBe(200); + const largeApp = yield* body(App, large); + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", `${prefix}/apps/${largeApp.id}`).pipe(Effect.orDie), + ); + expect( + yield* body( + Schema.Number, + yield* api.request(actors.owner, "POST", `${prefix}/apps/${largeApp.id}/tools/call`, { + tool: "count", + kind: "query", + input: {}, + }), + ), + ).toBe(largeModuleCount); + + const names = yield* appNames; + for (const name of [underName, ...accepted, packageName]) expect(names).toContain(name); + for (const name of refusedNames) expect(names).not.toContain(name); + }), + ), + ); + + it.effect( + scenarios.compiledFileLimitResponsive.title, + (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors, + evidence = yield* Evidence; + const prefix = `/api/organizations/${actors.organization.id}`; + const deploy = (name: string, files: ReadonlyArray<{ path: string; content: string }>) => + api.request(actors.owner, "POST", `${prefix}/apps/deploy`, { name, files }); + // A small app, whose deploy also starts the compiler, answers while nothing holds the + // runtime's thread. Every app's code and every build share that thread. + const pinger = yield* deploy(`Pinger ${randomUUID().slice(0, 8)}`, [ + { + path: "index.ts", + content: `import { defineApp, object, query, router, string } from "apps"; +export default defineApp({}, { + tools: router({ ping: query({ input: object({}), output: string() }, async () => "pong") }), +});`, + }, + appsManifest, + ]); + expect(pinger.status, JSON.stringify(pinger.body)).toBe(200); + const pingApp = yield* body(App, pinger); + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", `${prefix}/apps/${pingApp.id}`).pipe(Effect.orDie), + ); + const ping = api + .request(actors.owner, "POST", `${prefix}/apps/${pingApp.id}/tools/call`, { + tool: "ping", + kind: "query", + input: {}, + }) + .pipe(Effect.timeout("60 seconds")); + expect((yield* ping).status).toBe(200); + + /** + * Deploy `files` while calling the ping tool back to back, and return the longest wait + * between two answers, counted from before the request was sent until its answer came. + */ + const measured = ( + label: string, + files: ReadonlyArray<{ path: string; content: string }>, + ) => + Effect.gen(function* () { + const answers: number[] = []; + let failed = 0; + const sampler = yield* ping.pipe( + Effect.flatMap((response) => + Clock.currentTimeMillis.pipe( + Effect.map((at) => + response.status === 200 ? void answers.push(at) : void (failed += 1), + ), + ), + ), + Effect.catch(() => Effect.sync(() => void (failed += 1))), + Effect.forever, + Effect.forkChild, + ); + const sent = yield* Clock.currentTimeMillis; + const deployed = yield* deploy(`${label} ${randomUUID().slice(0, 8)}`, files).pipe( + Effect.timeout("120 seconds"), + ); + const answered = yield* Clock.currentTimeMillis; + yield* Fiber.interrupt(sampler); + expect(failed, "pings that failed").toBe(0); + const during = answers.filter((at) => at > sent && at < answered); + const times = [sent, ...during, answered]; + const longestGapMs = Math.max( + ...times.slice(1).map((at, index) => at - times[index]!), + ); + return { deployed, deployMs: answered - sent, pings: during.length, longestGapMs }; + }); + + // One file just under the limit: the work a legitimate build does on its largest file. + const accepted = yield* measured("Calibration", [ + { path: "index.ts", content: reader }, + { path: "filler.ts", content: denseModule(limit - 1_000).content }, + appsManifest, + ]); + expect(accepted.deployed.status, JSON.stringify(accepted.deployed.body)).toBe(200); + const acceptedApp = yield* body(App, accepted.deployed); + yield* Effect.addFinalizer(() => + api + .request(actors.owner, "DELETE", `${prefix}/apps/${acceptedApp.id}`) + .pipe(Effect.orDie), + ); + + // A dense file of 15,000,000 bytes, within the 16 MiB source limit. Compiling it would + // hold the thread for several times as long as the calibration did; refusing it does not + // hold the thread for even as long as that one file's compile. + const oversized = denseModule(15_000_000).content; + const refusedDeploy = yield* measured("Refused", [ + { path: "index.ts", content: reader }, + { path: "filler.ts", content: oversized }, + appsManifest, + ]); + yield* evidence.json("responsive.json", { + accepted: { + deployMs: accepted.deployMs, + pings: accepted.pings, + longestGapMs: accepted.longestGapMs, + }, + refused: { + status: refusedDeploy.deployed.status, + deployMs: refusedDeploy.deployMs, + pings: refusedDeploy.pings, + longestGapMs: refusedDeploy.longestGapMs, + }, + }); + expect(refusedDeploy.deployed.status, JSON.stringify(refusedDeploy.deployed.body)).toBe( + 422, + ); + expect((yield* body(BuildFailed, refusedDeploy.deployed)).message).toContain( + "filler.ts is 15,000,000 bytes", + ); + expect(accepted.pings).toBeGreaterThan(0); + expect(refusedDeploy.longestGapMs).toBeLessThan(accepted.longestGapMs); + + // What a large package file costs, which the limit does not cover: the apps wait while + // its 3,000,000-byte module compiles. Recorded, not bounded. + const packaged = yield* measured("Large package", largePackageApp); + expect(packaged.deployed.status, JSON.stringify(packaged.deployed.body)).toBe(200); + const packagedApp = yield* body(App, packaged.deployed); + yield* Effect.addFinalizer(() => + api + .request(actors.owner, "DELETE", `${prefix}/apps/${packagedApp.id}`) + .pipe(Effect.orDie), + ); + yield* evidence.json("large-package-cost.json", { + moduleBytes: largeModulePackage.bytes, + deployMs: packaged.deployMs, + pings: packaged.pings, + longestGapMs: packaged.longestGapMs, + }); + }), + ), + 180_000, + ); +}); diff --git a/e2e/tests/conflict-markers.spec.ts b/e2e/tests/conflict-markers.spec.ts new file mode 100644 index 0000000000..2d92a1d472 --- /dev/null +++ b/e2e/tests/conflict-markers.spec.ts @@ -0,0 +1,88 @@ +/** + * `bun run check` fails on a tracked file that still holds a merge conflict's markers + * (scripts/check-conflict-markers.ts). These cases run the check on this checkout, which must pass, + * and in a new repository with a conflict left in one tracked file, which must fail and name each + * marker's file and line. The markers are built here at run time, so this file holds none. + */ +import { expect, layer } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { Effect, FileSystem, Path, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +const [ours, base, middle, theirs] = ["<", "|", "=", ">"].map((mark) => mark.repeat(7)); + +/** Run a command in `cwd` and return its exit code and everything it printed. */ +const run = (command: string, args: ReadonlyArray, cwd: string) => + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* processes.spawn( + ChildProcess.make(command, args, { cwd, extendEnv: true }), + ); + const [output, exitCode] = yield* Effect.all( + [child.all.pipe(Stream.decodeText(), Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + return { exitCode: Number(exitCode), output }; + }).pipe(Effect.scoped); + +const check = (cwd: string) => + Effect.gen(function* () { + const path = yield* Path.Path; + return yield* run("node", [path.resolve("scripts/check-conflict-markers.ts")], cwd); + }); + +layer(NodeServices.layer)("Leftover conflict markers", (it) => { + it.effect("this checkout's tracked files hold none", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const result = yield* check(path.resolve(".")); + expect(result.output).toContain("No conflict markers in tracked files."); + expect(result.exitCode, result.output).toBe(0); + }), + ); + + it.effect("a conflict left in a tracked file fails, naming each marker's file and line", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem, + path = yield* Path.Path; + const repository = yield* fs.makeTempDirectoryScoped({ prefix: "executor-markers-" }); + yield* fs.makeDirectory(path.join(repository, "docs")); + // A diff3 conflict, as `git merge` leaves it with merge.conflictStyle=diff3. + yield* fs.writeFileString( + path.join(repository, "docs/notes.md"), + [ + "# Notes", + `${ours} HEAD`, + "ours", + `${base} base`, + "base", + middle, + "theirs", + `${theirs} topic`, + "", + ].join("\n"), + ); + // Lines that only look like markers: a longer rule, a marker run on into text, a quoted one. + yield* fs.writeFileString( + path.join(repository, "clean.md"), + ["Title", `${middle}===`, `${ours}x`, `> ${theirs} topic`, ""].join("\n"), + ); + // Not tracked, so not checked. + yield* fs.writeFileString(path.join(repository, "untracked.md"), `${ours} HEAD\n`); + expect((yield* run("git", ["init", "-q"], repository)).exitCode).toBe(0); + expect((yield* run("git", ["add", "docs/notes.md", "clean.md"], repository)).exitCode).toBe( + 0, + ); + + const result = yield* check(repository); + expect(result.exitCode, result.output).toBe(1); + const named = result.output.split("\n").filter((line) => line.startsWith(" ")); + expect(named).toEqual([ + ` docs/notes.md:2: ${ours} HEAD`, + ` docs/notes.md:4: ${base} base`, + ` docs/notes.md:6: ${middle}`, + ` docs/notes.md:8: ${theirs} topic`, + ]); + }).pipe(Effect.scoped), + ); +}); diff --git a/e2e/tests/credential-placement.spec.ts b/e2e/tests/credential-placement.spec.ts index 136d144f0a..5c3a4f63a4 100644 --- a/e2e/tests/credential-placement.spec.ts +++ b/e2e/tests/credential-placement.spec.ts @@ -11,6 +11,7 @@ import { FetchHttpClient, HttpClient } from "effect/http"; import { randomUUID } from "node:crypto"; import { Actors } from "../support/actors.ts"; import { Api, body } from "../support/api.ts"; +import { Browser } from "../support/browser.ts"; import { publishedRelease } from "../support/app-package.ts"; import { appsManifest, unscopedAccounts } from "../support/apps-release.ts"; import { HostedLive, withHostedCase } from "../support/case.ts"; @@ -18,7 +19,11 @@ import { Resource } from "../support/contracts.ts"; import { credentialUpstream, type ReceivedRequest } from "../support/credential-upstream.ts"; import { Evidence, Telemetry } from "../support/evidence.ts"; import type { OperatorSettings } from "../support/managed-server.ts"; -import { OperatorOAuthFixture, operatorOAuthFile } from "../support/operator-oauth.ts"; +import { + OperatorOAuthFixture, + operatorOAuthFile, + syntheticCloudGoogle, +} from "../support/operator-oauth.ts"; import { oauthSetupIssuer, oauthSetupIssuerOn, @@ -52,6 +57,22 @@ const Setup = Schema.Struct({ scopes: Schema.Array(Schema.String), firstParty: Schema.optional(Schema.Struct({ label: Schema.String })), userScopes: Schema.optional(Schema.Array(Schema.String)), + operatorScopeRefusal: Schema.optional( + Schema.Struct({ + client: Schema.String, + scopes: Schema.Array(Schema.String), + instead: Schema.Array( + Schema.Struct({ scope: Schema.String, scopes: Schema.Array(Schema.String) }), + ), + }), + ), +}); +/** A sign-in refused before the service because the operator's client may not ask for its scopes. */ +const ScopeRefused = Schema.Struct({ + _tag: Schema.Literal("OAuthSetupFailed"), + reason: Schema.Literal("operator_scope_refused"), + message: Schema.String, + recovery: Schema.Struct({ action: Schema.String, instructions: Schema.String }), }); const Health = Schema.Struct({ apps: Schema.Array( @@ -306,6 +327,11 @@ const operatorDefinition = ( readonly defaultScopes?: readonly string[]; /** What a sign-in may ask for and a grant may hold; `mail.read` and `mail.compose` by default. */ readonly allowedScopes?: readonly string[]; + /** What a grant may hold that no sign-in asks for, and the allowed scopes to ask for instead. */ + readonly heldScopes?: ReadonlyArray<{ + readonly scope: string; + readonly instead: readonly string[]; + }>; /** How the server separates the scopes it reports granting, such as GitHub's `,`. */ readonly grantedScopeSeparator?: string; /** Extra sign-in parameters the operator sends, such as Slack's `user_scope`. */ @@ -342,6 +368,7 @@ const operatorDefinition = ( tokenEndpointAuthMethod: "client_secret_basic", defaultScopes: options.defaultScopes ?? ["mail.read"], allowedScopes: options.allowedScopes ?? ["mail.read", "mail.compose"], + ...(options.heldScopes === undefined ? {} : { heldScopes: options.heldScopes }), placement: { hosts, ...(options.header === undefined ? {} : { header: options.header }), @@ -1037,40 +1064,46 @@ layer(HostedLive, { excludeTestServices: true })("Credential placement", (it) => ), ); + const google = { + authorizationUrl: "https://accounts.google.com/o/oauth2/v2/auth", + tokenUrl: "https://oauth2.googleapis.com/token", + }; + const calendar = "https://www.googleapis.com/auth/calendar"; + const gmail = "https://mail.google.com/"; + const gmailModify = "https://www.googleapis.com/auth/gmail.modify"; + const drive = "https://www.googleapis.com/auth/drive"; + const cloudPlatform = "https://www.googleapis.com/auth/cloud-platform"; + /** Deploy a service on Google's endpoints asking for `scopes`, start signing in and read setup. */ + const signInGoogle = (key: string, scopes: readonly string[]) => + Effect.gen(function* () { + const { api, actors, prefix, deploy, connection } = yield* harness; + const name = `Google ${key} ${randomUUID().slice(0, 8)}`; + const { path, app } = yield* deploy( + name, + endpointsApp(name, ["www.googleapis.com"], google, scopes), + ); + const id = yield* connection(path, "mail"); + const started = yield* api.request( + actors.owner, + "POST", + `${prefix}/connections/${id}/oauth/start`, + { method: "oauth", label: "Google" }, + ); + const provider = app.requirements.accounts.mail!.provider; + const answered = yield* api.request( + actors.owner, + "GET", + `${prefix}/providers/${provider}/oauth/oauth/setup`, + ); + return { started, setup: yield* body(Setup, answered) }; + }); + it.effect(scenarios.credentialManagedGoogle.title, (context) => withHostedCase( context, Effect.gen(function* () { - const { api, actors, prefix, deploy, connection } = yield* harness; const clientId = "synthetic-google-client.apps.googleusercontent.com"; - const google = { - authorizationUrl: "https://accounts.google.com/o/oauth2/v2/auth", - tokenUrl: "https://oauth2.googleapis.com/token", - }; - const calendar = "https://www.googleapis.com/auth/calendar"; - /** Deploy a Google service asking for `scopes`, then start signing in and read setup. */ - const signIn = (key: string, scopes: readonly string[]) => - Effect.gen(function* () { - const name = `Google ${key} ${randomUUID().slice(0, 8)}`; - const { path, app } = yield* deploy( - name, - endpointsApp(name, ["www.googleapis.com"], google, scopes), - ); - const id = yield* connection(path, "mail"); - const started = yield* api.request( - actors.owner, - "POST", - `${prefix}/connections/${id}/oauth/start`, - { method: "oauth", label: "Google" }, - ); - const provider = app.requirements.accounts.mail!.provider; - const answered = yield* api.request( - actors.owner, - "GET", - `${prefix}/providers/${provider}/oauth/oauth/setup`, - ); - return { started, setup: yield* body(Setup, answered) }; - }); + const signIn = signInGoogle; const cleared = { EXECUTOR_OAUTH_GOOGLE_CLIENT_ID: "", EXECUTOR_OAUTH_GOOGLE_CLIENT_SECRET: "", @@ -1087,13 +1120,20 @@ layer(HostedLive, { excludeTestServices: true })("Credential placement", (it) => EXECUTOR_OAUTH_GOOGLE_CLIENT_ID: clientId, EXECUTOR_OAUTH_GOOGLE_CLIENT_SECRET: "synthetic-google-secret", }); - const [allowed, defaults, wider] = yield* Effect.all( + const [allowed, defaults, wider, modify, fullGmail, mixed] = yield* Effect.all( [ signIn("calendar", [calendar]), // A provider that asks for no scopes gets the client's defaults. signIn("defaults", []), // A scope outside the client's allowed scopes. - signIn("wider", [calendar, "https://www.googleapis.com/auth/cloud-platform"]), + signIn("wider", [calendar, cloudPlatform]), + // Google has not approved `gmail.modify` for Executor's own registration, but a + // self-host operator's client is its own registration, which that does not limit. + signIn("modify", [gmailModify, drive, calendar]), + // The approved full Gmail scope, with the same others. + signIn("gmail", [gmail, drive, calendar]), + // `gmail.modify` beside a scope the client was never allowed. + signIn("mixed", [gmailModify, cloudPlatform]), ], { concurrency: "unbounded" }, ); @@ -1132,7 +1172,38 @@ layer(HostedLive, { excludeTestServices: true })("Credential placement", (it) => expect(url.searchParams.get("client_id"), key).toBe(clientId); expect(url.searchParams.get("scope")?.split(" "), key).toEqual(requested); } + // The approved Gmail scope signs in with the client. + expect(fullGmail.setup.firstParty).toEqual({ label: "Executor for Google" }); + expect(fullGmail.started.status, JSON.stringify(fullGmail.started.body)).toBe(200); + const approved = new URL((yield* body(Redirect, fullGmail.started)).authorizationUrl); + expect(approved.searchParams.get("client_id")).toBe(clientId); + expect(approved.searchParams.get("scope")?.split(" ")).toEqual([gmail, drive, calendar]); + + // The self-host operator's own Google client still signs in `gmail.modify`, as it did + // before Executor's own registration held it. + expect(modify.setup.operatorScopeRefusal).toBeUndefined(); + expect(modify.setup).toMatchObject({ + mode: "automatic", + firstParty: { label: "Executor for Google" }, + scopes: [gmailModify, drive, calendar], + }); + expect(modify.started.status, JSON.stringify(modify.started.body)).toBe(200); + const modifySent = new URL((yield* body(Redirect, modify.started)).authorizationUrl); + expect(`${modifySent.origin}${modifySent.pathname}`).toBe(google.authorizationUrl); + expect(modifySent.searchParams.get("client_id")).toBe(clientId); + expect(modifySent.searchParams.get("scope")?.split(" ")).toEqual([ + gmailModify, + drive, + calendar, + ]); + // Beside a scope the client was never allowed, it is simply not offered. + expect(mixed.setup.firstParty).toBeUndefined(); + expect(mixed.setup.operatorScopeRefusal).toBeUndefined(); + expect(mixed.started.body).toMatchObject({ _tag: "OAuthClientUnavailable" }); + + // A scope outside the client's menu that it never asked for leaves it simply not offered. expect(wider.setup.firstParty).toBeUndefined(); + expect(wider.setup.operatorScopeRefusal).toBeUndefined(); expect(wider.started.body).toMatchObject({ _tag: "OAuthClientUnavailable" }); }), ), @@ -1708,6 +1779,218 @@ layer(HostedLive, { excludeTestServices: true })("Credential placement", (it) => ), ); + it.effect(scenarios.credentialManagedHeld.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const { api, actors, prefix, deploy, list, connection, consent } = yield* harness; + const { issuer, store, issuerHost, storeHost } = yield* operatorWith({ + allowedScopes: ["mail.read", "mail.compose", "mail.full"], + }); + const hosts = [issuerHost, storeHost]; + // Token responses report the scopes each grant holds, at sign-in and on renewal, as + // Google's do. + yield* issuer.configure({ scopeGrant: { defaults: [] }, expiresIn: 3600 }); + const base = { + name: "Synthetic Mail", + discover: `${issuer.origin}/mcp`, + hosts, + resource: `${issuer.origin}/resource`, + store: store.origin, + undeclared: store.undeclaredOrigin, + }; + const asking = mailApp({ ...base, scopes: ["mail.read", "mail.compose"] }); + const [operatorApp, ownApp, approvedApp] = yield* Effect.all( + [ + deploy(`Mail operator ${randomUUID().slice(0, 8)}`, asking), + deploy(`Mail own ${randomUUID().slice(0, 8)}`, asking), + deploy( + `Mail approved ${randomUUID().slice(0, 8)}`, + mailApp({ ...base, scopes: ["mail.read", "mail.full"] }), + ), + ], + { concurrency: "unbounded" }, + ); + const removed = (account: string) => + Effect.addFinalizer(() => + api + .request(actors.owner, "DELETE", `${prefix}/accounts/${account}`) + .pipe(Effect.ignore), + ); + /** Start signing in again for `account` through `path`, with the client chosen for it. */ + const reconnect = (path: string, account: string) => + Effect.gen(function* () { + const replacing = yield* body( + Resource, + yield* api.request(actors.owner, "POST", connectionsPath(path, "user", "mail"), { + account, + }), + ); + return yield* api.request( + actors.owner, + "POST", + `${prefix}/connections/${replacing.id}/oauth/start`, + { method: "oauth", label: "Mail" }, + ); + }); + const Read = Schema.Struct({ token: Schema.String, status: Schema.Number }); + /** Call `resource` after the service refused every token so far, so the call renews. */ + const renewedRead = (path: string) => + Effect.gen(function* () { + const before = (yield* issuer.metrics).refreshes; + yield* issuer.expireAccessTokens; + const read = yield* api.request(actors.owner, "POST", `${path}/tools/call`, { + tool: "resource", + kind: "query", + input: {}, + }); + expect(read.status, JSON.stringify(read.body)).toBe(200); + expect((yield* body(Read, read)).status).toBe(200); + expect((yield* issuer.metrics).refreshes).toBeGreaterThan(before); + }); + + // An account the operator's client signed in while it could ask for `mail.compose`. + const first = yield* startSignIn(operatorApp.path); + expect(first.started.status, JSON.stringify(first.started.body)).toBe(200); + const firstRedirect = yield* body(Redirect, first.started); + expect(new URL(firstRedirect.authorizationUrl).searchParams.get("client_id")).toBe( + "first-party-client", + ); + const firstCompleted = yield* finishSignIn(issuer, first.connection, firstRedirect); + expect(firstCompleted.status, JSON.stringify(firstCompleted.body)).toBe(200); + const operatorAccount = (yield* body(Resource, firstCompleted)).id; + yield* removed(operatorAccount); + + // And one signed in with the user's own client for the same service. + const own = { clientId: "own-mail-client", clientSecret: "own-mail-secret" }; + const ownConnection = yield* connection(ownApp.path, "mail"); + const ownStarted = yield* api.request( + actors.owner, + "POST", + `${prefix}/connections/${ownConnection}/oauth/start`, + { method: "oauth", label: "Mail", client: own }, + ); + expect(ownStarted.status, JSON.stringify(ownStarted.body)).toBe(200); + const ownRedirect = yield* body(Redirect, ownStarted); + expect(new URL(ownRedirect.authorizationUrl).searchParams.get("client_id")).toBe( + own.clientId, + ); + yield* issuer.allowClient({ ...own, redirect: ownRedirect.redirectUri }); + const ownCompleted = yield* api.request( + actors.owner, + "POST", + `${prefix}/connections/${ownConnection}/oauth/complete`, + { callbackUrl: yield* consent(ownRedirect.authorizationUrl) }, + ); + expect(ownCompleted.status, JSON.stringify(ownCompleted.body)).toBe(200); + const ownAccount = (yield* body(Resource, ownCompleted)).id; + yield* removed(ownAccount); + + // The service stops letting the operator's client ask for `mail.compose`, so the operator + // holds it, naming `mail.full` instead. + yield* restartWith( + operatorClients(issuer.origin, hosts, { + allowedScopes: ["mail.read", "mail.full"], + heldScopes: [{ scope: "mail.compose", instead: ["mail.full"] }], + }), + ); + + // The account whose grant holds it keeps renewing: the renewal reports the held scope. + // The listing confirms `resource` only reads, so a refused call renews and repeats. + yield* list(operatorApp.path); + yield* renewedRead(operatorApp.path); + + // A new sign-in asking for it is refused before the browser goes to the service. + const provider = operatorApp.app.requirements.accounts.mail!.provider; + const setup = yield* body( + Setup, + yield* api.request( + actors.owner, + "GET", + `${prefix}/providers/${provider}/oauth/oauth/setup`, + ), + ); + expect(setup.firstParty).toBeUndefined(); + expect(setup).toMatchObject({ + mode: "client-required", + operatorScopeRefusal: { + client: "Executor for Synthetic Mail", + scopes: ["mail.compose"], + instead: [{ scope: "mail.compose", scopes: ["mail.full"] }], + }, + }); + const fresh = yield* startSignIn(operatorApp.path); + expect(fresh.started.status, JSON.stringify(fresh.started.body)).toBe(422); + expect(fresh.started.body).not.toHaveProperty("authorizationUrl"); + const refused = yield* body(ScopeRefused, fresh.started); + expect(refused.message).toContain("mail.compose"); + // The user's own client comes first; the replacement second, saying it grants more. + expect(refused.recovery.action).toBe( + "Enter your own OAuth client for this service. Or, if broader access is acceptable, change the app to ask for mail.full instead, which grants more than mail.compose.", + ); + // The connection page says so where it asks for the user's own client, before any attempt. + const browser = yield* Browser; + yield* browser.login(actors.owner); + yield* browser.use("Open the new connection", (page) => + page.goto(`/org/${actors.organization.slug}/connections/${fresh.connection}`), + ); + yield* browser.use("The form explains why Executor's client is not offered", (page) => + page + .getByRole("heading", { + name: "Executor for Synthetic Mail can’t ask for this permission", + }) + .waitFor({ state: "visible" }) + .then(() => + page + .getByText("This app asks for mail.compose", { exact: false }) + .waitFor({ state: "visible" }), + ) + .then(() => + page + .getByText( + "Enter your own OAuth client for this service. Or, if broader access is acceptable, change the app to ask for mail.full instead, which grants more than mail.compose.", + { exact: false }, + ) + .waitFor({ state: "visible" }), + ), + ); + yield* browser.checkpoint("Operator client refused for a held scope"); + + // So is reconnecting the operator's account, and the refusal leaves the account as it + // was: it still renews and reads. + const operatorReconnect = yield* reconnect(operatorApp.path, operatorAccount); + expect(operatorReconnect.status, JSON.stringify(operatorReconnect.body)).toBe(422); + expect(operatorReconnect.body).not.toHaveProperty("authorizationUrl"); + expect((yield* body(ScopeRefused, operatorReconnect)).message).toContain("mail.compose"); + yield* renewedRead(operatorApp.path); + + // Reconnecting the account on the user's own client keeps that client. + const ownReconnect = yield* reconnect(ownApp.path, ownAccount); + expect(ownReconnect.status, JSON.stringify(ownReconnect.body)).toBe(200); + expect( + new URL((yield* body(Redirect, ownReconnect)).authorizationUrl).searchParams.get( + "client_id", + ), + ).toBe(own.clientId); + + // An app asking only for scopes the client may ask for still signs in with it. + const approved = yield* startSignIn(approvedApp.path); + expect(approved.started.status, JSON.stringify(approved.started.body)).toBe(200); + const approvedRedirect = yield* body(Redirect, approved.started); + const approvedUrl = new URL(approvedRedirect.authorizationUrl); + expect(approvedUrl.searchParams.get("client_id")).toBe("first-party-client"); + expect(approvedUrl.searchParams.get("scope")).toBe("mail.read mail.full"); + const approvedCompleted = yield* finishSignIn( + issuer, + approved.connection, + approvedRedirect, + ); + expect(approvedCompleted.status, JSON.stringify(approvedCompleted.body)).toBe(200); + yield* removed((yield* body(Resource, approvedCompleted)).id); + }), + ), + ); + it.effect(scenarios.credentialManagedSettingsInvalid.title, (context) => withHostedCase( context, @@ -1790,6 +2073,86 @@ layer(HostedLive, { excludeTestServices: true })("Credential placement", (it) => ), ); + it.effect(scenarios.credentialManagedGoogleCloud.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + // Every managed local Cloud turns on the reviewed Executor for Google client with + // synthetic credentials, as Cloud's own registration. + const { clientId } = syntheticCloudGoogle; + const [fullGmail, modify, mixed, wider] = yield* Effect.all( + [ + signInGoogle("gmail", [gmail, drive, calendar]), + // Google has not approved `gmail.modify` for Executor's own registration: asked for, + // Google shows "This app is blocked" instead of its consent screen. + signInGoogle("modify", [gmailModify, drive, calendar]), + // `gmail.modify` beside a scope the client was never allowed. + signInGoogle("mixed", [gmailModify, cloudPlatform]), + // A scope the client was never allowed. + signInGoogle("wider", [calendar, cloudPlatform]), + ], + { concurrency: "unbounded" }, + ); + + // The approved full Gmail scope signs in with the client. + expect(fullGmail.setup.firstParty).toEqual({ label: "Executor for Google" }); + expect(fullGmail.started.status, JSON.stringify(fullGmail.started.body)).toBe(200); + const approved = new URL((yield* body(Redirect, fullGmail.started)).authorizationUrl); + expect(`${approved.origin}${approved.pathname}`).toBe(google.authorizationUrl); + expect(approved.searchParams.get("client_id")).toBe(clientId); + expect(approved.searchParams.get("scope")?.split(" ")).toEqual([gmail, drive, calendar]); + + // `gmail.modify` is refused before Google: no authorization URL with the client. The + // refusal leads with the user's own client, and names the full Gmail scope second, saying + // it grants more. + expect(modify.setup).toMatchObject({ + mode: "client-required", + operatorScopeRefusal: { + client: "Executor for Google", + scopes: [gmailModify], + instead: [{ scope: gmailModify, scopes: [gmail] }], + }, + }); + expect(modify.setup.firstParty).toBeUndefined(); + expect(modify.started.status, JSON.stringify(modify.started.body)).toBe(422); + expect(modify.started.body).not.toHaveProperty("authorizationUrl"); + const refused = yield* body(ScopeRefused, modify.started); + expect(refused.message).toContain(gmailModify); + expect(refused.message).toContain("Executor for Google"); + expect(refused.recovery.action).toMatch(/^Enter your own OAuth client for this service\./u); + expect(refused.recovery.action).toContain( + `if broader access is acceptable, change the app to ask for ${gmail} instead, which grants more than ${gmailModify}.`, + ); + expect(refused.recovery.instructions).toMatch(/First choice: Create an OAuth client/u); + expect(refused.recovery.instructions).toContain( + `Second choice, only if the person accepts broader access: change the provider's declared scopes to ask for ${gmail} instead of ${gmailModify}.`, + ); + expect(refused.recovery.instructions.indexOf("First choice")).toBeLessThan( + refused.recovery.instructions.indexOf(gmail), + ); + expect(JSON.stringify(modify.started.body)).not.toContain(clientId); + + // Beside a scope the client was never allowed, the refusal names both and offers only + // the user's own client, since no replacement covers them all. + expect(mixed.setup.operatorScopeRefusal).toEqual({ + client: "Executor for Google", + scopes: [gmailModify, cloudPlatform], + instead: [{ scope: gmailModify, scopes: [gmail] }], + }); + expect(mixed.started.status, JSON.stringify(mixed.started.body)).toBe(422); + const unreplaced = yield* body(ScopeRefused, mixed.started); + expect(unreplaced.message).toContain(cloudPlatform); + expect(unreplaced.recovery.action).toMatch(/^Enter your own OAuth client/u); + expect(unreplaced.recovery.action).not.toContain(gmail); + + // A scope outside the client's menu that it never asked for leaves it simply not offered. + expect(wider.setup.firstParty).toBeUndefined(); + expect(wider.setup.operatorScopeRefusal).toBeUndefined(); + expect(wider.started.body).toMatchObject({ _tag: "OAuthClientUnavailable" }); + }), + ), + ); + it.effect(scenarios.credentialManagedCloud.title, (context) => withHostedCase( context, diff --git a/e2e/tests/dashboard-batch-authentication.spec.ts b/e2e/tests/dashboard-batch-authentication.spec.ts index b4771a34d3..81006b037d 100644 --- a/e2e/tests/dashboard-batch-authentication.spec.ts +++ b/e2e/tests/dashboard-batch-authentication.spec.ts @@ -69,7 +69,7 @@ const authentication = (traceId: string, reads: number) => result.data.filter(({ span }) => span.operationName === "dashboard.batch.read").length === reads ? Effect.succeed(result.data) - : Effect.fail(new Error("The completed batch trace must reach Motel")), + : Effect.fail(new Error("The completed batch trace must reach the collector")), ), Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 40 }), Effect.timeout("25 seconds"), diff --git a/e2e/tests/database-connections.spec.ts b/e2e/tests/database-connections.spec.ts index 35edefc4e1..b0d635e80b 100644 --- a/e2e/tests/database-connections.spec.ts +++ b/e2e/tests/database-connections.spec.ts @@ -144,7 +144,7 @@ layer(NodeServices.layer, { excludeTestServices: true })("Database connections", Effect.provide(withPlanetscale), ), ); - expect(message).toContain(`PlanetScale branch test-${slug} was not confirmed deleted`); + expect(message).toContain(`PlanetScale branch test-${slug} is not confirmed gone`); expect(message).toContain("PlanetScale still answers 200"); expect(seen).toHaveLength(quickly.attempts + 1); }), diff --git a/e2e/tests/desktop-release.spec.ts b/e2e/tests/desktop-release.spec.ts index bcedef0f4f..96311b7bd6 100644 --- a/e2e/tests/desktop-release.spec.ts +++ b/e2e/tests/desktop-release.spec.ts @@ -433,7 +433,9 @@ createRoot(root).render();`, entry.span.tags["http.response.status_code"] === "200", ) ) - return yield* Effect.fail(new Error("The actual tool-call trace must reach Motel")); + return yield* Effect.fail( + new Error("The actual tool-call trace must reach the collector"), + ); }).pipe( Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 40 }), Effect.timeout("20 seconds"), diff --git a/e2e/tests/docker-release-collector-memory.spec.ts b/e2e/tests/docker-release-collector-memory.spec.ts index ff815cbc35..ec99757451 100644 --- a/e2e/tests/docker-release-collector-memory.spec.ts +++ b/e2e/tests/docker-release-collector-memory.spec.ts @@ -1,14 +1,13 @@ /** - * The telemetry collector runs in its own workerd process and shares the container's memory limit. - * Every export it stores leaves garbage in its isolate. Without memory settings, workerd does not - * collect an idle isolate or return freed memory to the system, so under a steady stream of - * scheduled apps' spans the collector grew by about 100 MB an hour to about 430 MB. Its process - * now collects the isolate once it has been idle for 2 seconds and returns what that frees. In - * seven runs of the 4 exports below it gave back about three quarters to nine tenths of what they - * took, in readings taken about 3 to 5 seconds after the last export. Six runs on beta.40 were - * read for up to a minute: one reached about 53% back, at 59 seconds, and the other five had - * given back about a fifth or less when the minute ended. Six more, read for the 20 seconds - * allowed here, gave back none in five runs and about a fifth in one. + * The telemetry collector runs in its own process and shares the container's memory limit. glibc's + * allocator keeps what a burst of exports freed resident unless it is asked to return it, so the + * collector asks once it has been idle (its README, "Memory"). Under Motel, the collector earlier + * images ran, a steady stream of scheduled apps' spans grew it by about 100 MB an hour to about + * 430 MB until its workerd process was set to collect an idle isolate. + * + * On the Rust collector the 4 exports below raise it from about 7 MB to about 32 MB (25.5 MiB in + * three runs) and it is back at 7 MB within a second of the last answer. A build that never + * returns freed memory still held 26 of the 26 MB they used 20 seconds later, in two runs. */ import { expect, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -37,12 +36,12 @@ const crypto = require("node:crypto"); (async () => { // The name is joined here so this script's own command line does not hold it. const collectors = fs.readdirSync("/proc").filter((p) => Number.isInteger(Number(p))).filter((p) => { - try { return fs.readFileSync("/proc/" + p + "/cmdline", "utf8").includes(["motel", "capnp"].join(".")); } catch { return false; } + try { return fs.readFileSync("/proc/" + p + "/cmdline", "utf8").split("\\0")[0] === "/app/" + ["executor", "collector"].join("-"); } catch { return false; } }); if (collectors.length !== 1) throw new Error("expected one collector process, found " + collectors.length); const pid = collectors[0]; const tmp = "/proc/" + pid + "/root/tmp"; - const socketPath = fs.readdirSync(tmp).filter((name) => name.startsWith("executor-host-")).map((name) => tmp + "/" + name + "/motel.sock").find((path) => fs.existsSync(path)); + const socketPath = fs.readdirSync(tmp).filter((name) => name.startsWith("executor-host-")).map((name) => tmp + "/" + name + "/collector.sock").find((path) => fs.existsSync(path)); // A process's start time is field 22 of its stat line. Another process that takes the id has another. const startTime = () => { const stat = fs.readFileSync("/proc/" + pid + "/stat", "utf8"); @@ -205,13 +204,14 @@ it.live( ); expect(burstOutput.inserted).toBe(exportCount * spansPerExport); const used = burstOutput.peak - burstOutput.before; + // For half of it coming back to mean anything, the burst must raise the collector well + // above its own resting size, about 7 MB: 16 MiB, where it measures 25.5. Motel's bound + // here was 64 MiB, which the Rust collector does not reach with these exports. expect(used, "the exports raised the collector's resident memory").toBeGreaterThan( - 64 * 1024, + 16 * 1024, ); - // With the collection after 2 seconds idle, the collector returned about three quarters of - // it or more in readings taken about 3 to 5 seconds after the last export. Without it, one - // run reached just over half at 59 seconds and five had returned about a fifth or less - // when a minute ended. + // The collector returned all of it within about a second of the last export. A build that + // does not return freed memory held all of it for the 20 seconds allowed. expect( burstOutput.after, `${Math.round(burstOutput.waitedMs / 1000)} s after the exports the collector still held ${megabytes(burstOutput.after - burstOutput.before)} of the ${megabytes(used)} MB they used`, diff --git a/e2e/tests/docker-release.spec.ts b/e2e/tests/docker-release.spec.ts index f10f5fc350..cbb43f98ff 100644 --- a/e2e/tests/docker-release.spec.ts +++ b/e2e/tests/docker-release.spec.ts @@ -958,7 +958,7 @@ http.createServer((request, response) => { ]); expect( discarded.trim(), - "Motel resets independently while product state is retained", + "the collector's telemetry resets independently while product state is retained", ).toBe("Executor: the telemetry collector answered 404"); expect((yield* run(["exec", id, "/app/workerd", "--version"])).trim()).toBe( yield* pinnedWorkerd, @@ -988,16 +988,17 @@ http.createServer((request, response) => { maxBytes: 16 * 1024 * 1024, refused: { queueFull: 0, tooLarge: 0, invalid: 0, storeFailed: 0 }, }); - // The collector runs in its own workerd process, so indexing spans no longer blocks the - // product's JavaScript thread. Stopping it leaves the product serving; the host - // restarts it after waiting a second, and the new collector stores the product's spans. + // The collector runs in its own process beside the product's workerd. Stopping it leaves + // the product serving; the host restarts it after waiting a second, and the new + // collector stores the product's spans. const collectorProcess = run([ "exec", id, "sh", "-c", - // The pattern is built at runtime so this shell's own command line does not match it. - 'm=motel; for p in /proc/[0-9]*; do case "$(cat "$p/cmdline" 2>/dev/null)" in *"$m.capnp"*) echo "${p#/proc/}";; esac; done', + // The collector's executable, the first word of its command line. The pattern is built + // at runtime so this shell's own command line does not match it. + 'm=executor-collector; for p in /proc/[0-9]*; do case "$(cat "$p/cmdline" 2>/dev/null)" in "/app/$m"*) echo "${p#/proc/}";; esac; done', ]).pipe(Effect.map((output) => output.trim())); const firstCollector = yield* collectorProcess; expect(firstCollector).toMatch(/^\d+$/); @@ -1051,7 +1052,7 @@ http.createServer((request, response) => { id, "sh", "-c", - "test ! -e /usr/local/bin/bun && test ! -L /usr/local/bin/bun && test ! -e /app/motel/bun", + "test ! -e /usr/local/bin/bun && test ! -L /usr/local/bin/bun", ]), ), "the image contains no Bun runtime", @@ -1615,7 +1616,8 @@ export default defineApp({}, async () => ({ .filter((fields) => /^[0-9A-F]+:[0-9A-F]{4}$/.test(fields[1] ?? "")), ), ); - // Every TCP port listening in the container's network namespace, and Motel's old port. + // Every TCP port listening in the container's network namespace, and the port the collector + // served on in earlier images. const listening = (yield* tcpTables) .filter((fields) => fields[3] === "0A") .map((fields) => Number.parseInt(fields[1]!.split(":")[1]!, 16)); @@ -1661,7 +1663,7 @@ export default defineApp({}, async () => ({ ).not.toContain("maxPending"); const written = yield* probe(`${base}/v1/traces`, injected); if (listener === 4318) { - expect(read, "nothing listens on Motel's old port").toMatch(/^refused /); + expect(read, "nothing listens on the collector's old port").toMatch(/^refused /); expect(counters).toMatch(/^refused /); expect(written).toMatch(/^refused /); } @@ -1682,7 +1684,7 @@ export default defineApp({}, async () => ({ ])).trim(), "no app wrote spans into the collector", ).toBe("Executor: the telemetry collector answered 404"); - // Motel's own process holds its Unix socket and no TCP socket. The image has no tools to + // The collector's own process holds its Unix socket and no TCP socket. The image has no tools to // read another process's descriptors, so a client in its process namespace reads them as // the same user. const collectorSockets = yield* run([ @@ -1697,7 +1699,7 @@ export default defineApp({}, async () => ({ "-e", `const fs = require("node:fs"); const pid = fs.readdirSync("/proc").filter((p) => /^\\d+$/.test(p)).find((p) => { - try { return fs.readFileSync("/proc/" + p + "/cmdline", "utf8").includes(["motel", "capnp"].join(".")); } catch { return false; } + try { return fs.readFileSync("/proc/" + p + "/cmdline", "utf8").split("\\0")[0] === "/app/" + ["executor", "collector"].join("-"); } catch { return false; } }); const inodes = fs.readdirSync("/proc/" + pid + "/fd").flatMap((fd) => { try { return [fs.readlinkSync("/proc/" + pid + "/fd/" + fd)]; } catch { return []; } @@ -1721,7 +1723,7 @@ process.stdout.write(JSON.stringify({ ); expect(collectorSockets.tcp, "the collector has no TCP socket").toEqual([]); expect(collectorSockets.unix).toContainEqual( - expect.stringMatching(/^\/tmp\/executor-host-\d+\/motel\.sock$/), + expect.stringMatching(/^\/tmp\/executor-host-\d+\/collector\.sock$/), ); }), ).pipe(Effect.provide(NodeServices.layer)), diff --git a/e2e/tests/durable-evaluated-results.spec.ts b/e2e/tests/durable-evaluated-results.spec.ts index 3d6ad57f3b..ffc3ebe40c 100644 --- a/e2e/tests/durable-evaluated-results.spec.ts +++ b/e2e/tests/durable-evaluated-results.spec.ts @@ -200,7 +200,7 @@ const listingApp = (files: string = listingSource(0)) => const listed = (names: ReadonlyArray) => names.includes("sdk.tools.listing"); /** - * A read that evaluated. The evaluation ends after the response, so its spans can reach Motel + * A read that evaluated. The evaluation ends after the response, so its spans can reach the collector * before the request's own: the read waits for the listing span, which holds the cache outcome. */ const evaluates = (names: ReadonlyArray) => diff --git a/e2e/tests/emulator-failures.spec.ts b/e2e/tests/emulator-failures.spec.ts index 9c7c8f1fa9..c9790da78a 100644 --- a/e2e/tests/emulator-failures.spec.ts +++ b/e2e/tests/emulator-failures.spec.ts @@ -7,11 +7,24 @@ */ import { expect, layer } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { Cause, Clock, Duration, Effect, Exit, Fiber, FileSystem, Option, Tracer } from "effect"; +import { + Cause, + Clock, + Duration, + Effect, + Exit, + Fiber, + FileSystem, + Layer, + Option, + Stream, + Tracer, +} from "effect"; import { FetchHttpClient, HttpClient } from "effect/http"; import { createHash } from "node:crypto"; import { createServer } from "node:http"; import type { AddressInfo } from "node:net"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { armedFaultHeader, emulatorRequest, @@ -294,6 +307,123 @@ const onTestClock = (call: Effect.Effect) => return { exit: yield* Effect.exit(Fiber.join(fiber)), elapsed }; }); +/** One request a call sent, when it started on the call's clock, and how it ended. */ +interface TimedAttempt { + readonly route: string; + readonly start: number; + ended?: "answered" | "failed" | "cut off"; +} +/** + * An HTTP client that notes when each request starts on the call's clock and how it ends. A + * request starts in the same synchronous step in which the policy decided to send it, so its start + * is the time of that decision. + */ +const timedClient = (attempts: Array) => + Layer.effect( + HttpClient.HttpClient, + Effect.map(HttpClient.HttpClient, (client) => + HttpClient.transform(client, (response, request) => + Effect.gen(function* () { + const attempt: TimedAttempt = { + route: `${request.method} ${new URL(request.url).pathname}`, + start: yield* Clock.currentTimeMillis, + }; + attempts.push(attempt); + return yield* response.pipe( + Effect.onExit((exit) => + Effect.sync(() => { + attempt.ended = Exit.isSuccess(exit) + ? "answered" + : Cause.hasInterrupts(exit.cause) + ? "cut off" + : "failed"; + }), + ), + ); + }), + ), + ), + ).pipe(Layer.provide(FetchHttpClient.layer)); + +/** + * The clock a call runs on, noting the call's first reading of the time and every sleep it + * schedules, with when and for how long. The policy reads the time first as it starts (its 30 s + * runs from there), and each attempt's time-out is a sleep scheduled as the attempt starts, for + * the time-out the policy chose. So both are read where the policy decides them, not after any + * work that takes real time while the test clock moves on. + */ +const notingClock = (clock: Clock.Clock) => { + const noted = { + first: undefined as number | undefined, + sleeps: [] as Array<{ at: number; ms: number }>, + }; + const millis = () => { + const now = clock.currentTimeMillisUnsafe(); + noted.first ??= now; + return now; + }; + const noting: Clock.Clock = { + currentTimeMillisUnsafe: millis, + currentTimeMillis: Effect.sync(millis), + currentTimeNanosUnsafe: () => clock.currentTimeNanosUnsafe(), + currentTimeNanos: clock.currentTimeNanos, + monotonicTimeNanosUnsafe: () => clock.monotonicTimeNanosUnsafe(), + monotonicTimeNanos: clock.monotonicTimeNanos, + sleep: (duration) => + Effect.suspend(() => { + noted.sleeps.push({ at: clock.currentTimeMillisUnsafe(), ms: Duration.toMillis(duration) }); + return clock.sleep(duration); + }), + }; + return { + noted, + run: (call: Effect.Effect) => + Effect.provideService(call, Clock.Clock, noting), + }; +}; + +/** + * The deadline as facts the policy decided: it runs from the call's own start, every attempt + * started before it, and as each attempt started a time-out of exactly the smaller of 10 s and + * what was left was scheduled for it. Every attempt was then cut off by it. And every sleep the + * call scheduled, time-outs and the spacing between attempts alike, ends by the deadline, so the + * call itself does: nothing the policy waits for runs past it. + */ +const withinDeadline = ( + attempts: ReadonlyArray, + noted: { + readonly first: number | undefined; + readonly sleeps: ReadonlyArray<{ at: number; ms: number }>; + }, +) => { + const started = noted.first ?? 0; + const deadline = started + 30_000; + const listed = JSON.stringify({ + attempts: attempts.map(({ route, start, ended }) => ({ route, start: start - started, ended })), + sleeps: noted.sleeps.map(({ at, ms }) => ({ at: at - started, ms })), + }); + for (const sleep of noted.sleeps) + expect( + sleep.at + sleep.ms, + `a sleep of ${sleep.ms} ms scheduled at ${sleep.at - started} ms ends after the deadline: ${listed}`, + ).toBeLessThanOrEqual(deadline); + for (const attempt of attempts) { + const timeOuts = noted.sleeps.filter(({ at }) => at === attempt.start).map(({ ms }) => ms); + const left = deadline - attempt.start; + expect(attempt.start, `an attempt started at or after the deadline: ${listed}`).toBeLessThan( + deadline, + ); + expect(timeOuts, `an attempt's time-out was not cut to what was left: ${listed}`).toContain( + Math.min(10_000, left), + ); + expect( + timeOuts.every((ms) => ms <= left), + `a time-out reached past the deadline: ${listed}`, + ).toBe(true); + expect(attempt.ended, `an attempt was not cut off by its time-out: ${listed}`).toBe("cut off"); + } +}; + // The retry cases run on the real clock: the spacing between attempts is real time. layer(NodeServices.layer, { excludeTestServices: true })("Emulator failures", (it) => { it.effect("a failure report keeps only emulate's diagnostic fields", () => @@ -614,6 +744,9 @@ layer(NodeServices.layer)("Emulator retry policy", (it) => { Effect.gen(function* () { const { origin, hits } = yield* stalling; const directory = yield* recordsDirectory; + const attempts: Array = []; + const client = timedClient(attempts); + const clock = yield* Clock.Clock; const shipped = (base: string, payload: unknown, applied?: Effect.Effect) => emulatorRequest( `${origin}${base}`, @@ -622,37 +755,97 @@ layer(NodeServices.layer)("Emulator retry policy", (it) => { token, undefined, applied === undefined ? undefined : { applied }, - ).pipe( - Effect.provideService(EmulatorRetryRecords, { directory }), - Effect.provide(FetchHttpClient.layer), - ); + ).pipe(Effect.provideService(EmulatorRetryRecords, { directory }), Effect.provide(client)); // Three attempts of 10, 10 and the 6 s left, 1 s then 3 s apart: 30 s in all. const google = `/google/stalled-${suffix}`; - const plain = yield* onTestClock(shipped(google, { users: [{ email }] })); + const plainClock = notingClock(clock); + const plain = yield* onTestClock(plainClock.run(shipped(google, { users: [{ email }] }))); expect(hits.get(`POST ${google}/_emulate/seed`)).toBe(3); - expect(plain.elapsed).toBeLessThanOrEqual(30_000); + expect(attempts.map(({ route }) => route)).toEqual( + Array.from({ length: 3 }, () => `POST ${google}/_emulate/seed`), + ); + withinDeadline(attempts, plainClock.noted); const error = Option.getOrThrow(Exit.findErrorOption(plain.exit)); expect(error.message).toContain("after 3 attempts"); expect(error.reason).toBe("timeout"); // The read back has only the 19 s left after the first attempt and its spacing; it stalls // too, so whether the seed applied stays unknown and it is not sent again. + attempts.length = 0; const workos = `/workos/stalled-${suffix}`; const readBack = emulatorRequest(`${origin}${workos}`, "/user_management/users").pipe( Effect.as(true), Effect.orElseSucceed(() => false), - Effect.provide(FetchHttpClient.layer), + Effect.provide(client), ); const seed = { users: [{ email }], organizations: [{ name: "V1 stalled", members: [email] }], }; - const organization = yield* onTestClock(shipped(workos, seed, readBack)); + const organizationClock = notingClock(clock); + const organization = yield* onTestClock( + organizationClock.run(shipped(workos, seed, readBack)), + ); expect(hits.get(`POST ${workos}/_emulate/seed`)).toBe(1); expect(hits.get(`GET ${workos}/user_management/users`)).toBeGreaterThanOrEqual(1); - expect(organization.elapsed).toBeLessThanOrEqual(30_000); + expect(attempts[0]?.route).toBe(`POST ${workos}/_emulate/seed`); + expect( + attempts.slice(1).every(({ route }) => route === `GET ${workos}/user_management/users`), + ).toBe(true); + withinDeadline(attempts, organizationClock.noted); expect(Exit.isFailure(organization.exit)).toBe(true); }), ); + + it.effect("every harness process that calls out waits 2.5 s per address, not Node's 250 ms", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + // Node's default on the pinned Node, which made a slow two-address connect fail in ~500 ms. + const read = (code: string) => + Effect.gen(function* () { + const child = yield* processes.spawn( + ChildProcess.make(process.execPath, ["--input-type=module", "-e", code]), + ); + const [out] = yield* Effect.all( + [child.stdout.pipe(Stream.decodeText(), Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + return Number(out.trim()); + }).pipe(Effect.scoped); + const value = `import net from "node:net"; console.log(net.getDefaultAutoSelectFamilyAttemptTimeout());`; + expect(yield* read(value)).toBe(250); + expect(yield* read(`import "./e2e/support/connect-timeout.ts"; ${value}`)).toBe(2_500); + // Each entry point that makes outbound calls imports it before anything else, and so does + // every setup file of the Vitest config the targets run under, so each worker has it. + const first = (text: string) => text.match(/^import (?:[^;]*? from )?"([^"]+)";/m)?.[1]; + const callers: Array = []; + for (const name of (yield* fs.readDirectory("e2e")).toSorted()) { + if (!name.endsWith(".ts")) continue; + const text = yield* fs.readFileString(`e2e/${name}`); + if ( + /NodeRuntime\.runMain/.test(text) && + /FetchHttpClient|runSuite\(|runDeployedSuite|startDeployment/.test(text) + ) { + callers.push(`e2e/${name}`); + expect(first(text), name).toBe("./support/connect-timeout.ts"); + } + } + expect(callers).toEqual([ + "e2e/cleanup-deployed.ts", + "e2e/cli.ts", + "e2e/create-emulators.ts", + "e2e/deployed.ts", + "e2e/run.ts", + ]); + const config = yield* fs.readFileString("e2e/vitest.config.ts"); + const setupFiles = [...config.matchAll(/setupFiles: \[([^\]]*)\]/g)].flatMap((match) => + [...match[1]!.matchAll(/"([^"]+)"/g)].map((file) => file[1]!), + ); + expect(setupFiles.length).toBeGreaterThan(0); + for (const file of setupFiles) + expect(first(yield* fs.readFileString(file)), file).toBe("./support/connect-timeout.ts"); + }), + ); }); diff --git a/e2e/tests/failure-text.spec.ts b/e2e/tests/failure-text.spec.ts index cc2be0a073..171d3d9a12 100644 --- a/e2e/tests/failure-text.spec.ts +++ b/e2e/tests/failure-text.spec.ts @@ -43,6 +43,19 @@ const PageErrors = Schema.Array( Schema.Struct({ kind: Schema.String, page: Schema.String, text: Schema.String }), ); +const PageRequests = Schema.Array( + Schema.Struct({ + method: Schema.String, + url: Schema.String, + kind: Schema.String, + startedAt: Schema.String, + status: Schema.optionalKey(Schema.Number), + respondedAt: Schema.optionalKey(Schema.String), + endedAt: Schema.optionalKey(Schema.String), + error: Schema.optionalKey(Schema.String), + }), +); + /** The evidence files a case writes as text; screenshots, videos and the trace are binary. */ const textFile = /\.(?:json|ndjson|txt|log|md)$/u; @@ -217,6 +230,30 @@ layer(TestLive, { excludeTestServices: true })("Failure text", (it) => { text: "Console saw ", }, ]); + // Every request a page sent, as method, origin and path, status and times: never the query + // or fragment that held the planted token, never a header or cookie. The scan above + // checks this file for every planted value too; a missing or empty record fails here. + const requests = yield* read( + PageRequests, + path.join(consoleErrors, "page-requests.json"), + ); + expect(requests).toEqual([ + { + method: "GET", + url: "http://failure-text.test/console", + kind: "document", + startedAt: expect.any(String), + status: 200, + respondedAt: expect.any(String), + endedAt: expect.any(String), + }, + ]); + const [document] = requests; + const times = [document?.startedAt, document?.respondedAt, document?.endedAt].map((at) => + Date.parse(at ?? ""), + ); + expect(times.every(Number.isFinite), JSON.stringify(document)).toBe(true); + expect(times.toSorted((a, b) => a - b)).toEqual(times); yield* reports(failureTextCases.filled, "Fill a disabled field", [ "TimeoutError: locator.fill: Timeout 500ms exceeded.", "- waiting for getByLabel('Key')", diff --git a/e2e/tests/fixture-removal-settlement.spec.ts b/e2e/tests/fixture-removal-settlement.spec.ts index 714def2da6..73dd708af5 100644 --- a/e2e/tests/fixture-removal-settlement.spec.ts +++ b/e2e/tests/fixture-removal-settlement.spec.ts @@ -220,11 +220,15 @@ layer(HostedLive, { excludeTestServices: true })("Fixture removal settlement", ( { concurrency: 4, discard: true }, ), ); - // The blocked removal's owner row stays locked against deletion; the product's own - // erasure never deletes users, so it still finishes. + // The blocked removal's owner row stays locked against deletion and nothing else: `for + // key share` conflicts only with the lock a `DELETE` takes. The organization's team + // installation may still be setting up this owner, and its member setup reads the owner + // `for share` while it holds the organization row (organization-defaults.ts); a stronger + // lock here held that setup, and with it the erasure's delete of the organization row + // (CI #13622). The product's own erasure never deletes users, so it still finishes. const [owner] = yield* locks .hold({ - sql: `select id from "user" where email = $1 for no key update`, + sql: `select id from "user" where email = $1 for key share`, params: [`agent-${blocked}-owner@example.test`], }) .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(Schema.Unknown)))); @@ -237,7 +241,26 @@ layer(HostedLive, { excludeTestServices: true })("Fixture removal settlement", ( }), ); - const looks: Array<{ ms: number; finished: number; checking: boolean }> = []; + const looks: Array<{ + ms: number; + finished: number; + blocked: { phase: string; status: string | null; checking: boolean } | null; + }> = []; + /** The blocked removal as one look reports it: never finished while its users are locked. */ + const blockedIn = (look: typeof SettleLook.Type) => { + expect( + look.finished.map((removal) => removal.scenario), + "the blocked removal is reported finished while its owner row is locked", + ).not.toContain(blocked); + const pending = look.unfinished.find((removal) => removal.scenario === blocked); + return pending === undefined + ? null + : { phase: pending.phase, status: pending.status, checking: pending.checking }; + }; + // The product finishes the erasures in no promised order: one whose purge raced its + // organization's team installation is retried seconds later (`purge-owner-store`). So + // the looks go on until the other 23 have finished and the blocked one's erasure is done + // too, leaving it held only by its users' deletion. const others = yield* evidence.step( "Every look answers within its limit and the others finish while one is blocked", Effect.gen(function* () { @@ -247,24 +270,45 @@ layer(HostedLive, { excludeTestServices: true })("Fixture removal settlement", ( const look = yield* fixtureRequest(fixtures, "/settle", { ids }, "15 seconds").pipe( Effect.flatMap(Schema.decodeUnknownEffect(SettleLook)), ); - const pending = look.unfinished.find((removal) => removal.scenario === blocked); + const state = blockedIn(look); looks.push({ ms: (yield* Clock.currentTimeMillis) - asked, finished: look.finished.length, - checking: pending?.checking ?? false, + blocked: state, }); - if (look.finished.length === many - 1) return look; + if ( + look.finished.length === many - 1 && + state?.phase === "remove-users" && + state.status === "done" && + state.checking + ) + return look; yield* Effect.sleep("1 second"); } - }).pipe(Effect.timeout("90 seconds")), + }).pipe( + Effect.timeoutOrElse({ + duration: "90 seconds", + orElse: () => + Effect.die( + new Error( + `The other removals and the blocked one's erasure did not finish within 90 s; last looks: ${JSON.stringify(looks.slice(-3))}`, + ), + ), + }), + ), ); yield* evidence.json("looks.json", looks); expect(others.finished.map((removal) => removal.scenario).sort()).toEqual( ids.slice(1).sort(), ); - expect(others.unfinished).toMatchObject([ - { scenario: blocked, phase: "remove-users", status: "done", checking: true }, - ]); + expect( + others.unfinished.map(({ scenario, phase, status, checking }) => ({ + scenario, + phase, + status, + checking, + })), + ).toEqual([{ scenario: blocked, phase: "remove-users", status: "done", checking: true }]); yield* locks.release; const files = { diff --git a/e2e/tests/framework-discovery.spec.ts b/e2e/tests/framework-discovery.spec.ts index 9240df91f4..83eef62a9a 100644 --- a/e2e/tests/framework-discovery.spec.ts +++ b/e2e/tests/framework-discovery.spec.ts @@ -247,8 +247,10 @@ return { found, described: await tools.search.describe({ paths: found.items.map( const intro = yield* Schema.decodeUnknownEffect(Document)(entry.structuredContent); expect(intro.content).toContain("`code-mode`"); expect(intro.content).toContain("`app-authoring`"); - // The intro only points at media; the code-mode skill it names holds the detail. - expect(intro.content).toContain("see images and hear audio"); + // The intro only points at media; the code-mode skill it names holds the detail. Pinned + // to what it must say (how to see images and hear audio a tool returns), not one wording: + // the list may grow, as it did with files. + expect(intro.content).toMatch(/see images\b[^.]*\bhear audio\b/); const codeMode = yield* client.use("Follow the intro to media guidance", (client, signal) => client.callTool( { name: "skills", arguments: { app: "executor", name: "code-mode" } }, @@ -258,10 +260,10 @@ return { found, described: await tools.search.describe({ paths: found.items.map( ); const media = (yield* Schema.decodeUnknownEffect(Document)(codeMode.structuredContent)) .content; - expect(media).toContain("## See images and hear audio"); - expect(media).toContain( - 'return { type: "image", mimeType: "image/png", data: file.base64 };', - ); + // A section of its own on media, and an example that returns a tool's bytes from + // `execute` as a block the client receives, not as base64 text. + expect(media).toMatch(/^## See images\b.*\bhear audio\b/m); + expect(media).toMatch(/return \{\s*type: "(?:image|audio|resource)"/); const router = yield* Schema.decodeUnknownEffect(Document)(guide.structuredContent); expect(router.content).toContain("[ui.md](ui.md)"); const topic = yield* client.use("Follow the pinned UI topic", (client, signal) => diff --git a/e2e/tests/hosted.spec.ts b/e2e/tests/hosted.spec.ts index ff4f14e328..aefd126c31 100644 --- a/e2e/tests/hosted.spec.ts +++ b/e2e/tests/hosted.spec.ts @@ -39,8 +39,8 @@ const submitStatements = 36; /** * The first saves, made one at a time and read before the others start, so each finds exactly as * many accounts already linked as saves before it. Their spans are complete only while the server - * exports at its own pace: the four writers send spans faster than the server's single Motel - * stores them, and the exporter the self-host server has always shipped drops what it holds, and + * exports at its own pace: the four writers sent spans faster than Motel, the server's collector + * before the Rust one, stored them (not measured since), and the exporter the self-host server has always shipped drops what it holds, and * every span for 60 seconds, once an export fails after its retries. A trace read during or just * after the other saves may never arrive. That a later save does the same work is checked in * webhook-source-lookup.spec.ts, by the query plans of a save beside 100,000 accounts. @@ -332,7 +332,7 @@ layer(HostedLive, { excludeTestServices: true })("Self-host", (it) => { entry.span.tags["http.response.status_code"] === "200", ) ? Effect.succeed(value) - : Effect.fail(new Error("The actual HTTP server span must reach Motel")), + : Effect.fail(new Error("The actual HTTP server span must reach the collector")), ), ); const result = yield* delivered.pipe( @@ -342,7 +342,7 @@ layer(HostedLive, { excludeTestServices: true })("Self-host", (it) => { ); expect( Result.isSuccess(result), - "The actual HTTP server span must reach Motel within 20 seconds", + "The actual HTTP server span must reach the collector within 20 seconds", ).toBe(true); if (Result.isFailure(result)) return yield* Effect.fail(result.failure); const queries = result.success.data.filter( diff --git a/e2e/tests/idle-poll-traces.spec.ts b/e2e/tests/idle-poll-traces.spec.ts index d91d08c4eb..876cc9d6d3 100644 --- a/e2e/tests/idle-poll-traces.spec.ts +++ b/e2e/tests/idle-poll-traces.spec.ts @@ -157,9 +157,13 @@ layer(TestLive, { excludeTestServices: true })("Idle poll traces", (it) => { yield* legacyStorage(statements); const from = yield* Clock.currentTimeMillis; yield* serverControl("start"); - // Each poll runs every one or two seconds and the product exports every second. - return yield* exported(collector, from, Number.MAX_SAFE_INTEGER).pipe( - Effect.map(failedRoots), + // Each poll runs every one or two seconds and the product exports every second. Each + // expected operation is its own search: one over every span the restarted server + // exports matches more than the collector answers at once. + return yield* Effect.forEach(Object.keys(expected), (operation) => + exported(collector, from, Number.MAX_SAFE_INTEGER, operation), + ).pipe( + Effect.map((found) => failedRoots(found.flat())), Effect.repeat({ schedule: Schedule.spaced("1 second"), until: (found) => diff --git a/e2e/tests/large-openapi-app.spec.ts b/e2e/tests/large-openapi-app.spec.ts index 693bfea72a..8ceb745ca7 100644 --- a/e2e/tests/large-openapi-app.spec.ts +++ b/e2e/tests/large-openapi-app.spec.ts @@ -10,6 +10,13 @@ * A document whose resources expand into one another, as Stripe's do, gives every tool all of * its definitions. Its catalog written tool by tool is larger than any string, yet it lists in * pages no larger than their bound. + * + * Linked through several accounts, as official catalog apps link theirs with `accountRouter`, the + * same document lists every tool once, with one input branch naming every account whose schema is + * the same, and each tool's schemas still complete. Merging the accounts' tools reads each shared + * definition once for the listing: before, it wrote every tool's definitions, which cost a cold + * listing of this document most of a Cloud app Worker's CPU time. The app counts the JSON text its + * isolate writes while it lists, so that cost fails the scenario on any machine, however fast. */ import { expect, layer } from "@effect/vitest"; import { Effect, Schema } from "effect"; @@ -22,6 +29,7 @@ import { App } from "../support/contracts.ts"; import { Evidence } from "../support/evidence.ts"; import { Target } from "../support/platform.ts"; import { appsManifest } from "../support/apps-release.ts"; +import { connectionsPath } from "../support/app-accounts.ts"; import { expandableOpenapiDocument, largeOpenapiDocument } from "../support/large-openapi.ts"; import { McpClient } from "../support/mcp-client.ts"; @@ -32,6 +40,56 @@ export default defineApp({}, async ({ cache, fetch, signal }) => ({ tools: liveOpenapiRouter({ ...configuration, cache, fetch, ...(signal === undefined ? {} : { signal }) }), }));`; +/** + * The most JSON text, in characters, the accounts app's isolate may write in one evaluation: its + * cold listing compiles the document, merges the accounts' tools and answers with the catalog. + * Counting text rather than time makes the bound the same on every machine and runner. Measured + * on this document with three accounts (2026-11-12, the framework run in Bun as the app runs it): + * merging each shared definition once wrote 78 million characters, and writing every tool's + * schemas, as merging did before, 9.0 billion. The bound is about ten times the first and a ninth + * of the second. + */ +const listingJsonBudget = 800_000_000; + +/** + * The same document behind a key provider, merged across every linked account as catalog apps do. + * `JSON.stringify` counts what it writes from the start of each evaluation and throws past the + * budget, so the listing fails with the count. + */ +const accountsIndex = `import { accountRouter, defineApp, defineProvider, object, router, secrets, string } from "apps"; +import { liveOpenapiRouter } from "apps/openapi"; +import configuration from "./openapi.json"; +const write = JSON.stringify; +let written = 0; +JSON.stringify = function (...args: Parameters) { + const text: string | undefined = write.apply(JSON, args); + written += text?.length ?? 0; + if (written > ${listingJsonBudget}) + throw new Error("The listing wrote " + written + " characters of JSON, over its budget of ${listingJsonBudget}."); + return text; +} as typeof JSON.stringify; +const service = defineProvider({ + name: "Expandable service", + auth: { key: secrets({ label: "Key", fields: object({ token: string({ minLength: 1 }) }) }) }, +}); +export default defineApp( + { user: { accounts: { service: service.many() } } }, + async ({ user, cache, fetch, signal }) => { + written = 0; + return { + tools: router( + await accountRouter( + user?.accounts.service ?? [], + (account) => liveOpenapiRouter({ ...configuration, cache, fetch, signal, account }), + { signal }, + ), + ), + }; + }, +);`; +/** Accounts the merged app links: several, as some organizations link to one catalog app. */ +const linkedAccounts = 3; + /** Prose with Latin-1, punctuation, symbols and an emoji, as vendor API descriptions use. */ const prose = "Returns the service’s “state” — café → ✅ or 🔴."; @@ -83,6 +141,21 @@ const ToolPage = Schema.Struct({ next: Schema.optional(Schema.String), }); const ToolIndex = Schema.Struct({ items: Schema.Array(Schema.Struct({ name: Schema.String })) }); +/** A merged tool: its input is a union of account branches, each naming the accounts it serves. */ +const AccountsTool = Schema.Struct({ + name: Schema.String, + inputSchema: Schema.Struct({ + anyOf: Schema.Array( + Schema.Struct({ + properties: Schema.Struct({ + accountId: Schema.Struct({ enum: Schema.optional(Schema.Array(Schema.String)) }), + }), + }), + ), + }), + outputSchema: Schema.optional(JsonSchemaDocument), +}); +const AccountsToolPage = Schema.Struct({ items: Schema.Array(AccountsTool) }); const SearchResult = Schema.Struct({ items: Schema.Array(Schema.Struct({ path: Schema.String })) }); /** Every `#/$defs/` name a JSON value references, anywhere in it. */ @@ -270,6 +343,106 @@ layer(HostedLive, { excludeTestServices: true })("Large OpenAPI app", (it) => { { timeout: 360_000 }, ); + it.effect( + scenarios.largeOpenapiAccountsApp.title, + (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors, + evidence = yield* Evidence; + const origin = (yield* Target).metadata.origin; + const configuration = JSON.stringify({ + source: { + document: expandableOpenapiDocument( + origin, + expandableResources, + expandableOperations, + ), + }, + allowedOrigin: origin, + securitySchemes: {}, + methods: {}, + oauth: [], + }); + const organization = `/api/organizations/${actors.organization.id}`; + const deployed = yield* api.request(actors.owner, "POST", `${organization}/apps/deploy`, { + name: `Expandable accounts ${randomUUID().slice(0, 8)}`, + files: [ + { path: "index.ts", content: accountsIndex }, + { path: "openapi.json", content: configuration }, + appsManifest, + ], + }); + expect(deployed.status, JSON.stringify(deployed.body).slice(0, 2000)).toBe(200); + const app = yield* body(App, deployed); + const path = `${organization}/apps/${app.id}`; + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", path).pipe(Effect.orDie), + ); + + // Several accounts of the owner's, each linked to the app's user requirement. + const accounts: string[] = []; + for (let index = 0; index < linkedAccounts; index++) { + const started = yield* api.request( + actors.owner, + "POST", + connectionsPath(path, "user", "service"), + {}, + ); + expect(started.status, JSON.stringify(started.body)).toBe(200); + const connection = (yield* body(Schema.Struct({ id: Schema.String }), started)).id; + const submitted = yield* api.request( + actors.owner, + "POST", + `${organization}/connections/${connection}/submit`, + { method: "key", label: `Key ${index + 1}`, fields: { token: `key-${index + 1}` } }, + ); + expect(submitted.status, JSON.stringify(submitted.body)).toBe(200); + accounts.push((yield* body(Schema.Struct({ id: Schema.String }), submitted)).id); + } + + // The cold listing evaluates the catalog for every account and merges their tools within + // the app's JSON budget. Its time is evidence only: it depends on the machine. + const started = Date.now(); + const first = yield* api.request(actors.owner, "GET", `${path}/tools`); + const firstText = JSON.stringify(first.body); + yield* evidence.json("expandable-accounts-first-page.json", { + status: first.status, + characters: firstText.length, + milliseconds: Date.now() - started, + body: first.status === 200 ? undefined : first.body, + }); + expect(first.status, firstText.slice(0, 2000)).toBe(200); + expect(firstText.length).toBeLessThanOrEqual(pageChars); + const page = yield* body(AccountsToolPage, first); + const resources = page.items.filter((tool) => tool.name.startsWith("resources.")); + expect(resources.length).toBeGreaterThan(0); + for (const tool of resources) { + // The accounts' input schemas are the same, so one branch names every account. + const branches = tool.inputSchema.anyOf; + expect(branches).toHaveLength(1); + expect([...(branches[0]?.properties.accountId.enum ?? [])].sort()).toEqual( + [...accounts].sort(), + ); + // Each output schema still holds every definition it reaches, once. + expect(ownDefinitions(tool.outputSchema)).toHaveLength(expandableResources); + } + + // Every tool is listed once, whichever account lists it. + const listed = yield* body( + ToolIndex, + yield* api.request(actors.owner, "GET", `${path}/tools/index`), + ); + expect(listed.items).toHaveLength(expandableOperations + 1); + expect(new Set(listed.items.map((tool) => tool.name)).size).toBe(listed.items.length); + }), + ), + // Deploying the document and listing it cold, as the expandable scenario does. + { timeout: 240_000 }, + ); + it.effect(scenarios.largeOpenapiApp.title, (context) => withHostedCase( context, diff --git a/e2e/tests/live-openapi-import.spec.ts b/e2e/tests/live-openapi-import.spec.ts index e94f08bbb2..ddf9b77089 100644 --- a/e2e/tests/live-openapi-import.spec.ts +++ b/e2e/tests/live-openapi-import.spec.ts @@ -8,9 +8,12 @@ import { clientCredentialsScheme, credentialEchoUpstream, deployPublicApp, + hujsonPolicy, invalidSchemaDocument, + jsonAnswers, jsonLines, malformedSwaggerDocument, + mediaTypeUpstream, openapi32Upstream, otherHostDocument, requestEchoUpstream, @@ -765,6 +768,80 @@ export default defineApp({}, async ({ cache, fetch, signal }) => ({ ), ); + it.effect(scenarios.liveOpenapiHujson.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const upstream = yield* mediaTypeUpstream; + const app = yield* deployPublicApp(`import { defineApp } from "apps"; +import { liveOpenapiRouter } from "apps/openapi"; +export default defineApp({}, async ({ cache, fetch, signal }) => ({ + tools: liveOpenapiRouter({ cache, fetch, signal, + source: { document: ${JSON.stringify(upstream.document)} }, + allowedOrigin: ${JSON.stringify(upstream.origin)}, + securitySchemes: {}, methods: {}, oauth: [] }), +})); +`); + const call = (tool: string, input: unknown) => + app + .call(tool, "mutation", input) + .pipe( + Effect.tap((response) => + Effect.sync(() => expect(response.status, JSON.stringify(response.body)).toBe(200)), + ), + ); + const set = (input: unknown) => call("policy.setPolicyFile", input); + // HuJSON is text: the policy goes out as written, comment and trailing commas included. + yield* set({ contentType: "application/hujson", body: hujsonPolicy }); + // JSON, the first declared type, and a +json type are still sent as JSON. + yield* set({ body: { grants: [] } }); + yield* set({ contentType: "application/merge-patch+json", body: { grants: null } }); + // AWS's JSON protocols and a +json type outside `application` take their declared object + // and answer it as parsed JSON, not as a base64 file. + const json = [ + ["aws.listTables", "/aws/json-1.0", { Limit: 1 }], + ["aws.listStreams", "/aws/json-1.1", { Limit: 2 }], + ["scene.putScene", "/scene", { scenes: [{ nodes: [0] }] }], + ] as const; + for (const [tool, path, input] of json) { + const answered = yield* call(tool, { body: input }); + expect(answered.body).toEqual(jsonAnswers[path].body); + } + expect(yield* upstream.requests).toEqual([ + { + method: "POST", + url: "/policy", + contentType: "application/hujson", + body: hujsonPolicy, + }, + { + method: "POST", + url: "/policy", + contentType: "application/json", + body: '{"grants":[]}', + }, + { + method: "POST", + url: "/policy", + contentType: "application/merge-patch+json", + body: '{"grants":null}', + }, + ...json.map(([, path, input]) => ({ + method: "POST", + url: path, + contentType: jsonAnswers[path].contentType, + body: JSON.stringify(input), + })), + ]); + + // A HuJSON answer comes back as its text, not parsed as JSON or returned as a file. + const read = yield* app.call("policy.getPolicyFile", "query", {}); + expect(read.status, JSON.stringify(read.body)).toBe(200); + expect(yield* body(Schema.String, read)).toBe(hujsonPolicy); + }), + ), + ); + it.effect(scenarios.liveOpenapiNaming.title, (context) => withHostedCase( context, diff --git a/e2e/tests/local-app-account-app-links.spec.ts b/e2e/tests/local-app-account-app-links.spec.ts index cfe00f5a4b..6c495d3106 100644 --- a/e2e/tests/local-app-account-app-links.spec.ts +++ b/e2e/tests/local-app-account-app-links.spec.ts @@ -57,6 +57,18 @@ const keyedProvider = (name: string) => const serviceProvider = keyedProvider("App service fixture"); const mailProvider = keyedProvider("Linked mail fixture"); +/** The links in the dashboard's overview; the provider is left out. */ +const Overview = Schema.Struct({ + links: Schema.Array(Schema.Struct({ app: Schema.String, account: Schema.String })), + appLinks: Schema.Array( + Schema.Struct({ + app: Schema.String, + requirement: Schema.String, + account: Schema.String, + label: Schema.String, + }), + ), +}); const Completed = Schema.Struct({ status: Schema.Literal("completed"), value: Schema.Unknown }); const AccountRequired = Schema.Struct({ _tag: Schema.Literal("AccountRequired"), @@ -296,6 +308,17 @@ layer(TestLive, { excludeTestServices: true })("Local app account app links", (i yield* linkLocal(agent, app.id, "svc", first, "app"); expect(yield* localAccountLinks(agent, app.id, "app")).toEqual([["svc", first]]); expect(yield* who).toEqual({ app: first, user: mine }); + // The dashboard's overview, which the Apps cards and the Accounts list read, lists the app + // link apart from the user's own links. + const overview = yield* api.request(agent, "GET", "/dashboard/api/overview"); + expect(overview.status, JSON.stringify(overview.body)).toBe(200); + const listed = yield* body(Overview, overview); + expect(listed.appLinks.filter((link) => link.app === app.id)).toEqual([ + { app: app.id, requirement: "svc", account: first, label: "synthetic-first" }, + ]); + expect(listed.links.filter((link) => link.app === app.id)).toEqual([ + { app: app.id, account: mine }, + ]); // The requirement lists its linked account first, then the owner's other accounts of its // provider, which could be linked; the user requirement's account of another provider is // never offered. diff --git a/e2e/tests/local-app-runtime-restart.spec.ts b/e2e/tests/local-app-runtime-restart.spec.ts index 98264a6641..d98be213dc 100644 --- a/e2e/tests/local-app-runtime-restart.spec.ts +++ b/e2e/tests/local-app-runtime-restart.spec.ts @@ -4,11 +4,13 @@ * in flight loses its connection. The product log records each replacement, a lost call says its * connection was lost, and a lost build tells its deployer the same instead of a class name. */ -import { randomUUID } from "node:crypto"; +import { randomBytes, randomUUID } from "node:crypto"; import { expect, layer } from "@effect/vitest"; import { Effect, Fiber, FileSystem, Queue, Schedule, Schema } from "effect"; import { appsManifest, withApps } from "../support/apps-release.ts"; import { TestLive, withCase } from "../support/case.ts"; +import { Evidence, Telemetry } from "../support/evidence.ts"; +import { Target } from "../support/platform.ts"; import { startLocalProduct, workerdProcess } from "../support/local-workflow-engines.ts"; import { holdingRegistry } from "../support/holding-registry.ts"; import { stalledPackage } from "../support/npm-registry.ts"; @@ -78,8 +80,14 @@ layer(TestLive, { excludeTestServices: true })("Local app runtime restart", (it) files: [ { path: "index.ts", - content: `import { defineApp, query, object, string, router } from "apps"; + content: `import { defineApp, query, object, string, router, workflow } from "apps"; export default defineApp({}, { + workflows: { + hold: workflow({ input: object({}) }, async (ctx) => { + await ctx.step.sleep("hold", "1 day"); + return null; + }), + }, tools: router({ held: query({ input: object({}), output: string() }, async () => (await fetch(${JSON.stringify(`${holding.origin}/held`)})).text()), ready: query({ input: object({}), output: string() }, async () => "ready"), @@ -107,6 +115,48 @@ export default defineApp({}, { }); const [first] = yield* restarts(1); expect(first).toMatchObject({ level: "WARN", annotations: { restarts: 1, ...killed } }); + // The lost call's span says how far its session got (runtime-hop.ts): it opened, the + // call went out, and the socket closed under it before the runtime was replaced. The + // span is read from this scenario's own product's collector. + const target = yield* Target; + /** Read this scenario's own product's collector. */ + const ownCollector = (read: Effect.Effect) => + read.pipe( + Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 60 }), + Effect.provide(Telemetry.layer), + Effect.provideService(Target, { ...target, directory: local.directory }), + ); + const lostSpan = yield* ownCollector( + Telemetry.pipe( + Effect.flatMap((telemetry) => + telemetry.spans("runtime.workerd.call", { app: app.id }), + ), + Effect.flatMap((spans) => { + const span = spans.find( + (tags) => tags["executor.runtime.rpc.opened"] !== undefined, + ); + return span === undefined + ? Effect.fail(new Error("The lost call's span has not reached the collector")) + : Effect.succeed(span); + }), + ), + ); + expect(lostSpan).toMatchObject({ + "executor.runtime.failure": "disconnected", + "executor.runtime.rpc.opened": "true", + }); + // The call fails as its socket closes and the replacement comes at least 250 ms later, + // but a stalled host may record the failure after it: the count and the time since the + // replacement go together either way. + expect( + lostSpan["executor.runtime.host.restarts"] === "0" + ? lostSpan["executor.runtime.host.restarted_ms"] === undefined + : lostSpan["executor.runtime.host.restarts"] === "1" && + lostSpan["executor.runtime.host.restarted_ms"] !== undefined, + JSON.stringify(lostSpan), + ).toBe(true); + expect(Number(lostSpan["executor.runtime.rpc.sent"])).toBeGreaterThan(0); + expect(lostSpan["executor.runtime.rpc.closed_ms"]).toBeDefined(); // The replacement serves the next call. expect( yield* local.json(Schema.Unknown, "POST", "/v1/tools/call", { @@ -154,6 +204,136 @@ export default defineApp({}, { tools: router({ }); const [, second] = yield* restarts(2); expect(second).toMatchObject({ level: "WARN", annotations: { restarts: 2, ...killed } }); + + // A workflow run's status read while workerd's port is closed. The runtime stops replacing + // workerd once it has exited three times within 60 seconds and logs that it stopped + // (`Runtime.ts` in @alchemy.run/cloudflare-runtime); its port then stays closed. Kill each + // replacement as soon as the log records it, until the log says the runtime stopped, and + // only then read: the read's request to the runtime finds nothing listening however long + // the runner takes, and the restart count it records is final. Its span says how far it + // got and how recently the runtime was replaced (runtime-hop.ts). + const run = yield* local.json( + Schema.Struct({ id: Schema.String }), + "POST", + `/v1/apps/${app.id}/workflow-runs`, + { workflow: "hold", input: {}, key: randomUUID() }, + ); + /** The log's runtime replacements, once it has more than `seen` or says it stopped. */ + const afterKill = (seen: number) => + fs.readFileString(log).pipe( + Effect.map((text) => ({ + stopped: text.includes("stopped after 3 restarts within 60 seconds"), + restarts: text + .split("\n") + .filter((line) => line.includes('"App runtime restarted"')).length, + })), + Effect.flatMap((state) => + state.stopped || state.restarts > seen + ? Effect.succeed(state) + : Effect.fail(new Error(`${state.restarts} restarts logged, not stopped`)), + ), + Effect.retry({ schedule: Schedule.spaced("200 millis"), times: 150 }), + ); + let state = { stopped: false, restarts: 2 }; + // Each kill in a minute counts toward the limit, so a few kills reach it. + for (let kill = 0; kill < 8 && !state.stopped; kill++) { + yield* killRuntime; + state = yield* afterKill(state.restarts); + } + expect(state.stopped, "the runtime stopped replacing workerd").toBe(true); + const traceId = randomBytes(16).toString("hex"); + const read = yield* local.request( + "GET", + `/v1/apps/${app.id}/workflow-runs/${run.id}`, + undefined, + { traceparent: `00-${traceId}-${randomBytes(8).toString("hex")}-01` }, + ); + yield* (yield* Evidence).json("status-read.json", { + status: read.status, + body: read.text, + logged: state, + }); + expect(read.status, read.text).toBe(500); + expect( + yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Failure))(read.text), + ).toMatchObject({ _tag: "WorkflowFailure", reason: "engine" }); + const readSpan = yield* ownCollector( + Telemetry.pipe( + Effect.flatMap((telemetry) => telemetry.query(traceId)), + Effect.flatMap((trace) => { + const span = trace.data.find( + ({ span }) => span.tags["executor.runtime.http.route"] === "/workflow", + ); + return span === undefined + ? Effect.fail(new Error("The status read's span has not reached the collector")) + : Effect.succeed(span.span.tags); + }), + ), + ); + expect(readSpan).toMatchObject({ + "executor.runtime.http.method": "POST", + "executor.runtime.http.stage": "request", + "executor.runtime.http.reason": "TransportError", + "executor.runtime.host.restarts": String(state.restarts), + }); + expect(readSpan["executor.runtime.http.code"]).toBeDefined(); + expect(readSpan["executor.runtime.host.restarted_ms"]).toBeDefined(); + // Refused at connect, so nothing of it was sent: the host ran it once more, from its start, + // and that was refused too (runtime-connect.ts). + expect(readSpan).toMatchObject({ + "executor.runtime.http.connect_attempts": "2", + "executor.runtime.http.first_attempt.error_code": "ECONNREFUSED", + "executor.runtime.http.connect_outcome": "failed", + }); + + // A tool call while nothing listens: its session never opens, twice, so nothing was sent, + // and the call's only exchange with the app never left the host. It says so, may be + // repeated, and does not say it may have written. + const unsent = yield* local.request("POST", "/v1/tools/call", { + app: app.id, + tool: "ready", + input: {}, + }); + expect(unsent.status, unsent.text).toBe(502); + const unsentBody = yield* Schema.decodeUnknownEffect( + Schema.fromJsonString( + Schema.Struct({ + _tag: Schema.String, + reason: Schema.String, + notSent: Schema.optionalKey(Schema.Boolean), + mayHaveWritten: Schema.optionalKey(Schema.Boolean), + }), + ), + )(unsent.text); + expect(unsentBody).toMatchObject({ + _tag: "ToolCallFailed", + reason: + "Executor could not connect to the app runtime, so it did not send this tool call.", + notSent: true, + }); + expect(unsentBody).not.toHaveProperty("mayHaveWritten"); + const unsentSpan = yield* ownCollector( + Telemetry.pipe( + Effect.flatMap((telemetry) => + telemetry.spans("runtime.workerd.call", { app: app.id }), + ), + Effect.flatMap((spans) => { + const span = spans.find( + (tags) => tags["executor.runtime.failure"] === "unreachable", + ); + return span === undefined + ? Effect.fail(new Error("The unsent call's span has not reached the collector")) + : Effect.succeed(span); + }), + ), + ); + expect(unsentSpan).toMatchObject({ + "executor.runtime.rpc.connect_attempts": "2", + "executor.runtime.rpc.first_attempt.error_code": "ECONNREFUSED", + "executor.runtime.rpc.connect_outcome": "failed", + "executor.runtime.rpc.opened": "false", + "executor.runtime.rpc.sent": "0", + }); }), ), 180_000, diff --git a/e2e/tests/local-compiled-file-limit.spec.ts b/e2e/tests/local-compiled-file-limit.spec.ts new file mode 100644 index 0000000000..92f462656c --- /dev/null +++ b/e2e/tests/local-compiled-file-limit.spec.ts @@ -0,0 +1,192 @@ +/** + * The local server builds with the same compiler as self-host, in its app runtime. A deploy from + * the real CLI that reaches a file over the compile limit fails with the file, its size and the + * limit, and the app keeps no version. + */ +import { expect, layer } from "@effect/vitest"; +import { Effect, FileSystem, Path, Queue, Redacted, Schema, Stream } from "effect"; +import { randomUUID } from "node:crypto"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { scenarios } from "../test-plan.ts"; +import { Api, body } from "../support/api.ts"; +import { TestLive, withCase } from "../support/case.ts"; +import { Target } from "../support/platform.ts"; +import { Evidence } from "../support/evidence.ts"; +import { helloIndex, helloPackage } from "../support/app-authoring.ts"; +import { appsVersion, withApps } from "../support/apps-release.ts"; +import { holdingRegistry } from "../support/holding-registry.ts"; +import { startLocalProduct } from "../support/local-workflow-engines.ts"; +import { stalledPackage } from "../support/npm-registry.ts"; +import { localEntryCommand } from "../support/local-entry.ts"; + +const Created = Schema.fromJsonString(Schema.Struct({ id: Schema.String })); +const Failure = Schema.fromJsonString( + Schema.Struct({ + error: Schema.Struct({ + code: Schema.String, + status: Schema.optional(Schema.Number), + message: Schema.String, + details: Schema.optional( + Schema.Struct({ stage: Schema.String, location: Schema.Struct({ file: Schema.String }) }), + ), + }), + }), +); +const Stored = Schema.Struct({ activeVersion: Schema.NullOr(Schema.String) }); +const oversized = 2_000_001; + +layer(TestLive, { excludeTestServices: true })("Local compiled file limit", (it) => { + it.effect(scenarios.localCompiledFileLimit.title, (context) => + withCase( + context, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem, + path = yield* Path.Path, + processes = yield* ChildProcessSpawner.ChildProcessSpawner, + target = yield* Target, + evidence = yield* Evidence, + api = yield* Api; + const entry = path.resolve("apps/local/server/src/bin.ts"); + const home = yield* fs.makeTempDirectoryScoped({ prefix: "executor-compiled-limit-" }); + const origin = target.metadata.origin; + const run = (args: ReadonlyArray) => + evidence.step( + `executor apps ${args.join(" ")}`, + Effect.scoped( + Effect.gen(function* () { + const command = localEntryCommand(entry, ["apps", ...args, "--host", origin]); + const child = yield* processes.spawn( + ChildProcess.make(command.command, command.args, { + extendEnv: false, + env: { + PATH: process.env.PATH ?? "", + HOME: home, + DO_NOT_TRACK: "1", + EXECUTOR_API_KEY: Redacted.value(target.apiKey), + }, + stdout: "pipe", + stderr: "pipe", + forceKillAfter: "3 seconds", + }), + ); + const [code, stdout, stderr] = yield* Effect.all( + [ + child.exitCode, + child.stdout.pipe(Stream.decodeText(), Stream.mkString), + child.stderr.pipe(Stream.decodeText(), Stream.mkString), + ], + { concurrency: 3 }, + ).pipe(Effect.timeout("60 seconds")); + return { code: Number(code), stdout, stderr }; + }), + ), + ); + + const source = yield* fs.makeTempDirectoryScoped({ prefix: "executor-compiled-source-" }); + yield* fs.writeFileString(`${source}/index.ts`, helloIndex); + yield* fs.writeFileString(`${source}/package.json`, helloPackage(appsVersion)); + const created = yield* run(["create", "--name", "Compiled file limit", "--files", source]); + expect(created.code, created.stderr).toBe(0); + const app = yield* Schema.decodeUnknownEffect(Created)(created.stdout); + const session = yield* api.session(); + const authorization = { authorization: `Bearer ${Redacted.value(target.apiKey)}` }; + yield* Effect.addFinalizer(() => + session.send("DELETE", `/v1/apps/${app.id}`, undefined, authorization).pipe(Effect.orDie), + ); + + // One byte over the limit: the module the app imports is refused before it compiles. + const head = 'export const filler = "'; + const tail = '";\n'; + yield* fs.writeFileString( + `${source}/filler.ts`, + `${head}${"x".repeat(oversized - head.length - tail.length)}${tail}`, + ); + yield* fs.writeFileString( + `${source}/index.ts`, + `import { filler } from "./filler.ts";\n${helloIndex}\nvoid filler;\n`, + ); + const deployed = yield* run(["deploy", "--app", app.id, "--files", source]); + yield* evidence.json("over-the-limit.json", deployed); + expect(deployed.code).toBe(1); + const message = + "filler.ts is 2,000,001 bytes; a JavaScript or TypeScript file the build compiles can be at most 2,000,000 bytes (2 MB)."; + expect(deployed.stderr).toContain("Code: VersionBuildFailed (HTTP 422)"); + expect(deployed.stderr).toContain(message); + const failed = (yield* Schema.decodeUnknownEffect(Failure)(deployed.stdout)).error; + expect(failed).toMatchObject({ + code: "VersionBuildFailed", + status: 422, + details: { stage: "source", location: { file: "filler.ts" } }, + }); + expect(failed.message).toContain(message); + + // The app keeps no version: none is active and the server lists none. + const stored = yield* body( + Stored, + yield* session.send("GET", `/v1/apps/${app.id}`, undefined, authorization), + ); + expect(stored.activeVersion).toBeNull(); + const versions = yield* session.send( + "GET", + `/v1/apps/${app.id}/versions`, + undefined, + authorization, + ); + expect(versions.status).toBe(200); + expect(yield* body(Schema.Array(Schema.Unknown), versions)).toEqual([]); + }), + ), + ); +}); + +/** + * No import of the oversized module is resolved before the refusal. The module imports a declared + * package whose registry metadata never arrives: a build that resolved that import would ask the + * registry for the package and wait on it. This does not show that the file was not parsed (a + * parse need not resolve imports); `compiledFileLimitResponsive` checks that no compile work held + * the runtime's thread. + */ +layer(TestLive, { excludeTestServices: true })("Local compiled file limit imports", (it) => { + it.effect(scenarios.localCompiledFileLimitImports.title, (context) => + withCase( + context, + Effect.gen(function* () { + const evidence = yield* Evidence; + const holding = yield* holdingRegistry; + const local = yield* startLocalProduct({ EXECUTOR_NPM_REGISTRY: holding.origin }); + const importLine = `import stalled from "${stalledPackage}";\nexport const marker = stalled;\n`; + const head = 'export const filler = "'; + const tail = '";\n'; + const filler = `${importLine}${head}${"x".repeat(oversized - importLine.length - head.length - tail.length)}${tail}`; + expect(new TextEncoder().encode(filler).length).toBe(oversized); + const deployed = yield* local + .request("POST", "/v1/apps/deploy", { + owner: "compiled-file-limit-e2e", + name: `Before compiling ${randomUUID().slice(0, 8)}`, + files: [ + { + path: "index.ts", + content: `import { marker } from "./filler.ts";\n${helloIndex}\nvoid marker;\n`, + }, + { path: "filler.ts", content: filler }, + { + path: "package.json", + content: JSON.stringify({ + type: "module", + dependencies: withApps({ [stalledPackage]: "1.0.0" }), + }), + }, + ], + }) + .pipe(Effect.timeout("60 seconds")); + yield* evidence.json("before-compiling.json", deployed); + expect(deployed.status, deployed.text).toBe(422); + expect(deployed.text).toContain( + "filler.ts is 2,000,001 bytes; a JavaScript or TypeScript file the build compiles can be at most 2,000,000 bytes (2 MB).", + ); + // Nothing asked the registry for the package the oversized module imports. + expect(yield* Queue.size(holding.arrived)).toBe(0); + }), + ), + ); +}); diff --git a/e2e/tests/lost-connections-lifecycle.spec.ts b/e2e/tests/lost-connections-lifecycle.spec.ts new file mode 100644 index 0000000000..32455320f2 --- /dev/null +++ b/e2e/tests/lost-connections-lifecycle.spec.ts @@ -0,0 +1,55 @@ +/** + * The scenario lifecycle saves lost stage requests (`support/lifecycle.ts`), so a broken reporter + * cannot print zero for a run that lost some. Runs under `lost-connections-lifecycle.config.ts`, + * which loads the real setup file against a stand-in stage that resets a request for `/lose`. A request lost while a `layer` is built is outside any scenario; the setup file saves + * it under this file. The config's global teardown checks both records were written. + */ +import { expect, layer } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { Config, Effect, FileSystem, Layer, Schema } from "effect"; +import { FetchHttpClient, HttpClient } from "effect/http"; +import { readLostRequests } from "../lost-requests.ts"; +import { addStageOrigins } from "../support/lost-connections.ts"; + +/** The stand-in stage's origin, from the run's `run.json`. */ +const origin = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const run = yield* Config.String("EXECUTOR_E2E_RUN"); + const { origin } = yield* Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Struct({ origin: Schema.String })), + )(yield* fs.readFileString(`${run}/run.json`)); + return origin; +}); + +/** A request the stand-in resets on its new connection: it is lost. */ +const loseOne = Effect.gen(function* () { + const stage = yield* origin; + const http = yield* HttpClient.HttpClient; + return yield* http.get(`${stage}/lose`); +}).pipe(Effect.provide(FetchHttpClient.layer)); + +// Built before any scenario: a lost request here belongs to no scenario. +const OutsideAnyScenario = Layer.effectDiscard( + Effect.gen(function* () { + addStageOrigins([yield* origin]); + yield* loseOne.pipe(Effect.ignore); + }), +).pipe(Layer.provideMerge(NodeServices.layer)); + +layer(OutsideAnyScenario)("Lost stage requests reach the run's records", (it) => { + it.effect("a request lost in a scenario is saved by the scenario lifecycle", () => + Effect.gen(function* () { + expect((yield* loseOne.pipe(Effect.result))._tag).toBe("Failure"); + }), + ); + + it.effect("the record the scenario before made is on disk", () => + Effect.gen(function* () { + const run = yield* Config.String("EXECUTOR_E2E_RUN"); + const records = yield* readLostRequests([run]); + expect( + records.filter((record) => record.scenario?.endsWith("is saved by the scenario lifecycle")), + ).toHaveLength(1); + }), + ); +}); diff --git a/e2e/tests/lost-connections.spec.ts b/e2e/tests/lost-connections.spec.ts new file mode 100644 index 0000000000..edb70d2f46 --- /dev/null +++ b/e2e/tests/lost-connections.spec.ts @@ -0,0 +1,894 @@ +/** + * Stage requests on connections of their own (`support/lost-connections.ts`), against a local + * stand-in stage. The stand-in for 16 November's resets answers the first request on a connection, + * kept alive, and resets the connection when a second request arrives on it: a request sent down a + * used connection crosses the close, as #1082's 16 lost requests did (all on reused connections, + * 12 of them written 1-5 ms after the previous response). Whether a request is lost is decided by + * that fact, not by a timer. + * + * - Under load, GETs and POSTs from eight concurrent callers, over plain connections and over TLS: + * `fetch` as it was (`main`'s client, keep-alive) loses requests, POSTs among them; with each + * stage request on its own connection none is lost, and the stand-in counts every request + * arriving exactly once: nothing is sent twice. Over TLS the reset is the TCP connection's, + * under the TLS session, as an edge's would be. + * - Each entry point installs the wrapper before anything can read `fetch`: a request made through + * Effect's client first does not leave the requests after it on a shared connection. + * - A streamed response still arrives whole. + * - A stand-in that resets brand-new connections: the request fails, and the failure and the record + * say "new connection", a different fault. A reset after a response began says so. + * - A TLS alert in the handshake (#1088's preparation): its code, `ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE`, + * holds a slash. The request is recorded as never written, and its failure says TLS. + * - A burst of refused or reset new connections, then a stage that answers, plain (the listener is + * closed, then opened again) and over TLS (each new connection is reset in its handshake), the + * shape of #1089's losses: `main`'s client fails; with the wrapper a GET and a POST are each sent + * again on a new connection, 1 and then 2 s later, and answered on the third attempt, each + * arriving at the stand-in exactly once. The record counts the repeats and the wait, and the job + * summary says a job passed only after them. + * - A connection that opens, takes the whole request and dies is never sent again, plain or TLS. + * - The repeats stop at the scenario's bound, when the caller gives up, and for a body that cannot + * be sent twice; the failure and the failed scenario's note say for how long the request was + * unsent. A failure that is not the connection's (a certificate for another name) is not repeated. + * - A second copy of the module in one process does not wrap `fetch` again, so no wrapper can read a + * written request as never written. + * - Each stage request a scenario sends is kept for its `stage-requests.json`: method, path with ids + * shortened, status or failure, times, trace id and connection, never its query, headers or + * cookies; at most `stageRequestLimit` rows, the oldest dropped and counted. + * - The job summary counts each lost request with its time, and an unreadable count is never zero. + */ +import { expect, layer } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { Effect, FileSystem, Stream } from "effect"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { createServer, type Server, type Socket } from "node:net"; +import { getCACertificates, setDefaultCACertificates, TLSSocket } from "node:tls"; +import { + lostRequestLines, + lostRequestsUncounted, + readLostRequests, + saveLostRequests, +} from "../lost-requests.ts"; +import { + configureLostConnections, + installLostConnections, + lostOutcome, + lostRequestsOf, + requestIdHeader, + stageRequestLimit, + stageRequestsOf, + unsentNote, +} from "../support/lost-connections.ts"; +import { CallFailed, readResponse, requestFailureText } from "../support/request-failure.ts"; + +// `main`'s client: Node's fetch before the wrapper is installed, keep-alive on. +const unwrapped = globalThis.fetch; +installLostConnections(); + +/** + * - `reset-on-reuse`: answer a connection's first request, kept alive; reset it when another + * request arrives on it. + * - `reset-new`: reset every connection at its first request. + * - `partial-then-reset`: write the first bytes of a status line, then reset. + * - `stream`: answer with a body in three chunks over 60 ms; reset the connection if it is used again. + */ +type Mode = "reset-on-reuse" | "reset-new" | "partial-then-reset" | "stream"; + +interface TlsMaterial { + readonly key: string; + readonly cert: string; +} + +interface Stage { + readonly origin: string; + /** Requests the stand-in received, by `METHOD /path`. */ + readonly received: Map; + /** How many times each request id arrived. */ + readonly arrivals: Map; +} + +const answer = + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 2\r\nconnection: keep-alive\r\n\r\n{}"; + +const stage = (mode: Mode, material?: TlsMaterial) => + Effect.acquireRelease( + Effect.callback void }>((resume) => { + const received = new Map(); + const arrivals = new Map(); + const sockets = new Set(); + const server: Server = createServer((raw) => { + // Over TLS the reset is the TCP connection's, under the TLS session, as an edge's would be. + const socket: Socket = + material === undefined ? raw : new TLSSocket(raw, { isServer: true, ...material }); + sockets.add(raw); + raw.on("error", () => undefined); + let buffered = ""; + // Set once a request was answered: another request on this connection crosses its close. + let used = false; + socket.on("data", (chunk) => { + buffered += chunk.toString("latin1"); + for (;;) { + // A request is whole when its headers end and its declared body has arrived. + const end = buffered.indexOf("\r\n\r\n"); + if (end === -1) return; + const head = buffered.slice(0, end); + const length = Number(/content-length:\s*(\d+)/iu.exec(head)?.[1] ?? "0"); + if (buffered.length < end + 4 + length) return; + buffered = buffered.slice(end + 4 + length); + const [method, path] = head.split("\r\n")[0]!.split(" "); + const key = `${method} ${path}`; + received.set(key, (received.get(key) ?? 0) + 1); + const id = new RegExp(`^${requestIdHeader}:\\s*(\\S+)`, "imu").exec(head)?.[1] ?? key; + arrivals.set(id, (arrivals.get(id) ?? 0) + 1); + if (mode === "reset-new" || used) return raw.resetAndDestroy(); + used = true; + if (mode === "partial-then-reset") + return socket.write("HTTP/1.1 2", () => setTimeout(() => raw.resetAndDestroy(), 20)); + if (mode === "stream") { + socket.write( + "HTTP/1.1 200 OK\r\ncontent-type: text/plain\r\ntransfer-encoding: chunked\r\nconnection: keep-alive\r\n\r\n", + ); + const chunks = ["a", "b", "c"]; + const next = () => { + const chunk = chunks.shift(); + if (chunk === undefined) return socket.write("0\r\n\r\n"); + socket.write(`1\r\n${chunk}\r\n`, () => setTimeout(next, 20)); + }; + return next(); + } + socket.write(answer); + } + }); + raw.on("close", () => sockets.delete(raw)); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + const origin = `${material === undefined ? "http" : "https"}://127.0.0.1:${port}`; + configureLostConnections({ + origins: [origin], + scenario: () => expect.getState().currentTestName, + deadline: () => undefined, + }); + resume( + Effect.succeed({ + origin, + received, + arrivals, + close: () => { + for (const socket of sockets) socket.destroy(); + server.close(); + }, + }), + ); + }); + }), + ({ close }) => Effect.sync(close), + ); + +/** Send a request through Effect's fetch client and read it; the result, success or failure. */ +const send = ( + method: "GET" | "POST", + url: string, + options: { + readonly body?: unknown; + readonly main?: boolean; + /** Send a streamed body, which `fetch` cannot be given twice. */ + readonly stream?: boolean; + } = {}, +) => + Effect.gen(function* () { + const http = yield* HttpClient.HttpClient; + let request = HttpClientRequest.make(method)(url); + if (options.body !== undefined) + request = yield* HttpClientRequest.bodyJson(request, options.body); + if (options.stream === true) + request = HttpClientRequest.bodyStream( + request, + Stream.make(new TextEncoder().encode("synthetic")), + ); + return yield* readResponse(http.execute(request), "10 seconds").pipe(Effect.result); + }).pipe( + Effect.provide(FetchHttpClient.layer), + options.main === true + ? Effect.provideService(FetchHttpClient.Fetch, unwrapped) + : (self) => self, + ); + +/** + * Eight callers, each sending 25 requests one after another, GETs and POSTs in turn, so a caller's + * next request follows its previous response at once, as a scenario's do. + */ +const load = (origin: string, main: boolean) => + Effect.forEach( + Array.from({ length: 8 }, (_, caller) => caller), + (caller) => + Effect.forEach( + Array.from({ length: 25 }, (_, index) => index), + (index) => + (index % 2 === 0 + ? send("GET", `${origin}/api/organizations/org/resources`, { main }) + : send("POST", `${origin}/api/organizations/org/apps/app/tools/call`, { + body: { caller, index }, + main, + }) + ).pipe(Effect.map((result) => ({ method: index % 2 === 0 ? "GET" : "POST", result }))), + ), + { concurrency: "unbounded" }, + ).pipe(Effect.map((results) => results.flat())); + +const failureOf = (result: Effect.Success>) => { + expect(result._tag).toBe("Failure"); + const failure = result._tag === "Failure" ? result.failure : undefined; + expect(failure).toBeInstanceOf(CallFailed); + return requestFailureText((failure as CallFailed).failure); +}; + +/** A certificate for 127.0.0.1, trusted for the case only, the new connections included. */ +const trustedCertificate = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "lost-connections-tls-" }); + yield* processes.string( + ChildProcess.make( + "openssl", + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + `${directory}/key.pem`, + "-out", + `${directory}/cert.pem`, + "-days", + "1", + "-subj", + "/CN=127.0.0.1", + "-addext", + "subjectAltName=IP:127.0.0.1", + ], + { stderr: "pipe" }, + ), + ); + const material = { + key: yield* fs.readFileString(`${directory}/key.pem`), + cert: yield* fs.readFileString(`${directory}/cert.pem`), + }; + yield* Effect.acquireRelease( + Effect.sync(() => { + const trusted = getCACertificates("default"); + setDefaultCACertificates([...trusted, material.cert]); + return trusted; + }), + (trusted) => Effect.sync(() => setDefaultCACertificates(trusted)), + ); + return material; +}); + +layer(NodeServices.layer)("Stage requests on connections of their own", (it) => { + for (const transport of ["plain", "TLS"] as const) { + const material = transport === "TLS" ? trustedCertificate : Effect.succeed(undefined); + + it.effect( + `main's client loses requests, POSTs among them, sent down a used connection (${transport})`, + () => + Effect.gen(function* () { + const { origin } = yield* stage("reset-on-reuse", yield* material); + const results = yield* load(origin, true); + const lost = results.filter(({ result }) => result._tag === "Failure"); + expect(lost.length).toBeGreaterThan(0); + expect(lost.some(({ method }) => method === "POST")).toBe(true); + }), + ); + + it.effect( + `loses none and sends none twice with each stage request on its own connection (${transport})`, + () => + Effect.gen(function* () { + const { origin, arrivals } = yield* stage("reset-on-reuse", yield* material); + const results = yield* load(origin, false); + expect(results.filter(({ result }) => result._tag === "Failure")).toHaveLength(0); + expect(arrivals.size).toBe(200); + expect([...arrivals.values()].every((count) => count === 1)).toBe(true); + expect( + lostRequestsOf(`each stage request on its own connection (${transport})`), + ).toHaveLength(0); + }), + ); + } + + it.effect("every entry point installs the wrapper before anything can read fetch", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + // The first two imports of each entry point that makes outbound calls, and of every setup + // file of the Vitest config the targets run under, are the connect time-out, then this. + const firstTwo = (text: string) => + [...text.matchAll(/^import (?:[^;]*? from )?"([^"]+)";/gm)] + .slice(0, 2) + .map((match) => match[1]); + const order = ["./support/connect-timeout.ts", "./support/stage-connections.ts"]; + const callers: Array = []; + for (const name of (yield* fs.readDirectory("e2e")).toSorted()) { + if (!name.endsWith(".ts")) continue; + const text = yield* fs.readFileString(`e2e/${name}`); + if ( + /NodeRuntime\.runMain/.test(text) && + /FetchHttpClient|runSuite\(|runDeployedSuite|startDeployment/.test(text) + ) { + callers.push(`e2e/${name}`); + expect(firstTwo(text), name).toEqual(order); + } + } + expect(callers).toEqual([ + "e2e/cleanup-deployed.ts", + "e2e/cli.ts", + "e2e/create-emulators.ts", + "e2e/deployed.ts", + "e2e/run.ts", + ]); + const config = yield* fs.readFileString("e2e/vitest.config.ts"); + const setupFiles = [...config.matchAll(/setupFiles: \[([^\]]*)\]/g)].flatMap((match) => + [...match[1]!.matchAll(/"([^"]+)"/g)].map((file) => file[1]!), + ); + expect(setupFiles.length).toBeGreaterThan(0); + for (const file of setupFiles) + expect(firstTwo(yield* fs.readFileString(file)), file).toEqual(order); + }), + ); + + it.effect( + "a request through Effect's client before the stage is known leaves later stage requests each on its own connection", + () => + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + // A plain keep-alive server that counts the connections stage requests arrive on. + const counted = yield* Effect.acquireRelease( + Effect.callback<{ + readonly origin: string; + readonly connections: Set; + readonly close: () => void; + }>((resume) => { + const connections = new Set(); + const sockets = new Set(); + const server = createServer((socket) => { + sockets.add(socket); + socket.on("data", (chunk) => { + const text = chunk.toString("latin1"); + if (!text.includes("\r\n\r\n")) return; + if (text.startsWith("GET /api/")) connections.add(socket); + socket.write(answer); + }); + socket.on("error", () => undefined); + socket.on("close", () => sockets.delete(socket)); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + resume( + Effect.succeed({ + origin: `http://127.0.0.1:${port}`, + connections, + close: () => { + for (const socket of sockets) socket.destroy(); + server.close(); + }, + }), + ); + }); + }), + ({ close }) => Effect.sync(close), + ); + /** + * A process that, like the deployed suite, makes a request through Effect's client before + * the stage is named (its `/health` check), then names the stage and sends ten requests. + * `early` imports the wrappers as the entry points do; otherwise it installs the wrapper + * after the first request, as the preparation used to. + */ + const child = (early: boolean) => + Effect.gen(function* () { + const code = [ + 'import "./e2e/support/connect-timeout.ts";', + early ? 'import "./e2e/support/stage-connections.ts";' : "", + 'import { Effect } from "effect";', + 'import { FetchHttpClient, HttpClient } from "effect/http";', + 'import { addStageOrigins, installLostConnections } from "./e2e/support/lost-connections.ts";', + "const origin = process.env.STAGE;", + "await Effect.runPromise(Effect.gen(function* () {", + " const http = yield* HttpClient.HttpClient;", + " yield* http.get(`${origin}/health`).pipe(Effect.flatMap((r) => r.text));", + early ? "" : " installLostConnections();", + " addStageOrigins([origin]);", + " for (let i = 0; i < 10; i++) yield* http.get(`${origin}/api/${i}`).pipe(Effect.flatMap((r) => r.text));", + "}).pipe(Effect.provide(FetchHttpClient.layer)));", + ].join("\n"); + counted.connections.clear(); + const spawned = yield* processes.spawn( + ChildProcess.make(process.execPath, ["--input-type=module", "-e", code], { + env: { STAGE: counted.origin }, + extendEnv: true, + }), + ); + const [, , exit] = yield* Effect.all( + [ + spawned.stdout.pipe(Stream.decodeText(), Stream.mkString), + spawned.stderr.pipe(Stream.decodeText(), Stream.mkString), + spawned.exitCode, + ], + { concurrency: "unbounded" }, + ); + expect(Number(exit)).toBe(0); + return counted.connections.size; + }).pipe(Effect.scoped); + // Installed first: ten stage requests, ten connections. + expect(yield* child(true)).toBe(10); + // Installed after the first request, as the deployed suite's preparation did: the ten + // share Node's keep-alive pool. This is the gap the entry points' import closes. + expect(yield* child(false)).toBeLessThan(10); + }), + ); + + it.effect("a streamed response arrives whole", () => + Effect.gen(function* () { + const { origin } = yield* stage("stream"); + const http = yield* HttpClient.HttpClient.pipe(Effect.provide(FetchHttpClient.layer)); + const text = yield* http + .get(`${origin}/api/organizations/org/events`) + .pipe(Effect.flatMap((response) => response.text)); + expect(text).toBe("abc"); + }), + ); + + it.effect("a reset on a brand-new connection fails and is recorded as a new connection", () => + Effect.gen(function* () { + const { origin, received } = yield* stage("reset-new"); + const line = failureOf(yield* send("POST", `${origin}/api/onboarding/prepare`, { body: {} })); + expect(line).toMatch( + /before response \(reset\).*on a new connection \d+ ms old, whole request written, no response/u, + ); + expect(received.get("POST /api/onboarding/prepare")).toBe(1); + expect(lostRequestsOf("is recorded as a new connection")[0]).toMatchObject({ + method: "POST", + path: "/api/onboarding/prepare", + outcome: "lost: new connection", + sent: { + connection: "new", + earlierRequests: 0, + written: "whole request", + responseStarted: false, + }, + }); + }), + ); + + it.effect("a TLS alert in the handshake is recorded as a request never written", () => + Effect.gen(function* () { + // Answers the client's hello with a fatal handshake_failure alert, as an edge without a + // certificate for the name does: record type 21, TLS 1.2, length 2, fatal (2), alert 40. + const { origin } = yield* Effect.acquireRelease( + Effect.callback<{ readonly origin: string; readonly server: Server }>((resume) => { + const server = createServer((socket) => { + socket.on("error", () => undefined); + socket.once("data", () => socket.end(Buffer.from([21, 3, 3, 0, 2, 2, 40]))); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + resume(Effect.succeed({ origin: `https://127.0.0.1:${port}`, server })); + }); + }), + ({ server }) => Effect.sync(() => server.close()), + ); + // A scenario bound half a second away: no repeat can start before it. + configureLostConnections({ + origins: [origin], + scenario: () => expect.getState().currentTestName, + deadline: () => Date.now() + 500, + }); + const line = failureOf(yield* send("POST", `${origin}/api/onboarding/prepare`, { body: {} })); + expect(line).toContain("before response (tls)"); + expect(line).toContain("never written to a connection, not sent again: the scenario's bound"); + expect(lostRequestsOf("recorded as a request never written")).toEqual([ + expect.objectContaining({ + method: "POST", + path: "/api/onboarding/prepare", + code: "ERR_SSL_SSL/TLS_ALERT_HANDSHAKE_FAILURE", + outcome: "lost: never written", + sent: { written: "nothing", responseStarted: false }, + stopped: "the scenario's bound", + }), + ]); + }), + ); + + it.effect("a reset after a response began is recorded as such", () => + Effect.gen(function* () { + const { origin } = yield* stage("partial-then-reset"); + const line = failureOf(yield* send("GET", `${origin}/api/organizations/org/resources`)); + expect(line).toContain("a response had begun"); + expect(lostRequestsOf("recorded as such")[0]?.outcome).toBe("lost: response had begun"); + }), + ); + + it.effect( + "the job summary counts lost requests with their times, and never shows an unread count as zero", + () => + Effect.gen(function* () { + const { origin } = yield* stage("reset-new"); + yield* send("GET", `${origin}/api/organizations/org/resources`); + const fs = yield* FileSystem.FileSystem; + const run = yield* fs.makeTempDirectoryScoped(); + yield* saveLostRequests( + run, + "scenario-1", + lostRequestsOf("never shows an unread count as zero"), + ); + const records = yield* readLostRequests([run]); + expect(lostRequestLines(records)).toMatch( + /^Stage requests lost in transport: 1 \(1 lost: new connection\)\.\n- \d\d:\d\d:\d\d GET \/api\/organizations\/org\/resources \(new; lost: new connection\)$/u, + ); + expect(lostRequestLines([])).toBe("Stage requests lost in transport: 0."); + expect(lostRequestsUncounted).toContain("NOT COUNTED"); + expect(lostRequestsUncounted).toContain("not zero"); + expect( + lostOutcome({ connection: "reused", written: "whole request", responseStarted: false }), + ).toBe("lost: reused connection"); + }), + ); +}); + +/** + * A stage that refuses or resets every new connection for its first `burstMs`, and then answers, + * closing each connection after its response. Plain: its listener is closed, so a connect is + * refused, and opened again on the same port. TLS: each new connection is reset at the client's + * hello, in the handshake. `take-and-die` instead reads each whole request and then resets. + */ +const burstStage = ( + transport: "plain" | "TLS", + burstMs: number, + material?: TlsMaterial, + mode: "burst" | "take-and-die" = "burst", +) => + Effect.acquireRelease( + Effect.callback void }>((resume) => { + const received = new Map(); + const arrivals = new Map(); + const sockets = new Set(); + const until = Date.now() + burstMs; + let reopen: ReturnType | undefined; + const server: Server = createServer((raw) => { + sockets.add(raw); + raw.on("close", () => sockets.delete(raw)); + raw.on("error", () => undefined); + if (transport === "TLS" && Date.now() < until) + return void raw.once("data", () => raw.resetAndDestroy()); + const socket: Socket = + transport === "plain" ? raw : new TLSSocket(raw, { isServer: true, ...material }); + socket.on("error", () => undefined); + let buffered = ""; + socket.on("data", (chunk) => { + buffered += chunk.toString("latin1"); + const end = buffered.indexOf("\r\n\r\n"); + if (end === -1) return; + const head = buffered.slice(0, end); + const length = Number(/content-length:\s*(\d+)/iu.exec(head)?.[1] ?? "0"); + if (buffered.length < end + 4 + length) return; + const [method, path] = head.split("\r\n")[0]!.split(" "); + const key = `${method} ${path}`; + received.set(key, (received.get(key) ?? 0) + 1); + const id = new RegExp(`^${requestIdHeader}:\\s*(\\S+)`, "imu").exec(head)?.[1] ?? key; + arrivals.set(id, (arrivals.get(id) ?? 0) + 1); + if (mode === "take-and-die") return raw.resetAndDestroy(); + socket.end(answer); + }); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + const origin = `${transport === "plain" ? "http" : "https"}://127.0.0.1:${port}`; + configureLostConnections({ + origins: [origin], + scenario: () => expect.getState().currentTestName, + deadline: () => undefined, + }); + if (transport === "plain" && mode === "burst") { + server.close(); + if (Number.isFinite(burstMs)) + reopen = setTimeout(() => server.listen(port, "127.0.0.1"), burstMs); + } + resume( + Effect.succeed({ + origin, + received, + arrivals, + close: () => { + clearTimeout(reopen); + for (const socket of sockets) socket.destroy(); + server.close(); + }, + }), + ); + }); + }), + ({ close }) => Effect.sync(close), + ); + +layer(NodeServices.layer)("Stage requests of which nothing was written", (it) => { + for (const transport of ["plain", "TLS"] as const) { + const material = transport === "TLS" ? trustedCertificate : Effect.succeed(undefined); + + it.effect(`${transport}: main's client fails in a burst of refused or reset connections`, () => + Effect.gen(function* () { + const { origin, received } = yield* burstStage(transport, 1_500, yield* material); + const line = failureOf( + yield* send("GET", `${origin}/api/organizations/org/resources`, { main: true }), + ); + expect(line).toContain("before response"); + expect(received.size).toBe(0); + }), + ); + + for (const method of ["GET", "POST"] as const) + it.effect( + `${transport}: a ${method} never written is sent again after 1 and 2 s and answered once`, + () => + Effect.gen(function* () { + // Refused or reset for 1.5 s: the attempts at 0 and 1 s fail, the one at 3 s is answered. + const { origin, received, arrivals } = yield* burstStage( + transport, + 1_500, + yield* material, + ); + const path = `/api/organizations/org/${method === "GET" ? "resources" : "apps/app/tools/call"}`; + const result = yield* send(method, `${origin}${path}`, { + ...(method === "POST" && { body: { synthetic: true } }), + }); + expect(result._tag).toBe("Success"); + // Exactly once: one arrival, under the one attempt id that reached the stand-in. + expect(received.get(`${method} ${path}`)).toBe(1); + expect([...arrivals.values()]).toEqual([1]); + const [kept] = lostRequestsOf(expect.getState().currentTestName!); + expect(kept).toMatchObject({ + method, + path, + outcome: "sent again: answered", + sent: { written: "nothing", responseStarted: false }, + }); + expect(kept!.repeats).toHaveLength(2); + expect(kept!.elapsedMs).toBeGreaterThanOrEqual(3_000); + expect(kept!.elapsedMs).toBeLessThan(4_500); + }), + ); + + it.effect( + `${transport}: a connection that takes the whole request and dies is not repeated`, + () => + Effect.gen(function* () { + const { origin, received } = yield* burstStage( + transport, + 0, + yield* material, + "take-and-die", + ); + const line = failureOf(yield* send("GET", `${origin}/api/organizations/org/resources`)); + expect(line).toMatch( + /on a new connection \d+ ms old, whole request written, no response/u, + ); + expect(received.get("GET /api/organizations/org/resources")).toBe(1); + const [kept] = lostRequestsOf(expect.getState().currentTestName!); + expect(kept).toMatchObject({ outcome: "lost: new connection" }); + expect(kept!.repeats).toBeUndefined(); + }), + ); + } + + it.effect("the repeats stop at the scenario's bound, and the failure says how long", () => + Effect.gen(function* () { + const { origin, received } = yield* burstStage("plain", Number.POSITIVE_INFINITY); + // The bound is 2.5 s away: the repeat after 1 s starts, the one 2 s later would cross it. + const bound = Date.now() + 2_500; + configureLostConnections({ + origins: [origin], + scenario: () => expect.getState().currentTestName, + deadline: () => bound, + }); + const line = failureOf(yield* send("POST", `${origin}/api/onboarding/prepare`, { body: {} })); + expect(Date.now()).toBeLessThan(bound); + expect(line).toMatch( + /never written to a connection on any of 2 attempts over 1\.\d s, not sent again: the scenario's bound/u, + ); + expect(received.size).toBe(0); + const [kept] = lostRequestsOf(expect.getState().currentTestName!); + expect(kept).toMatchObject({ + outcome: "lost: never written", + code: "ECONNREFUSED", + stopped: "the scenario's bound", + }); + expect(kept!.repeats).toHaveLength(2); + expect(unsentNote(expect.getState().currentTestName!)).toMatch( + /^one stage request was unsent for 1\.\d s: its connect or TLS handshake failed on every attempt/u, + ); + }), + ); + + it.effect("the repeats stop when the caller gives up, and a stream body is not sent twice", () => + Effect.gen(function* () { + const { origin } = yield* burstStage("plain", Number.POSITIVE_INFINITY); + // The caller's own bound, a real 1.5 s (the case's clock is a test one), interrupts the request + // and so aborts the wait before the second repeat. + const http = yield* HttpClient.HttpClient.pipe(Effect.provide(FetchHttpClient.layer)); + const timedOut = yield* http.get(`${origin}/api/organizations/org/resources`).pipe( + Effect.raceFirst( + Effect.callback((resume) => { + setTimeout(() => resume(Effect.fail("the caller's bound")), 1_500); + }), + ), + Effect.result, + ); + expect(timedOut._tag).toBe("Failure"); + // The abort reaches the wait on the next turn of the event loop. + yield* Effect.callback((resume) => { + setTimeout(() => resume(Effect.void), 100); + }); + expect(lostRequestsOf(expect.getState().currentTestName!)[0]).toMatchObject({ + path: "/api/organizations/org/resources", + outcome: "lost: never written", + stopped: "the caller gave up", + }); + const streamed = yield* send("POST", `${origin}/api/organizations/org/apps/app/files`, { + stream: true, + }); + expect(failureOf(streamed)).toContain("not sent again: its body cannot be sent twice"); + }), + ); + + it.effect("a certificate that does not verify is not repeated", () => + Effect.gen(function* () { + // The certificate names 127.0.0.1; asked for localhost, the handshake fails verification. + const { origin, received } = yield* burstStage("TLS", 0, yield* trustedCertificate); + const named = origin.replace("127.0.0.1", "localhost"); + configureLostConnections({ + origins: [named], + scenario: () => expect.getState().currentTestName, + deadline: () => undefined, + }); + const started = Date.now(); + const line = failureOf(yield* send("GET", `${named}/api/organizations/org/resources`)); + expect(Date.now() - started).toBeLessThan(900); + expect(line).toContain("not sent again: its failure is not a connection's"); + expect(received.size).toBe(0); + expect(lostRequestsOf(expect.getState().currentTestName!)[0]).toMatchObject({ + code: "ERR_TLS_CERT_ALTNAME_INVALID", + outcome: "lost: never written", + stopped: "its failure is not a connection's", + }); + }), + ); + + it.effect("a second copy of the module does not wrap fetch again", () => + Effect.gen(function* () { + // A process that loads the module twice, under two module ids, and installs both copies. + const module = new URL("../support/lost-connections.ts", import.meta.url).href; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* processes.string( + ChildProcess.make(process.execPath, [ + "--input-type=module", + "--eval", + [ + `const first = await import(${JSON.stringify(`${module}?first`)});`, + `const second = await import(${JSON.stringify(`${module}?second`)});`, + "first.installLostConnections();", + "const wrapped = globalThis.fetch;", + "second.installLostConnections();", + "console.log(JSON.stringify({ copies: first !== second, wrappedOnce: globalThis.fetch === wrapped }));", + ].join("\n"), + ]), + ); + expect(JSON.parse(output)).toEqual({ copies: true, wrappedOnce: true }); + }), + ); + + it.effect("the job summary counts repeats, and a job that passed only after them says so", () => + Effect.gen(function* () { + const { origin } = yield* burstStage("plain", 1_500); + expect((yield* send("GET", `${origin}/api/organizations/org/resources`))._tag).toBe( + "Success", + ); + const fs = yield* FileSystem.FileSystem; + const run = yield* fs.makeTempDirectoryScoped(); + yield* saveLostRequests( + run, + "scenario-1", + lostRequestsOf(expect.getState().currentTestName!), + ); + const records = yield* readLostRequests([run]); + expect(lostRequestLines(records)).toMatch( + /^Stage requests lost in transport: 0\.\nStage requests sent again because nothing of them was written to a connection, then answered: 1, after 2 repeats and 3\.\d s of waiting in all\.\n- \d\d:\d\d:\d\d GET \/api\/organizations\/org\/resources \(unwritten; sent again: answered, 3 attempts over 3\.\d s\)$/u, + ); + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const summary = yield* processes.string( + ChildProcess.make(process.execPath, ["e2e/lost-request-summary.ts", "success", run], { + env: { GITHUB_STEP_SUMMARY: `${run}/summary.md` }, + extendEnv: true, + }), + ); + expect(summary).toMatch( + /This job passed only after sending 1 stage request again \(2 repeats, 3\.\d s of waiting in all\): nothing of them had been written to a connection\./u, + ); + expect(summary).toContain("::warning title=Stage requests sent again::"); + expect(summary).not.toContain("title=Lost stage requests::"); + }), + ); +}); + +layer(NodeServices.layer)("Each scenario's stage requests", (it) => { + it.effect( + "records method, short path, status, trace and connection, and never a query, header or cookie", + () => + Effect.gen(function* () { + const name = expect.getState().currentTestName!; + // Effect's client puts each request in a trace of its own: the id the stage receives. + const traceId = expect.stringMatching(/^[0-9a-f]{32}$/u); + const answered = yield* stage("reset-on-reuse"); + const http = yield* HttpClient.HttpClient.pipe(Effect.provide(FetchHttpClient.layer)); + const response = yield* http.execute( + HttpClientRequest.get( + `${answered.origin}/api/organizations/0123456789abcdefABCDEF/inventory?token=synthetic-token`, + ).pipe( + HttpClientRequest.setHeaders({ + cookie: "session=synthetic-cookie", + authorization: "Bearer synthetic-bearer", + }), + ), + ); + expect(response.status).toBe(200); + const reset = yield* stage("reset-new"); + failureOf(yield* send("POST", `${reset.origin}/api/onboarding/prepare`, { body: {} })); + const { rows, dropped, limit } = stageRequestsOf(name); + expect({ dropped, limit }).toEqual({ dropped: 0, limit: stageRequestLimit }); + expect(rows).toEqual([ + { + method: "GET", + path: "/api/organizations/:id/inventory", + startedAt: expect.any(String), + endedAt: expect.any(String), + durationMs: expect.any(Number), + status: 200, + traceId, + connection: "new", + }, + { + method: "POST", + path: "/api/onboarding/prepare", + startedAt: expect.any(String), + endedAt: expect.any(String), + durationMs: expect.any(Number), + traceId, + failure: "ECONNRESET", + outcome: "lost: new connection", + connection: "new", + }, + ]); + const text = JSON.stringify(rows); + for (const secret of ["synthetic-token", "synthetic-cookie", "synthetic-bearer", "?"]) + expect(text).not.toContain(secret); + }), + ); + + it.effect("keeps at most its limit of rows, dropping the oldest and counting them", () => + Effect.gen(function* () { + const name = expect.getState().currentTestName!; + const { origin } = yield* stage("reset-on-reuse"); + const http = yield* HttpClient.HttpClient.pipe(Effect.provide(FetchHttpClient.layer)); + const extra = 3; + // One after another, so the oldest rows are the first paths. + for (let index = 0; index < stageRequestLimit + extra; index++) + expect((yield* http.get(`${origin}/n${index}`)).status).toBe(200); + const { rows, dropped } = stageRequestsOf(name); + expect(rows).toHaveLength(stageRequestLimit); + expect(dropped).toBe(extra); + expect(rows[0]!.path).toBe(`/n${extra}`); + expect(rows.at(-1)!.path).toBe(`/n${stageRequestLimit + extra - 1}`); + }), + ); +}); diff --git a/e2e/tests/mcp-catalog-scale.spec.ts b/e2e/tests/mcp-catalog-scale.spec.ts index b5852580f3..bc7470eb48 100644 --- a/e2e/tests/mcp-catalog-scale.spec.ts +++ b/e2e/tests/mcp-catalog-scale.spec.ts @@ -17,6 +17,7 @@ import { type TraceUnsettled, } from "../support/evidence.ts"; import { HostedLive, withHostedCase } from "../support/case.ts"; +import { machineWork, referenceMachineWorkMs } from "../support/machine-speed.ts"; import { McpClient } from "../support/mcp-client.ts"; import { expireEvaluated, writeRows } from "../support/cloud-rows.ts"; import { @@ -116,6 +117,25 @@ const Completed = Schema.Struct({ }), unavailableApps: Schema.Array(Schema.Struct({ app: Schema.String, reason: Schema.String })), }); +/** + * A search answers in part when an app's tools did not load in time, and names each such app in + * `unavailableApps`. This is checked before the tools the search found, so a partial answer fails + * as the product reporting apps unavailable and not as a missing tool. + */ +const expectNoneUnavailable = ( + step: string, + completed: typeof Completed.Type, + apps: ReadonlyArray<{ readonly id: string; readonly slug: string }>, +) => { + const reported = completed.unavailableApps; + const named = reported.map( + (entry) => `${apps.find((app) => app.id === entry.app)?.slug ?? entry.app} (${entry.reason})`, + ); + expect( + reported, + `${step}: the product reported ${reported.length} apps unavailable: ${named.join("; ")}`, + ).toEqual([]); +}; const SearchValue = Schema.Struct({ items: Schema.Array(Schema.Struct({ path: Schema.String, input: Schema.String })), }); @@ -132,12 +152,29 @@ const noAppStatements = 3; */ const oneAppStatements = 24; /** - * The first catalog-wide search evaluates all 29 apps and ranks 7,000 tools, 3.7–10.4 s on an - * M-series laptop depending on its load when search still rendered every signature. It must leave - * the program at least half of the 30 s execution budget. This bound has the least headroom; a - * failure on a slower runner is a regression to investigate, not a reason to raise it. + * The first catalog-wide search evaluates all 29 apps and ranks 7,000 tools. It must leave the + * program at least half of the 30 s execution budget: 15 s on the reference machine below. A + * runner's speed is not fixed: CI's 8 vCPU runners did the same work up to three times slower on + * weekday afternoons than at night (4.5 s to 15.1 s for this search, 14-16 Nov), so the scenario + * times a fixed piece of machine work (`machineWork`, support/machine-speed.ts) just before and + * after the search and allows + * 15 s scaled by how much slower than the reference the slower of the two was. On a machine faster + * than the reference it is still 15 s of real time. A failure after that scaling is a regression to + * investigate, not a reason to raise the budget or the reference. + * + * The search beside stalled apps has the same budget once discovery has given up (`searchBoundMs`). */ const catalogSearchBoundMs = 15_000; +/** + * A second check, on the shape of the search: the first catalog-wide search evaluates the 28 apps + * the call before it did not, and ranks 7,000 tools, about 20 times the cold call into one app + * ("Call one tool"). On main from 14 to 16 Nov the ratio stayed 12.7-24.0 (median 19.6, 54 runs) + * while the search took 4.5-15.1 s. The budget above leaves a normal machine about three times its + * search, so a search-only regression of two or three times passes it there; the ratio catches it. + * A regression in every app's evaluation slows the call too, so the ratio misses it; the budget + * catches that. 40 is twice the median. + */ +const catalogSearchPerCall = 40; /** * SQL statements a catalog-wide search that evaluates nothing runs for each app it searches: two * access checks of the member's role and groups (one to bind the app's accounts, one to search @@ -672,14 +709,17 @@ layer(HostedLive, { excludeTestServices: true })("MCP catalog scale", (it) => { }); yield* check(loadsOnly("Call one tool", called.traceId, [target], oneAppStatements)); yield* catalog.joinChecks; - // A catalog-wide search lists every app and finds the one matching tool. + // A catalog-wide search lists every app and finds the one matching tool. The machine is + // timed just before and just after it, as its speed moves within a job. + const machineBefore = yield* machineWork; const searchedAll = yield* execute("Search the whole catalog", needle); + const machineAfter = yield* machineWork; expect(searchedAll.completed.execution.ok).toBe(true); + expectNoneUnavailable("Search the whole catalog", searchedAll.completed, apps); expect( (yield* Schema.decodeUnknownEffect(SearchValue)(searchedAll.completed.execution.value)) .items, ).toMatchObject(found.map((path) => ({ path }))); - expect(searchedAll.completed.unavailableApps).toEqual([]); // It lists every app but the one called, whose search rows the call kept, and evaluates // each listing once. yield* check( @@ -690,7 +730,33 @@ layer(HostedLive, { excludeTestServices: true })("MCP catalog scale", (it) => { expect(cold.evaluations, "listings evaluated").toBe(cold.listed.size); }), ); - expect(searchedAll.elapsed).toBeLessThan(catalogSearchBoundMs); + const evidence = yield* Evidence; + const machine = Math.max(machineBefore, machineAfter); + // A slower machine than the reference gets proportionally more time; a faster one does not + // get less, so the bound is never below 15 s of real time. + const allowed = catalogSearchBoundMs * Math.max(1, machine / referenceMachineWorkMs); + yield* evidence.measure("Search the whole catalog", searchedAll.elapsed); + yield* evidence.measure("Call one tool", called.elapsed); + yield* evidence.measure("Machine work before the search", machineBefore); + yield* evidence.measure("Machine work after the search", machineAfter); + yield* Console.log( + JSON.stringify({ + label: "Search the whole catalog, calibrated", + elapsed: searchedAll.elapsed, + machine, + reference: referenceMachineWorkMs, + calibrated: Math.round((searchedAll.elapsed * referenceMachineWorkMs) / machine), + allowed: Math.round(allowed), + }), + ); + expect( + searchedAll.elapsed, + `Search the whole catalog took ${searchedAll.elapsed} ms; the machine work took ${machine} ms against the reference's ${referenceMachineWorkMs} ms, so it was allowed ${Math.round(allowed)} ms`, + ).toBeLessThan(allowed); + expect( + searchedAll.elapsed / called.elapsed, + `Search the whole catalog took ${searchedAll.elapsed} ms, Call one tool ${called.elapsed} ms`, + ).toBeLessThan(catalogSearchPerCall); // Later searches reuse the evaluated listings instead of evaluating all 29 apps again: // every app's search rows are kept, and no app is listed or evaluated. The three run one // after another; their traces are then read together. @@ -699,11 +765,11 @@ layer(HostedLive, { excludeTestServices: true })("MCP catalog scale", (it) => { const label = `Search the whole catalog again (${attempt})`; const searched = yield* execute(label, needle); expect(searched.completed.execution.ok).toBe(true); + expectNoneUnavailable(label, searched.completed, apps); expect( (yield* Schema.decodeUnknownEffect(SearchValue)(searched.completed.execution.value)) .items, ).toMatchObject(found.map((path) => ({ path }))); - expect(searched.completed.unavailableApps).toEqual([]); expect(searched.elapsed).toBeLessThan(repeatedSearchBoundMs); return { label, traceId: searched.traceId }; }), diff --git a/e2e/tests/mcp-connections.spec.ts b/e2e/tests/mcp-connections.spec.ts index d1198eb545..69976c1885 100644 --- a/e2e/tests/mcp-connections.spec.ts +++ b/e2e/tests/mcp-connections.spec.ts @@ -113,18 +113,31 @@ layer(HostedLive, { excludeTestServices: true })("Scoped MCP connections", (it) "Authorize the full-access URL for comparison", oauth.authorize, ); - const fullBearer = { - authorization: `Bearer ${Redacted.value(full.tokens).access_token}`, - }; + const fullRun = executor( + yield* mcp.connect(Redacted.make(Redacted.value(full.tokens).access_token), "full"), + ); + const agentListed = yield* fullRun( + "The agent lists the member's connections", + `return await tools.executor.mcpConnections.list(${JSON.stringify({ path: { organization: actors.organization.id } })});`, + ); + expect(agentListed.execution.ok, JSON.stringify(agentListed)).toBe(true); expect( - (yield* api.request( - yield* api.session(), - "GET", - `${prefix}/mcp-connections`, - undefined, - fullBearer, - )).status, - ).toBe(401); + (yield* Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ id: Schema.String })))( + agentListed.execution.value, + )).map((listedConnection) => listedConnection.id), + ).toContain(connection.id); + const changeTools = ["create", "update", "revoke", "agents", "revokeAgent"].map( + (operation) => `tools.executor.mcpConnections.${operation}`, + ); + const described = yield* fullRun( + "The agent has no tool that changes a connection", + `return await tools.search.describe({ paths: ${JSON.stringify(changeTools)} });`, + ); + expect( + (yield* Schema.decodeUnknownEffect( + Schema.Struct({ missing: Schema.Array(Schema.Struct({ path: Schema.String })) }), + )(described.execution.value)).missing.map((item) => item.path), + ).toEqual(changeTools); const grant = yield* evidence.step( "Authorize the connection's own URL through browser consent", oauth.authorizeConnection(connection.id), diff --git a/e2e/tests/mcp-direct-calls.spec.ts b/e2e/tests/mcp-direct-calls.spec.ts index 5265b579c7..476419b1db 100644 --- a/e2e/tests/mcp-direct-calls.spec.ts +++ b/e2e/tests/mcp-direct-calls.spec.ts @@ -57,6 +57,11 @@ const syntheticApprovalId = "apr_00000000-0000-4000-8000-000000000001"; const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNQWPX/PwAFgALJpImt0wAAAABJRU5ErkJggg=="; const wav = "UklGRiQAAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YQAAAAA="; +// The head and tail of a PDF, as a file API returns an attachment. +const pdf = Buffer.from("%PDF-1.4\n%\u00e2\u00e3\u00cf\u00d3\n%%EOF\n", "latin1").toString( + "base64", +); +const pdfUri = "file:///reports/Q3%20summary.pdf"; const mediaSource = `import { defineApp, query, object, router } from "apps"; export default defineApp({}, async () => ({ tools: router({ screenshot: query({ input: object({}), description: "Return a screenshot" }, @@ -67,6 +72,11 @@ export default defineApp({}, async () => ({ tools: router({ async () => ({ contentBase64url: ${JSON.stringify(png.replaceAll("/", "_").replaceAll("+", "-").replace(/=+$/u, ""))} })), lookalike: query({ input: object({}), description: "Return image-shaped data that is not base64" }, async () => ({ type: "image", mimeType: "image/png", data: "not base64!" })), + report: query({ input: object({}), description: "Return a PDF as unpadded base64url, and a text note, as embedded resources" }, + async () => ({ + file: { type: "resource", resource: { uri: ${JSON.stringify(pdfUri)}, mimeType: "application/pdf", blob: ${JSON.stringify(pdf.replaceAll("/", "_").replaceAll("+", "-").replace(/=+$/u, ""))} } }, + note: { type: "resource", resource: { uri: "file:///note.txt", mimeType: "text/plain", text: "read me" } }, + })), }) }));`; /** A personal access token and the app above, deployed for one scenario. */ @@ -280,6 +290,7 @@ layer(HostedLive, { excludeTestServices: true })("MCP direct calls", (it) => { expect(JSON.parse(String(text?.text))).toEqual(result.structuredContent); expect(String(text?.text)).not.toContain(png); expect(String(text?.text)).not.toContain(wav); + expect(String(text?.text)).not.toContain(pdf.slice(0, 16)); return { completed, media }; }); @@ -323,6 +334,37 @@ return { first: shot.content[0], audio: audio.content[0], again: shot, lookalike: { type: "image", mimeType: "image/png", data: "not base64!" }, }, }); + + // A file reaches the client as an embedded resource block, its blob in the standard form, + // so a client can save it and the model can read it. The result keeps its URI and type. + // A text resource is already readable and stays in the result. + const reported = yield* run("Call a tool that returns a PDF", "call", { + calls: { path: tool("report"), input: {} }, + }); + expect(reported.media).toEqual([ + { + type: "resource", + resource: { uri: pdfUri, mimeType: "application/pdf", blob: pdf }, + }, + ]); + expect(reported.completed.execution).toMatchObject({ + ok: true, + value: { + path: tool("report"), + ok: true, + value: { + file: { + type: "resource", + resource: { uri: pdfUri, mimeType: "application/pdf" }, + contentIndex: 1, + }, + note: { + type: "resource", + resource: { uri: "file:///note.txt", mimeType: "text/plain", text: "read me" }, + }, + }, + }, + }); }).pipe(Effect.provide(McpClient.layer)), ), ); diff --git a/e2e/tests/mcp-failure-reference.spec.ts b/e2e/tests/mcp-failure-reference.spec.ts index 0cb2fcd6bb..040cf5220a 100644 --- a/e2e/tests/mcp-failure-reference.spec.ts +++ b/e2e/tests/mcp-failure-reference.spec.ts @@ -357,6 +357,22 @@ layer(HostedLive, { excludeTestServices: true })("MCP failure reference", (it) = message: workerStopped, }); expect(forgedSearch.response.recovery?.action).toMatch(/^Try again once\. /); + // One more attempt is allowed, then unchanged retries stop. The match is unconfirmed and a + // later attempt can still pass, so the copy never says a repeat proves the cause, and it + // advises a rebuild only for an app built on an older apps version than the host's. + expect(forgedSearch.response.recovery?.action).toBe( + "Try again once. If it fails the same way, stop retrying it unchanged and investigate. If the app was built on an older apps version than this host’s, rebuild it on this host’s version and deploy it again; if it already runs this host’s version, or still fails, reduce the work the app does to list its tools.", + ); + const limitInstructions = forgedSearch.response.recovery?.instructions ?? ""; + expect(limitInstructions).toContain("Retry the unchanged request at most once."); + expect(limitInstructions).toContain( + "stop retrying the unchanged request, even though a later attempt might succeed, and investigate", + ); + expect(limitInstructions).toContain( + "Compare the apps version in the app's package.json with framework.release, the version this host runs. If the app's version is older", + ); + for (const claim of ["not transient", "will not help"]) + expect(limitInstructions + forgedSearch.response.recovery?.action).not.toContain(claim); expect(forgedSearch.response.recovery?.instructions).not.toContain( "exceeded its CPU time limit.", ); @@ -373,6 +389,10 @@ layer(HostedLive, { excludeTestServices: true })("MCP failure reference", (it) = expect(forgedExecute.error.message.endsWith(located("discover", forgedExecute.trace))).toBe( true, ); + // The agent's summary line carries the same recovery. + expect(forgedExecute.error.message).toContain( + ` Recovery: ${forgedSearch.response.recovery?.action} Retryable (unchanged call): yes.`, + ); // The framework's refusal without details is named as the framework's report. expect(duplicateSearch.response).toMatchObject({ code: "AppEvaluationFailed", diff --git a/e2e/tests/npm-channel.spec.ts b/e2e/tests/npm-channel.spec.ts new file mode 100644 index 0000000000..19df796d12 --- /dev/null +++ b/e2e/tests/npm-channel.spec.ts @@ -0,0 +1,223 @@ +/** + * `scripts/releases/npm-channel.ts` waits for npm's dist-tags to name a release npm already serves, + * as `publish-npm.ts` does after publishing. These cases run it as a process against a local + * registry that replays what npm's dist-tags endpoint answered after beta.51's publish: the old + * tag for a while, then the new one; or the old one for good. + */ +import { expect, layer } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { createServer, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { Effect, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +type Tags = Readonly>; + +/** One answer of the dist-tags endpoint: tags, a status with a body, or no answer at all. */ +type Answer = Tags | { readonly status: number; readonly body: string } | "hang"; + +const isStatus = (answer: Answer): answer is { readonly status: number; readonly body: string } => + typeof answer === "object" && typeof answer.status === "number"; + +/** + * A registry whose `/-/package/executor/dist-tags` gives `answers` in order, repeating the last. + * Counts the requests. + */ +const registry = (answers: ReadonlyArray) => + Effect.acquireRelease( + Effect.callback<{ url: string; requests: () => number; close: () => void }>((resume) => { + let requests = 0; + const hanging: Array = []; + const server = createServer((request, response) => { + const answer = answers[Math.min(requests, answers.length - 1)]; + if (request.url !== "/-/package/executor/dist-tags" || answer === undefined) { + response.writeHead(404).end(); + return; + } + requests += 1; + if (answer === "hang") { + hanging.push(response); + return; + } + const [status, body] = isStatus(answer) + ? [answer.status, answer.body] + : [200, JSON.stringify(answer)]; + response.writeHead(status, { "content-type": "application/json" }); + response.end(body); + }); + server.listen(0, "127.0.0.1", () => + resume( + Effect.succeed({ + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + requests: () => requests, + close: () => { + for (const response of hanging) response.destroy(); + server.close(); + }, + }), + ), + ); + }), + (stub) => Effect.sync(stub.close), + ); + +const before: Tags = { latest: "1.6.11", beta: "2.0.0-beta.50" }; +const published: Tags = { latest: "1.6.11", beta: "2.0.0-beta.51" }; + +/** The wait as a process, against `registry` with a 3 s limit, unless `env` says otherwise. */ +const wait = (registry: string, env: Readonly> = {}) => + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* processes.spawn( + ChildProcess.make("node", ["scripts/releases/npm-channel.ts"], { + env: { + NPM_CHANNEL_REGISTRY: registry, + NPM_CHANNEL: "beta", + NPM_CHANNEL_VERSION: "2.0.0-beta.51", + NPM_CHANNEL_BEFORE: JSON.stringify(before), + NPM_CHANNEL_LIMIT_SECONDS: "3", + NPM_CHANNEL_POLL_SECONDS: "0.1", + NPM_CHANNEL_ATTEMPT_SECONDS: "0.5", + ...env, + }, + extendEnv: true, + }), + ); + const started = Date.now(); + const [log, exitCode] = yield* Effect.all( + [child.all.pipe(Stream.decodeText(), Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + return { exitCode: Number(exitCode), log, seconds: (Date.now() - started) / 1000 }; + }).pipe(Effect.scoped); + +layer(NodeServices.layer)("publication waits for npm's channel tag", (it) => { + it.effect("a tag npm moves after it serves the version is waited for", () => + Effect.gen(function* () { + const stub = yield* registry([before, before, before, published]); + const result = yield* wait(stub.url); + expect(result.exitCode, result.log).toBe(0); + expect(result.log).toContain("npm's beta tag names 2.0.0-beta.51; latest=1.6.11"); + expect(result.log).toContain("npm's beta tag still names 2.0.0-beta.50"); + expect(stub.requests()).toBe(4); + }).pipe(Effect.scoped), + ); + + it.effect("a tag that never moves fails at the limit, naming what it still says", () => + Effect.gen(function* () { + const stub = yield* registry([before]); + const result = yield* wait(stub.url); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain("its beta tag did not name it within 3 s"); + expect(result.log).toContain("the last read: npm's beta tag still names 2.0.0-beta.50"); + expect(result.seconds).toBeLessThan(15); + expect(stub.requests()).toBeGreaterThan(5); + }).pipe(Effect.scoped), + ); + + it.effect("a beta publication that moved latest fails at once", () => + Effect.gen(function* () { + const stub = yield* registry([{ latest: "2.0.0-beta.51", beta: "2.0.0-beta.51" }]); + const result = yield* wait(stub.url); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain("latest tag names 2.0.0-beta.51, not 1.6.11"); + expect(stub.requests()).toBe(1); + }).pipe(Effect.scoped), + ); + + it.effect("a tag naming some older third version fails at once", () => + Effect.gen(function* () { + const stub = yield* registry([{ latest: "1.6.11", beta: "2.0.0-beta.49" }]); + const result = yield* wait(stub.url); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain("names 2.0.0-beta.49: neither 2.0.0-beta.51 nor 2.0.0-beta.50"); + expect(stub.requests()).toBe(1); + }).pipe(Effect.scoped), + ); + + it.effect("a tag naming a newer version fails at once: the run is stale", () => + Effect.gen(function* () { + const stub = yield* registry([{ latest: "1.6.11", beta: "2.0.0-beta.52" }]); + const result = yield* wait(stub.url); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain( + "npm's beta tag names 2.0.0-beta.52, newer than 2.0.0-beta.51: a newer release has published since, so this run is stale. Do not re-run it", + ); + expect(stub.requests()).toBe(1); + }).pipe(Effect.scoped), + ); + + it.effect( + "an old run re-run after a newer release stops on the first read, not at the limit", + () => + Effect.gen(function* () { + const newer = { latest: "1.6.11", beta: "2.0.0-beta.52" }; + const stub = yield* registry([newer]); + // Its tags before publishing were read when the re-run started, after the newer release. + const result = yield* wait(stub.url, { NPM_CHANNEL_BEFORE: JSON.stringify(newer) }); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain("this run is stale"); + expect(stub.requests()).toBe(1); + }).pipe(Effect.scoped), + ); + + it.effect("an answer without a latest tag is asked again, not taken for a moved latest", () => + Effect.gen(function* () { + const stub = yield* registry([{ beta: "2.0.0-beta.51" }, {}, published]); + const result = yield* wait(stub.url); + expect(result.exitCode, result.log).toBe(0); + expect(result.log).toContain("the registry answered without a latest tag"); + expect(result.log).not.toContain("Restore latest"); + expect(stub.requests()).toBe(3); + }).pipe(Effect.scoped), + ); + + it.effect("an error, an unreadable answer and no answer are asked again", () => + Effect.gen(function* () { + const stub = yield* registry([ + { status: 503, body: "{}" }, + { status: 200, body: "not json" }, + "hang", + published, + ]); + const result = yield* wait(stub.url); + expect(result.exitCode, result.log).toBe(0); + expect(result.log).toContain("the registry answered 503"); + expect(result.log).toContain("the registry answered unreadably"); + expect(result.log).toContain("the registry did not answer within 0.5 s"); + expect(stub.requests()).toBe(4); + }).pipe(Effect.scoped), + ); + + it.effect("a resumed job whose tag already names the release passes on one read", () => + Effect.gen(function* () { + const stub = yield* registry([published]); + const result = yield* wait(stub.url, { NPM_CHANNEL_BEFORE: JSON.stringify(published) }); + expect(result.exitCode, result.log).toBe(0); + expect(stub.requests()).toBe(1); + }).pipe(Effect.scoped), + ); + + it.effect("a stable release waits for latest and may move it", () => + Effect.gen(function* () { + const stub = yield* registry([before, { latest: "2.0.0", beta: "2.0.0-beta.50" }]); + const result = yield* wait(stub.url, { NPM_CHANNEL: "latest", NPM_CHANNEL_VERSION: "2.0.0" }); + expect(result.exitCode, result.log).toBe(0); + expect(stub.requests()).toBe(2); + }).pipe(Effect.scoped), + ); + + it.effect("npm's own wait cannot be shortened, however npm's address is written", () => + Effect.gen(function* () { + for (const npm of [ + "https://registry.npmjs.org", + "https://registry.npmjs.org/", + "https://REGISTRY.npmjs.org/", + ]) { + const result = yield* wait(npm); + expect(result.exitCode, `${npm}\n${result.log}`).toBe(1); + expect(result.log).toContain("apply only to another registry"); + } + }).pipe(Effect.scoped), + ); +}); diff --git a/e2e/tests/prepare-cache.spec.ts b/e2e/tests/prepare-cache.spec.ts index 7183e40ce7..c49fc38dae 100644 --- a/e2e/tests/prepare-cache.spec.ts +++ b/e2e/tests/prepare-cache.spec.ts @@ -10,7 +10,8 @@ import { ChildProcess, ChildProcessSpawner } from "effect/process"; const steps = [ "apps:build", - "telemetry:build", + "collector:build", + "viewer:build", "e2e:runtime", "e2e:apps", "hosted:self-host:web:build", @@ -18,9 +19,19 @@ const steps = [ "e2e:npm", ] as const; +/** + * Steps whose command runs every time, never this cache: the collector's build script checks the + * pinned toolchain and reuses builds through its own identity and store. + */ +const uncached: ReadonlyArray<(typeof steps)[number]> = ["collector:build"]; +/** `ran` as a run lists it: these steps, and every uncached one, in the order of `steps`. */ +const runs = (built: ReadonlyArray) => + steps.filter((step) => built.includes(step) || uncached.includes(step)); + const outputs: Record<(typeof steps)[number], string> = { "apps:build": "packages/apps/dist/index.js", - "telemetry:build": "packages/telemetry/dist/motel/index.js", + "collector:build": "packages/telemetry/dist/collector/executor-collector", + "viewer:build": "packages/telemetry/dist/viewer/index.html", "e2e:runtime": ".local/test-runtime/host.json", "e2e:apps": ".local/test-runtime/apps.tgz", "hosted:self-host:web:build": "apps/hosted/self-host/web/dist/index.html", @@ -43,6 +54,8 @@ import { dirname } from "node:path"; const [step, ...outputs] = process.argv.slice(2); const state = process.env.FAKE_STATE; if (process.env.FAKE_FAIL === step) process.exit(3); +if (process.env.FAKE_NOTE === step) + appendFileSync(process.env.EXECUTOR_PREPARE_NOTES, "The shared store cannot be written (EACCES).\\n"); let ran = []; try { ran = readFileSync(\`\${state}/ran\`, "utf8").split("\\n").filter(Boolean); @@ -61,13 +74,19 @@ const workspace = (name: string, dependencies: ReadonlyArray = []) => }); /** - * The layout the script reads: `@fake/utils` <- `apps` <- `@fake/sdk` <- local web, and - * `@fake/ui` <- both dashboards. + * The layout the script reads: `@fake/utils` <- `apps` <- `@fake/sdk` <- local web, + * `@fake/telemetry` <- `@fake/sdk`, and `@fake/ui` <- both dashboards. As in executor-next, the + * telemetry package holds the collector's crate, its build script and the viewer's package. */ const files: Record = { "package.json": JSON.stringify({ type: "module", - workspaces: ["packages/*", "apps/local/web", "apps/hosted/self-host/web"], + workspaces: [ + "packages/*", + "packages/telemetry/viewer", + "apps/local/web", + "apps/hosted/self-host/web", + ], scripts: { "e2e:prepare": "node scripts/e2e-prepare.ts", ...Object.fromEntries( @@ -87,13 +106,17 @@ const files: Record = { "packages/utils/index.ts": "export const utils = 1;\n", "packages/apps/package.json": workspace("apps", ["@fake/utils"]), "packages/apps/index.ts": "export const apps = 1;\n", - "packages/sdk/package.json": workspace("@fake/sdk", ["apps"]), + "packages/sdk/package.json": workspace("@fake/sdk", ["apps", "@fake/telemetry"]), "packages/sdk/index.ts": "export const sdk = 1;\n", "packages/ui/package.json": workspace("@fake/ui"), "packages/ui/index.ts": "export const ui = 1;\n", "packages/telemetry/package.json": workspace("@fake/telemetry"), - "packages/telemetry/motel/source.json": "{}\n", - "packages/telemetry/scripts/build-motel.ts": "\n", + "packages/telemetry/src/index.ts": "export const telemetry = 1;\n", + "packages/telemetry/collector/Cargo.toml": "\n", + "packages/telemetry/collector/src/main.rs": "\n", + "packages/telemetry/scripts/build-collector.ts": "\n", + "packages/telemetry/viewer/package.json": workspace("@fake/telemetry-viewer"), + "packages/telemetry/viewer/index.ts": "\n", "playground/demo-apps/live-inbox/index.ts": "\n", "scripts/build-test-runtime.ts": "\n", "scripts/pack-test-apps.ts": "\n", @@ -191,9 +214,11 @@ layer(NodeServices.layer, { excludeTestServices: true })("Cached e2e:prepare", ( yield* fs.writeFileString(path.join(input.repo, "apps/local/web/dist/extra.js"), "stale\n"); const second = yield* prepare(input); expect(second.exitCode).toBe(0); - expect(second.ran).toEqual([]); - for (const step of steps) expect(second.stdout).toContain(`${step}: cached`); + // The collector's build script runs again, as it does every time; every other step is cached. + expect(second.ran).toEqual(uncached); for (const step of steps) + expect(second.stdout).toContain(`${step}: ${uncached.includes(step) ? "built" : "cached"}`); + for (const step of steps.filter((step) => !uncached.includes(step))) expect(yield* read(input, outputs[step])).toBe( `${step} build ${steps.indexOf(step) + 1}\n`, ); @@ -209,7 +234,13 @@ layer(NodeServices.layer, { excludeTestServices: true })("Cached e2e:prepare", ( ["packages/ui/index.ts", ["hosted:self-host:web:build", "web:build"]], // apps <- sdk <- local web; e2e:apps packs the apps build. ["packages/utils/index.ts", ["apps:build", "e2e:runtime", "e2e:apps", "web:build"]], - ["packages/telemetry/motel/source.json", ["telemetry:build"]], + // The telemetry package's own sources are inputs of what depends on it: sdk <- local web. + ["packages/telemetry/src/index.ts", ["e2e:runtime", "web:build"]], + // Its crate, build script and nested viewer package are not: only their own steps. + ["packages/telemetry/collector/src/main.rs", ["collector:build"]], + ["packages/telemetry/collector/README.md", ["collector:build"]], + ["packages/telemetry/scripts/build-collector.ts", ["collector:build"]], + ["packages/telemetry/viewer/index.ts", ["viewer:build"]], ["playground/demo-apps/live-inbox/index.ts", ["apps:build", "e2e:apps"]], ["apps/hosted/self-host/package.json", ["hosted:self-host:web:build"]], ["scripts/pack-test-apps.ts", ["e2e:apps"]], @@ -223,7 +254,7 @@ layer(NodeServices.layer, { excludeTestServices: true })("Cached e2e:prepare", ( ]; for (const [file, rebuilt] of changes) { yield* append(input, file); - expect((yield* prepare(input)).ran, file).toEqual(rebuilt); + expect((yield* prepare(input)).ran, file).toEqual(runs(rebuilt)); } }).pipe(Effect.scoped), ); @@ -237,11 +268,16 @@ layer(NodeServices.layer, { excludeTestServices: true })("Cached e2e:prepare", ( const file = path.join(input.repo, "packages/ui/index.ts"); const original = yield* fs.readFileString(file); yield* append(input, "packages/ui/index.ts"); - expect((yield* prepare(input)).ran).toEqual(["hosted:self-host:web:build", "web:build"]); + expect((yield* prepare(input)).ran).toEqual( + runs(["hosted:self-host:web:build", "web:build"]), + ); yield* fs.writeFileString(file, original); const reverted = yield* prepare(input); - expect(reverted.ran).toEqual([]); - expect(yield* read(input, outputs["web:build"])).toBe("web:build build 6\n"); + expect(reverted.ran).toEqual(runs([])); + // The first run's build, restored: its number is the step's place in that run. + expect(yield* read(input, outputs["web:build"])).toBe( + `web:build build ${steps.indexOf("web:build") + 1}\n`, + ); }).pipe(Effect.scoped), ); @@ -259,21 +295,52 @@ layer(NodeServices.layer, { excludeTestServices: true })("Cached e2e:prepare", ( }).pipe(Effect.scoped), ); + it.effect("a skipped step neither runs nor touches its outputs", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const input = yield* repository; + // As CI does: the collector is fetched after prepare, from its run's collector job. + const before = yield* prepare(input, { args: ["--skip", "collector:build"] }); + expect(before.exitCode, before.stdout).toBe(0); + expect(before.ran).toEqual(steps.filter((step) => step !== "collector:build")); + expect(before.stdout).toContain("collector:build: skipped, provided elsewhere"); + const fetched = path.join(input.repo, outputs["collector:build"]); + expect(yield* fs.exists(fetched), "nothing wrote the skipped step's output").toBe(false); + yield* fs.makeDirectory(path.dirname(fetched), { recursive: true }); + yield* fs.writeFileString(fetched, "fetched\n"); + const after = yield* prepare(input, { args: ["--skip", "collector:build"] }); + expect(after.exitCode, after.stdout).toBe(0); + expect(after.ran).not.toContain("collector:build"); + expect(yield* read(input, outputs["collector:build"])).toBe("fetched\n"); + const unknown = yield* prepare(input, { args: ["--skip", "collector"] }); + expect(unknown.exitCode).toBe(2); + expect(unknown.ran).toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect("what a step notes is repeated in the summary", () => + Effect.gen(function* () { + const input = yield* repository; + const run = yield* prepare(input, { env: { FAKE_NOTE: "collector:build" } }); + expect(run.exitCode, run.stdout).toBe(0); + expect(run.stdout).toMatch( + /collector:build: built in [\d.]+s\n collector:build: The shared store cannot be written \(EACCES\)\.\n/u, + ); + }).pipe(Effect.scoped), + ); + it.effect("a failing step stops the run and stores nothing for it", () => Effect.gen(function* () { const input = yield* repository; const failed = yield* prepare(input, { env: { FAKE_FAIL: "e2e:runtime" } }); expect(failed.exitCode).toBe(3); - expect(failed.ran).toEqual(["apps:build", "telemetry:build"]); + expect(failed.ran).toEqual(["apps:build", "collector:build", "viewer:build"]); const next = yield* prepare(input); expect(next.exitCode).toBe(0); - expect(next.ran).toEqual([ - "e2e:runtime", - "e2e:apps", - "hosted:self-host:web:build", - "web:build", - "e2e:npm", - ]); + expect(next.ran).toEqual( + runs(["e2e:runtime", "e2e:apps", "hosted:self-host:web:build", "web:build", "e2e:npm"]), + ); }).pipe(Effect.scoped), ); }); diff --git a/e2e/tests/publish-guard.spec.ts b/e2e/tests/publish-guard.spec.ts new file mode 100644 index 0000000000..65605c277b --- /dev/null +++ b/e2e/tests/publish-guard.spec.ts @@ -0,0 +1,1228 @@ +/** + * `scripts/releases/channel-guard.ts` stops a publish step that would move a release channel + * backwards, and `scripts/releases/publication-report.ts` says what a failed publish job made + * public. These cases run both as processes against one local stub that answers as each public + * surface does: npm's dist-tags and versions, GHCR's token, manifests and image configs, the + * desktop feed's files, the tap's formula and the GitHub release. No case reaches the network. + */ +import { expect, layer } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { createHash } from "node:crypto"; +import { createServer, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { Effect, FileSystem, Path, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; + +const repository = "SyntheticOrg/executor"; +const image = "ghcr.io/syntheticorg/executor-selfhost"; +const imageRepository = "syntheticorg/executor-selfhost"; +const tap = "SyntheticOrg/homebrew-tap"; +const feedTag = "executor-v2-desktop-updates"; + +/** What each public surface names. Absent keys answer 404. */ +type State = { + readonly npmTags?: Readonly>; + readonly npmVersions?: ReadonlyArray; + /** Versions npm holds with this archive, and when and whether it deprecated them. */ + readonly npmArchives?: Readonly< + Record< + string, + { readonly integrity: string; readonly deprecated?: string; readonly time?: string } + > + >; + /** Image tags and the version label of the image each names. */ + readonly images?: Readonly>; + /** Feed file names and the version each names. */ + readonly feed?: Readonly>; + /** Formula names (`executor-beta`) and the version each names. */ + readonly formulas?: Readonly>; + readonly release?: { readonly draft: boolean; readonly assets: number; readonly target?: string }; + /** Every request answers 500. */ + readonly broken?: boolean; + /** These surfaces answer 500 to every request. */ + readonly failing?: ReadonlyArray<"npm" | "ghcr" | "feed" | "raw" | "api">; + /** These surfaces never answer. */ + readonly hanging?: ReadonlyArray<"npm" | "ghcr" | "feed" | "raw" | "api">; + /** npm's dist-tags answer 503 to this many requests first. */ + readonly flakyTags?: number; +}; + +const json = (body: unknown) => ({ status: 200, body: JSON.stringify(body) }); +const missing = { status: 404, body: "{}" }; + +/** The answer `state` gives for `url`. Image manifests are an index, then one image, then a config. */ +/** Which surface a route belongs to, for `failing`. */ +const surfaceOf = (route: string) => + route === "/token" || route.startsWith("/v2/") + ? "ghcr" + : route.startsWith("/repos/") + ? "api" + : route.includes("/releases/download/") + ? "feed" + : route.includes("/HEAD/Formula/") + ? "raw" + : "npm"; + +const answer = (state: State, url: string, tagReads: number): { status: number; body: string } => { + if (state.broken === true) return { status: 500, body: "{}" }; + const route = new URL(url, "http://stub").pathname; + if (state.failing?.includes(surfaceOf(route)) === true) return { status: 500, body: "{}" }; + if (route === "/-/package/executor/dist-tags") { + if (tagReads <= (state.flakyTags ?? 0)) return { status: 503, body: "{}" }; + return state.npmTags === undefined ? missing : json(state.npmTags); + } + if (route === "/executor") + return json({ + time: Object.fromEntries( + Object.entries(state.npmArchives ?? {}).flatMap(([held, archive]) => + archive.time === undefined ? [] : [[held, archive.time]], + ), + ), + }); + const version = /^\/executor\/(.+)$/u.exec(route)?.[1]; + if (version !== undefined) { + const archive = state.npmArchives?.[version]; + if (archive !== undefined) + return json({ + version, + dist: { integrity: archive.integrity }, + ...(archive.deprecated === undefined ? {} : { deprecated: archive.deprecated }), + }); + return state.npmVersions?.includes(version) === true + ? json({ version, dist: { integrity: "sha512-synthetic" } }) + : missing; + } + if (route === "/token") return json({ token: "synthetic-token" }); + const manifest = new RegExp(`^/v2/${imageRepository}/manifests/(.+)$`, "u").exec(route)?.[1]; + if (manifest !== undefined) { + const single = /^sha256:image-(.+)$/u.exec(manifest)?.[1]; + if (single !== undefined) return json({ config: { digest: `sha256:config-${single}` } }); + if (state.images?.[manifest] === undefined) return missing; + return json({ + manifests: [ + { digest: `sha256:image-${manifest}`, platform: { os: "linux", architecture: "amd64" } }, + ], + }); + } + const config = new RegExp(`^/v2/${imageRepository}/blobs/sha256:config-(.+)$`, "u").exec( + route, + )?.[1]; + if (config !== undefined) { + const labelled = state.images?.[config]; + return labelled === undefined + ? missing + : json({ config: { Labels: { "org.opencontainers.image.version": labelled } } }); + } + const file = new RegExp(`^/${repository}/releases/download/${feedTag}/(.+)$`, "u").exec( + route, + )?.[1]; + if (file !== undefined) { + const named = state.feed?.[file]; + return named === undefined ? missing : json({ version: named, files: [] }); + } + const formula = new RegExp(`^/${tap}/HEAD/Formula/(.+)\\.rb$`, "u").exec(route)?.[1]; + if (formula !== undefined) { + const named = state.formulas?.[formula]; + return named === undefined + ? missing + : { status: 200, body: `class ExecutorBeta < Formula\n version "${named}"\nend\n` }; + } + if (route.startsWith(`/repos/${repository}/releases/tags/`)) + return state.release === undefined + ? missing + : json({ + draft: state.release.draft, + assets: Array.from({ length: state.release.assets }), + ...(state.release.target === undefined ? {} : { target_commitish: state.release.target }), + }); + return missing; +}; + +/** One stub serving every surface from `state`. */ +const surfaces = (state: State) => + Effect.acquireRelease( + Effect.callback<{ url: string; paths: () => ReadonlyArray; close: () => void }>( + (resume) => { + let tagReads = 0; + const asked: Array = []; + const hung: Array = []; + const server = createServer((request, response) => { + const route = new URL(request.url ?? "/", "http://stub").pathname; + asked.push(route); + if (state.hanging?.includes(surfaceOf(route)) === true) { + hung.push(response); + return; + } + if ( + new URL(request.url ?? "/", "http://stub").pathname === "/-/package/executor/dist-tags" + ) + tagReads += 1; + const { status, body } = answer(state, request.url ?? "/", tagReads); + response.writeHead(status, { "content-type": "application/json" }); + response.end(body); + }); + server.listen(0, "127.0.0.1", () => + resume( + Effect.succeed({ + url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + paths: () => [...asked], + close: () => { + for (const response of hung) response.destroy(); + server.close(); + }, + }), + ), + ); + }, + ), + (stub) => Effect.sync(stub.close), + ); + +/** Runs `script` with `args` against the stub, as the publish job would for VERSION and CHANNEL. */ +const run = ( + stub: string, + script: "channel-guard.ts" | "publication-report.ts", + args: ReadonlyArray, + env: Readonly>, +) => + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* processes.spawn( + ChildProcess.make("node", [`scripts/releases/${script}`, ...args], { + env: { + EXECUTOR_PUBLIC_NPM: stub, + EXECUTOR_PUBLIC_GHCR: stub, + EXECUTOR_PUBLIC_GITHUB: stub, + EXECUTOR_PUBLIC_GITHUB_API: stub, + EXECUTOR_PUBLIC_RAW: stub, + EXECUTOR_PUBLIC_RETRY_SECONDS: "0.05", + IMAGE: image, + PUBLIC_REPOSITORY: repository, + RELEASE_TAG: `executor@${env.VERSION ?? ""}`, + GITHUB_RUN_ID: "4242", + ...env, + }, + extendEnv: true, + }), + ); + const [log, exitCode] = yield* Effect.all( + [child.all.pipe(Stream.decodeText(), Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + return { exitCode: Number(exitCode), log }; + }).pipe(Effect.scoped); + +const guard = (state: State, surface: string, version: string, extra: ReadonlyArray = []) => + Effect.gen(function* () { + const stub = yield* surfaces(state); + return yield* run(stub.url, "channel-guard.ts", [surface, ...extra], { + VERSION: version, + CHANNEL: version.includes("-beta.") ? "beta" : "latest", + }); + }).pipe(Effect.scoped); + +const betaFeed = (version: string) => ({ + "executor-v2-beta-mac.yml": version, + "executor-v2-beta.yml": version, + "executor-v2-beta-linux.yml": version, + "executor-v2-beta-linux-arm64.yml": version, +}); + +layer(NodeServices.layer)("a publish step never moves a channel backwards", (it) => { + it.effect( + "an image channel naming a newer beta stops the step, ordering beta.10 after beta.9", + () => + Effect.gen(function* () { + const result = yield* guard({ images: { beta: "2.0.0-beta.10" } }, "image", "2.0.0-beta.9"); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain( + `${image}:beta names 2.0.0-beta.10: newer than 2.0.0-beta.9. Nothing was changed`, + ); + }), + ); + + it.effect("the same, an older or no image channel lets the step go on", () => + Effect.gen(function* () { + for (const [images, version] of [ + [{ beta: "2.0.0-beta.10" }, "2.0.0-beta.10"], + [{ beta: "2.0.0-beta.9" }, "2.0.0-beta.10"], + [{}, "2.0.0-beta.10"], + ] as const) { + const result = yield* guard({ images }, "image", version); + expect(result.exitCode, result.log).toBe(0); + } + }), + ); + + it.effect("npm's tag naming a newer release stops the npm step", () => + Effect.gen(function* () { + const result = yield* guard( + { npmTags: { latest: "1.6.11", beta: "2.0.0-beta.52" } }, + "npm", + "2.0.0-beta.51", + ); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain("npm's executor@beta names 2.0.0-beta.52: newer than"); + }), + ); + + it.effect("a stable release may move latest past Executor 1, and never back", () => + Effect.gen(function* () { + const forward = yield* guard({ npmTags: { latest: "1.6.11" } }, "npm", "2.0.0"); + expect(forward.exitCode, forward.log).toBe(0); + const back = yield* guard({ npmTags: { latest: "2.0.1" } }, "npm", "2.0.0"); + expect(back.exitCode, back.log).toBe(1); + }), + ); + + it.effect("the tap's formula naming a newer beta stops the Homebrew step", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "executor-tap-" }); + yield* fs.makeDirectory(path.join(directory, "Formula")); + const fresh = yield* guard({}, "homebrew", "2.0.0-beta.51", [directory]); + expect(fresh.exitCode, fresh.log).toBe(0); + yield* fs.writeFileString( + path.join(directory, "Formula/executor-beta.rb"), + 'class ExecutorBeta < Formula\n version "2.0.0-beta.52"\nend\n', + ); + const result = yield* guard({}, "homebrew", "2.0.0-beta.51", [directory]); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain("the executor-beta formula names 2.0.0-beta.52: newer than"); + }).pipe(Effect.scoped), + ); + + it.effect("the Homebrew step's check stops when the tap directory is missing", () => + Effect.gen(function* () { + const result = yield* guard({}, "homebrew", "2.0.0-beta.51", ["/nonexistent/executor-tap"]); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain( + "The tap directory /nonexistent/executor-tap does not exist. Nothing was changed", + ); + }), + ); + + it.effect("one desktop feed file naming a newer beta stops the feed step", () => + Effect.gen(function* () { + const feed = { ...betaFeed("2.0.0-beta.51"), "executor-v2-beta-linux.yml": "2.0.0-beta.52" }; + const result = yield* guard({ feed }, "desktop-feed", "2.0.0-beta.51"); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain("executor-v2-beta-linux.yml names 2.0.0-beta.52"); + const ok = yield* guard({ feed: betaFeed("2.0.0-beta.50") }, "desktop-feed", "2.0.0-beta.51"); + expect(ok.exitCode, ok.log).toBe(0); + }), + ); + + it.effect("a beta never moves the stable channel", () => + Effect.gen(function* () { + const stub = yield* surfaces({ npmTags: { latest: "1.6.11" } }); + const result = yield* run(stub.url, "channel-guard.ts", ["npm"], { + VERSION: "2.0.0-beta.51", + CHANNEL: "latest", + }); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain("a beta never moves the stable channel"); + }).pipe(Effect.scoped), + ); + + it.effect("a value it cannot order, or a surface it cannot read, stops the step", () => + Effect.gen(function* () { + const odd = yield* guard( + { npmTags: { latest: "1.6.11", beta: "next" } }, + "npm", + "2.0.0-beta.51", + ); + expect(odd.exitCode, odd.log).toBe(1); + expect(odd.log).toContain("cannot be ordered"); + const broken = yield* guard({ broken: true }, "npm", "2.0.0-beta.51"); + expect(broken.exitCode, broken.log).toBe(1); + expect(broken.log).toContain("after about two minutes of asking"); + expect(broken.log).toContain("answered 500"); + }), + ); + + it.effect("a step's own check outlasts a short outage that quick reads would not", () => + Effect.gen(function* () { + // Four failed reads: the quick schedule's four reads all fail; the patient one's fifth answers. + const result = yield* guard( + { npmTags: { latest: "1.6.11", beta: "2.0.0-beta.51" }, flakyTags: 4 }, + "npm", + "2.0.0-beta.52", + ); + expect(result.exitCode, result.log).toBe(0); + expect(result.log).toContain( + "npm's executor@beta names 2.0.0-beta.51; 2.0.0-beta.52 may move it", + ); + }), + ); + + it.effect( + "npm's dist-tags without a latest tag are unreadable, not a channel naming nothing", + () => + Effect.gen(function* () { + const result = yield* guard({ npmTags: { beta: "2.0.0-beta.50" } }, "npm", "2.0.0-beta.51"); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain("answered without a latest tag"); + }), + ); +}); + +/** Preflight against `state`, as the publish job's first step runs it, with the tap. */ +const preflight = (state: State, version: string) => + Effect.gen(function* () { + const stub = yield* surfaces(state); + return yield* run(stub.url, "channel-guard.ts", ["preflight"], { + VERSION: version, + CHANNEL: "beta", + TAP: tap, + }); + }).pipe(Effect.scoped); + +/** Every channel a step after beta.51 finds, before beta.52 publishes. */ +const afterBeta51: State = { + npmTags: { latest: "1.6.11", beta: "2.0.0-beta.51" }, + images: { beta: "2.0.0-beta.51" }, + formulas: { "executor-beta": "2.0.0-beta.50" }, + feed: betaFeed("2.0.0-beta.50"), +}; + +layer(NodeServices.layer)("a stale publish stops before anything moves", (it) => { + it.effect("a release newer than every channel passes the first step", () => + Effect.gen(function* () { + const result = yield* preflight(afterBeta51, "2.0.0-beta.52"); + expect(result.exitCode, result.log).toBe(0); + expect(result.log).toContain( + "npm's executor@beta names 2.0.0-beta.51; 2.0.0-beta.52 may move it", + ); + expect(result.log).toContain(`${image}:beta names 2.0.0-beta.51`); + expect(result.log).toContain(`${tap}'s executor-beta formula names 2.0.0-beta.50`); + expect(result.log).toContain("executor-v2-beta-mac.yml names 2.0.0-beta.50"); + }), + ); + + it.effect( + "an old release re-run after a newer one stops at the first step, naming the channel", + () => + Effect.gen(function* () { + const result = yield* preflight(afterBeta51, "2.0.0-beta.50"); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain( + `npm's executor@beta names 2.0.0-beta.51; ${image}:beta names 2.0.0-beta.51: newer than 2.0.0-beta.50. Nothing was changed`, + ); + }), + ); + + it.effect("a newer formula alone stops the first step too", () => + Effect.gen(function* () { + const result = yield* preflight( + { ...afterBeta51, formulas: { "executor-beta": "2.0.0-beta.52" } }, + "2.0.0-beta.51", + ); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain(`${tap}'s executor-beta formula names 2.0.0-beta.52`); + }), + ); + + it.effect( + "a newer release half published, its image moved and npm not, stops the first step", + () => + Effect.gen(function* () { + const result = yield* preflight( + { ...afterBeta51, images: { beta: "2.0.0-beta.52" } }, + "2.0.0-beta.51", + ); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain(`${image}:beta names 2.0.0-beta.52: newer than 2.0.0-beta.51`); + }), + ); + + it.effect("an image, formula or feed it cannot read or order is a warning, and it goes on", () => + Effect.gen(function* () { + const result = yield* preflight( + { + ...afterBeta51, + failing: ["ghcr", "feed"], + formulas: { "executor-beta": "not-a-version" }, + }, + "2.0.0-beta.52", + ); + expect(result.exitCode, result.log).toBe(0); + expect(result.log).toContain("::warning title=Channel guard::Cannot read the image"); + expect(result.log).toContain("::warning title=Channel guard::Cannot read the desktop feed"); + expect(result.log).toContain( + `::warning title=Channel guard::${tap}'s executor-beta formula names not-a-version, which cannot be ordered`, + ); + expect(result.log).toContain( + "npm's executor@beta names 2.0.0-beta.51; 2.0.0-beta.52 may move it", + ); + }), + ); + + it.effect("npm it cannot read stops the first step", () => + Effect.gen(function* () { + const result = yield* preflight({ ...afterBeta51, failing: ["npm"] }, "2.0.0-beta.52"); + expect(result.exitCode, result.log).toBe(1); + // The first read, whether npm holds this version, already cannot be answered. + expect(result.log).toContain("Cannot read whether npm holds executor@2.0.0-beta.52"); + expect(result.log).toContain("Nothing was changed"); + }), + ); + + it.effect( + "a stable release passes the first step over Executor 1, and stops behind a newer one", + () => + Effect.gen(function* () { + const stub = yield* surfaces({ + npmTags: { latest: "1.6.11", beta: "2.0.0-beta.52" }, + images: { latest: "1.6.11", beta: "2.0.0-beta.52" }, + }); + const stable = { VERSION: "2.0.0", CHANNEL: "latest", TAP: tap }; + const passes = yield* run(stub.url, "channel-guard.ts", ["preflight"], stable); + expect(passes.exitCode, passes.log).toBe(0); + expect(passes.log).toContain("npm's executor@latest names 1.6.11; 2.0.0 may move it"); + expect(passes.log).toContain(`${tap}'s executor formula names nothing yet`); + const newer = yield* surfaces({ + npmTags: { latest: "1.6.11" }, + feed: { "executor-v2-latest-mac.yml": "2.0.1" }, + }); + const stops = yield* run(newer.url, "channel-guard.ts", ["preflight"], stable); + expect(stops.exitCode, stops.log).toBe(1); + expect(stops.log).toContain("executor-v2-latest-mac.yml names 2.0.1: newer than 2.0.0"); + }).pipe(Effect.scoped), + ); +}); + +/** A launcher archive on disk, and its integrity as npm records it. */ +const launcherFile = (contents: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "executor-launcher-" }); + const file = path.join(directory, "executor.tgz"); + yield* fs.writeFileString(file, contents); + return { file, integrity: `sha512-${createHash("sha512").update(contents).digest("base64")}` }; + }); + +/** What npm holds as 2.0.0 today: an archive published in error and deprecated. */ +const strayStable = { + integrity: "sha512-stray", + deprecated: "Published in error from a stale historical tag", + time: "2026-03-16T04:49:22.003Z", +}; + +layer(NodeServices.layer)( + "a version npm already holds is published only as its own resume", + (it) => { + it.effect( + "a version npm holds with no published release is refused first, and nothing else is read", + () => + Effect.gen(function* () { + const stub = yield* surfaces({ + npmTags: { latest: "1.6.11" }, + npmArchives: { "2.0.0": strayStable }, + images: { latest: "1.6.11" }, + }); + const result = yield* run(stub.url, "channel-guard.ts", ["preflight"], { + VERSION: "2.0.0", + CHANNEL: "latest", + TAP: tap, + }); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain( + 'npm already serves executor@2.0.0 (published 2026-03-16T04:49:22.003Z, deprecated: "Published in error from a stale historical tag"), and this is not a resumed publication of it', + ); + expect(result.log).toContain( + "this release needs another version number. Nothing was changed.", + ); + // Refused on npm's and GitHub's answers alone: no image, formula or feed was asked for. + expect( + stub.paths().filter((route) => !["npm", "api"].includes(surfaceOf(route))), + result.log, + ).toEqual([]); + }).pipe(Effect.scoped), + ); + + it.effect( + "a draft release does not make it a resume: npm gets the launcher only after the release is public", + () => + Effect.gen(function* () { + const result = yield* preflight( + { + ...afterBeta51, + npmArchives: { "2.0.0-beta.52": { integrity: "sha512-elsewhere" } }, + release: { draft: true, assets: 3 }, + }, + "2.0.0-beta.52", + ); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain("has no published release executor@2.0.0-beta.52"); + }), + ); + + it.effect( + "a version npm holds with its published release is a resume, and the preflight goes on", + () => + Effect.gen(function* () { + const result = yield* preflight( + { + ...afterBeta51, + npmArchives: { "2.0.0-beta.51": { integrity: "sha512-this-build" } }, + release: { draft: false, assets: 21 }, + }, + "2.0.0-beta.51", + ); + expect(result.exitCode, result.log).toBe(0); + expect(result.log).toContain("a resumed publication"); + }), + ); + + it.effect( + "the launcher check passes npm holding nothing on one read, or this very launcher", + () => + Effect.gen(function* () { + const launcher = yield* launcherFile("this build's launcher"); + const absent = yield* surfaces({}); + const first = yield* run(absent.url, "channel-guard.ts", ["launcher", launcher.file], { + VERSION: "2.0.0-beta.52", + CHANNEL: "beta", + }); + expect(first.exitCode, first.log).toBe(0); + expect(first.log).toContain("npm does not hold executor@2.0.0-beta.52 yet"); + expect(absent.paths()).toEqual(["/executor/2.0.0-beta.52"]); + const resumed = yield* surfaces({ + npmArchives: { "2.0.0-beta.51": { integrity: launcher.integrity } }, + }); + const again = yield* run(resumed.url, "channel-guard.ts", ["launcher", launcher.file], { + VERSION: "2.0.0-beta.51", + CHANNEL: "beta", + }); + expect(again.exitCode, again.log).toBe(0); + expect(again.log).toContain("byte for byte: a resumed publication"); + }).pipe(Effect.scoped), + ); + + it.effect("the launcher check refuses npm holding the version with another archive", () => + Effect.gen(function* () { + const launcher = yield* launcherFile("this build's launcher"); + const stub = yield* surfaces({ npmArchives: { "2.0.0": strayStable } }); + const result = yield* run(stub.url, "channel-guard.ts", ["launcher", launcher.file], { + VERSION: "2.0.0", + CHANNEL: "latest", + }); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain( + `npm serves a different archive as executor@2.0.0 (sha512-stray, deprecated: "Published in error from a stale historical tag"), not the launcher this run built (${launcher.integrity})`, + ); + expect(result.log).toContain( + `${repository} has no published release executor@2.0.0 from this run's export commit`, + ); + expect(result.log).toContain("this release needs another version number"); + expect(result.log).toContain("Nothing public was changed"); + }).pipe(Effect.scoped), + ); + + it.effect( + "the launcher check says re-run the original run's failed jobs when this source published the release", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const launcher = yield* launcherFile("this attempt's rebuilt launcher"); + const exported = path.join(path.dirname(launcher.file), "public-commit"); + yield* fs.writeFileString(exported, "exportcommit51\n"); + const stub = yield* surfaces({ + npmArchives: { "2.0.0-beta.51": { integrity: "sha512-first-attempt" } }, + release: { draft: false, assets: 21, target: "exportcommit51" }, + }); + const result = yield* run(stub.url, "channel-guard.ts", ["launcher", launcher.file], { + VERSION: "2.0.0-beta.51", + CHANNEL: "beta", + PUBLIC_EXPORT_COMMIT_FILE: exported, + }); + expect(result.exitCode, result.log).toBe(1); + expect(result.log).toContain( + "The release executor@2.0.0-beta.51 is published and targets this run's export commit (exportcommit51), so the likely cause is a re-run of the whole workflow", + ); + expect(result.log).toContain("re-run the failed jobs of the original run instead"); + expect(result.log).not.toContain("needs another version number"); + }).pipe(Effect.scoped), + ); + + it.effect( + "with no launcher built, npm holding the version without a published release is the stray archive", + () => + Effect.gen(function* () { + // The preflight refused a stable 2.0.0: no launcher, npm's 2.0.0 is somebody else's. + const result = yield* report( + { npmTags: { latest: "1.6.11" }, npmArchives: { "2.0.0": strayStable } }, + { + VERSION: "2.0.0", + CHANNEL: "latest", + RELEASE_TAG: "executor@2.0.0", + LAUNCHER: "/nonexistent/executor-2.0.0.tgz", + }, + ); + expect(result.summary).toContain( + "| npm executor@2.0.0 | npm serves an archive this publication did not put there (no published release executor@2.0.0) | not done |", + ); + expect(result.summary).toContain("Re-running cannot finish 2.0.0"); + expect(result.summary).toContain("This release needs another version number."); + expect(result.summary).not.toContain("should finish"); + }), + ); + + it.effect( + "with no launcher built, a published release keeps npm's row 'not compared', and a version npm lacks stays absent", + () => + Effect.gen(function* () { + const resumed = yield* report(beta51, { LAUNCHER: "/nonexistent/executor.tgz" }); + expect(resumed.summary).toContain( + "| npm executor@2.0.0-beta.51 | published (not compared: this run built no launcher) | done |", + ); + expect(resumed.summary).toContain("should finish 2.0.0-beta.51"); + const fresh = yield* report( + { ...afterBeta51, release: { draft: true, assets: 3 } }, + { + VERSION: "2.0.0-beta.52", + RELEASE_TAG: "executor@2.0.0-beta.52", + LAUNCHER: "/nonexistent/executor.tgz", + }, + ); + expect(fresh.summary).toContain("| npm executor@2.0.0-beta.52 | absent | not done |"); + expect(fresh.summary).toContain("should finish 2.0.0-beta.52"); + }), + ); + + it.effect("the report calls a stray archive what it is, and says a re-run cannot finish", () => + Effect.gen(function* () { + const launcher = yield* launcherFile("this build's launcher"); + const stray = yield* report( + { ...beta51, npmArchives: { "2.0.0-beta.51": { integrity: "sha512-elsewhere" } } }, + { LAUNCHER: launcher.file }, + ); + expect(stray.summary).toContain( + "| npm executor@2.0.0-beta.51 | npm serves a different archive under this version | not done |", + ); + expect(stray.summary).toContain( + "Re-running cannot finish 2.0.0-beta.51: npm already serves executor@2.0.0-beta.51 with an archive this publication did not build", + ); + const same = yield* report( + { ...beta51, npmArchives: { "2.0.0-beta.51": { integrity: launcher.integrity } } }, + { LAUNCHER: launcher.file }, + ); + expect(same.summary).toContain( + "| npm executor@2.0.0-beta.51 | published, this run's launcher | done |", + ); + }).pipe(Effect.scoped), + ); + }, +); + +/** What beta.51 left public: everything but the formula and the desktop feed. */ +const beta51: State = { + release: { draft: false, assets: 21 }, + images: { "2.0.0-beta.51": "2.0.0-beta.51", beta: "2.0.0-beta.51" }, + npmVersions: ["2.0.0-beta.51"], + npmTags: { latest: "1.6.11", beta: "2.0.0-beta.51" }, + formulas: { "executor-beta": "2.0.0-beta.50" }, + feed: betaFeed("2.0.0-beta.50"), +}; + +const report = (state: State, extra: Readonly> = {}) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "executor-report-" }); + const summary = path.join(directory, "summary.md"); + const stub = yield* surfaces(state); + const result = yield* run(stub.url, "publication-report.ts", [], { + VERSION: "2.0.0-beta.51", + CHANNEL: "beta", + TAP: tap, + GH_TOKEN: "synthetic-token", + GITHUB_STEP_SUMMARY: summary, + ...extra, + }); + const written = (yield* fs.exists(summary)) ? yield* fs.readFileString(summary) : ""; + return { ...result, summary: written }; + }).pipe(Effect.scoped); + +layer(NodeServices.layer)("a failed publish job says what it published", (it) => { + it.effect( + "beta.51's state: published everywhere but Homebrew and the feed, and a re-run finishes it", + () => + Effect.gen(function* () { + const result = yield* report(beta51); + expect(result.exitCode, result.log).toBe(0); + expect(result.summary).toContain( + "| GitHub release executor@2.0.0-beta.51 | public, 21 assets | done |", + ); + expect(result.summary).toContain(`| ${image}:beta | 2.0.0-beta.51 | done |`); + expect(result.summary).toContain("| npm tag beta | 2.0.0-beta.51 | done |"); + expect(result.summary).toContain("| npm tag latest (a beta leaves it) | 1.6.11 | done |"); + expect(result.summary).toContain("| Homebrew executor-beta | 2.0.0-beta.50 | not done |"); + expect(result.summary).toContain( + "| desktop feed executor-v2-beta-mac.yml | 2.0.0-beta.50 | not done |", + ); + expect(result.summary).toContain( + "`gh run rerun 4242 --failed` should finish 2.0.0-beta.51", + ); + }), + ); + + it.effect("a channel a newer release moved says a re-run would not finish it", () => + Effect.gen(function* () { + const result = yield* report({ + ...beta51, + images: { ...beta51.images, beta: "2.0.0-beta.52" }, + }); + expect(result.exitCode, result.log).toBe(0); + expect(result.summary).toContain("Re-running would not finish 2.0.0-beta.51"); + expect(result.summary).toContain(`${image}:beta names 2.0.0-beta.52`); + }), + ); + + it.effect("a draft release is not done, and a release that finished says nothing is left", () => + Effect.gen(function* () { + const draft = yield* report({ ...beta51, release: { draft: true, assets: 3 } }); + expect(draft.summary).toContain("| draft, 3 assets | not done |"); + const finished = yield* report({ + ...beta51, + formulas: { "executor-beta": "2.0.0-beta.51" }, + feed: betaFeed("2.0.0-beta.51"), + }); + expect(finished.summary).toContain("Nothing is left to finish"); + }), + ); + + it.effect( + "a stable release lists the beta feed files it also moves, and is not done while they lag", + () => + Effect.gen(function* () { + const stable = (betaFeed2: Readonly>): State => ({ + release: { draft: false, assets: 21 }, + images: { "2.0.0": "2.0.0", latest: "2.0.0" }, + npmVersions: ["2.0.0"], + npmTags: { latest: "2.0.0", beta: "2.0.0-beta.60" }, + formulas: { executor: "2.0.0" }, + feed: { + "executor-v2-latest-mac.yml": "2.0.0", + "executor-v2-latest.yml": "2.0.0", + "executor-v2-latest-linux.yml": "2.0.0", + "executor-v2-latest-linux-arm64.yml": "2.0.0", + ...betaFeed2, + }, + }); + const lagging = yield* report(stable(betaFeed("2.0.0-beta.60")), { + VERSION: "2.0.0", + CHANNEL: "latest", + RELEASE_TAG: "executor@2.0.0", + }); + expect(lagging.exitCode, lagging.log).toBe(0); + expect(lagging.summary).toContain( + "| desktop feed executor-v2-beta-mac.yml (beta installs follow) | 2.0.0-beta.60 | not done |", + ); + expect(lagging.summary).not.toContain("Nothing is left to finish"); + const followed = yield* report(stable(betaFeed("2.0.0")), { + VERSION: "2.0.0", + CHANNEL: "latest", + RELEASE_TAG: "executor@2.0.0", + }); + expect(followed.summary).toContain("Nothing is left to finish"); + // A newer beta kept there is done too, and tells of no newer release. + const kept = yield* report(stable(betaFeed("2.0.1-beta.1")), { + VERSION: "2.0.0", + CHANNEL: "latest", + RELEASE_TAG: "executor@2.0.0", + }); + expect(kept.summary).toContain("Nothing is left to finish"); + }), + ); + + it.effect( + "surfaces that never answer are rows within their group's limit, and the table is written", + () => + Effect.gen(function* () { + const started = Date.now(); + const result = yield* report( + { ...beta51, hanging: ["ghcr", "feed"] }, + { EXECUTOR_PUBLIC_REPORT_GROUP_SECONDS: "1" }, + ); + expect(result.exitCode, result.log).toBe(0); + expect(result.summary).toContain( + `| ${image}:beta | could not read: no answer within 1 s | ? |`, + ); + expect(result.summary).toContain( + "| desktop feed (beta) | could not read: no answer within 1 s | ? |", + ); + expect(result.summary).toContain("| npm tag beta | 2.0.0-beta.51 | done |"); + expect(result.summary).toContain("Some surfaces could not be read"); + expect(Date.now() - started).toBeLessThan(30_000); + }), + ); + + it.effect("surfaces it cannot read are rows that say so, and it still exits 0", () => + Effect.gen(function* () { + const result = yield* report({ broken: true }); + expect(result.exitCode, result.log).toBe(0); + expect(result.summary).toContain("could not read"); + expect(result.summary).toContain("Some surfaces could not be read"); + }), + ); + + it.effect("missing inputs are printed, never a failure", () => + Effect.gen(function* () { + const result = yield* report(beta51, { IMAGE: "", RELEASE_TAG: "" }); + expect(result.exitCode, result.log).toBe(0); + }), + ); +}); + +/** One step of the `publish` job: its fields at step level and its run block's lines. */ +type Step = { + readonly lines: ReadonlyArray; + readonly name: string; + readonly fields: ReadonlyMap; + /** The run block's lines, as written; for `run: ` the one command at `indent`. */ + readonly run: ReadonlyArray; + /** The indentation of the run block's own lines: a line indented further is inside something. */ + readonly indent: number; +}; + +/** The `publish` job of release-artifacts.yml: its steps in order. */ +const publishSteps = (workflow: string): ReadonlyArray => { + const lines = workflow.split("\n"); + const start = lines.indexOf(" publish:"); + if (start === -1) return []; + const end = lines.findIndex((line, index) => index > start && /^ {2}[a-z0-9-]+:$/u.test(line)); + const blocks: Array> = []; + for (const line of lines.slice(start + 1, end === -1 ? undefined : end)) { + if (/^ {6}- /u.test(line)) blocks.push([line]); + else if (blocks.length > 0 && (line.trim() === "" || /^ {8}/u.test(line))) + blocks.at(-1)!.push(line); + } + return blocks.map((block) => { + const fields = new Map(); + let run: Array = []; + let indent = 10; + block.forEach((line, index) => { + const field = /^ {6}(?:- | {2})([a-z-]+):(?: (.*))?$/u.exec(line); + if (field === null) return; + fields.set(field[1]!, field[2]?.trim() ?? ""); + if (field[1] !== "run") return; + if (field[2] !== undefined && field[2].trim() !== "|") { + run = [`${" ".repeat(10)}${field[2].trim()}`]; + return; + } + const body: Array = []; + for (const next of block.slice(index + 1)) { + if (next.trim() !== "" && !/^ {10}/u.test(next)) break; + body.push(next); + } + run = body; + indent = Math.min( + ...body + .filter((next) => next.trim() !== "") + .map((next) => next.length - next.trimStart().length), + ); + }); + return { + lines: block, + name: fields.get("name") ?? fields.get("uses") ?? "(unnamed)", + fields, + run, + indent, + }; + }); +}; + +/** + * The commands that move a channel, and the check the step that runs one must run first. A list + * cannot catch a moving command nobody thought of: a new way to move a channel must be added here + * with its check, and to the step that runs it. + */ +const movers: ReadonlyArray<{ readonly move: RegExp; readonly check: string }> = [ + { move: /\bimagetools create\b|\bcrane\b|\bskopeo\b|\boras\b/u, check: "channel-guard.ts image" }, + { move: /\bnpm publish\b|\bnpm dist-tag\b|publish-npm\.ts/u, check: "channel-guard.ts npm" }, + { move: /\bgit push\b/u, check: 'channel-guard.ts homebrew "$RUNNER_TEMP/tap"' }, + { + move: /\bgh release upload "\$FEED"|executor-v2-desktop-updates/u, + check: "channel-guard.ts desktop-feed", + }, +]; +/** Anything that changes something public: the release, the image, npm, the tap or the feed. */ +const changesPublic = (line: string) => + /\bgh release (create|upload|edit)\b/u.test(line) || movers.some(({ move }) => move.test(line)); +/** The only condition a moving step may carry: no tap skips the formula's check and move alike. */ +const allowedConditions: ReadonlyMap = new Map([ + ['channel-guard.ts homebrew "$RUNNER_TEMP/tap"', "vars.HOMEBREW_TAP != ''"], +]); + +/** A run line with its comment, if any, removed. */ +const command = (line: string) => line.replace(/(^|\s)#.*$/u, "").trimEnd(); + +/** + * Whether `step` runs `node scripts/releases/` as a whole line of its own, at its run block's + * own indentation (not inside an `if`, a loop or a group), with nothing after it (no `|| true`) + * and no `set +e` before it, before line `before` of its run block. + */ +const runsCheck = (step: Step, check: string, before = step.run.length) => { + const exact = `node scripts/releases/${check}`; + const at = step.run.findIndex( + (line, index) => + index < before && + line.trim() === exact && + line.length - line.trimStart().length === step.indent, + ); + return at !== -1 && !step.run.slice(0, at).some((line) => /\bset \+e\b/u.test(command(line))); +}; + +/** Why the publish job's wiring of the guard and the report is wrong; empty when it is right. */ +const wiring = (workflow: string): ReadonlyArray => { + const steps = publishSteps(workflow); + const problems: Array = []; + const unconditional = (step: Step, allowed?: string) => { + const condition = step.fields.get("if"); + if (condition !== undefined && condition !== allowed) + problems.push(`the step "${step.name}" has if: ${condition}; it must always run`); + if (step.fields.has("continue-on-error")) + problems.push(`the step "${step.name}" has continue-on-error; its failure must stop the job`); + }; + + const preflight = steps.findIndex((step) => runsCheck(step, "channel-guard.ts preflight")); + const firstChange = steps.findIndex((step) => + step.run.some((line) => changesPublic(command(line))), + ); + if (preflight === -1) + problems.push( + "no step runs `node scripts/releases/channel-guard.ts preflight` as a line of its own", + ); + else { + if (firstChange !== -1 && preflight > firstChange) + problems.push("the preflight runs after a step that changes something public"); + unconditional(steps[preflight]!); + } + + // npm never accepts a version twice: once the launcher exists, and before anything exposes it + // (the upload, the image, the release's publication), npm must not hold another archive. + const launcherCheck = + 'channel-guard.ts launcher ".local/releases/$VERSION/wrapper/executor-$VERSION.tgz"'; + const wrapper = steps.findIndex((step) => + step.run.some((line) => command(line).includes("release:wrapper")), + ); + const checksLauncher = steps.findIndex((step) => runsCheck(step, launcherCheck)); + const exposes = steps.findIndex((step) => + step.run.some((line) => + /\bgh release (upload|edit)\b|\bimagetools create\b/u.test(command(line)), + ), + ); + if (checksLauncher === -1) + problems.push( + `no step runs \`node scripts/releases/${launcherCheck}\` as a line of its own: add it after release:wrapper`, + ); + else { + if (wrapper === -1 || checksLauncher < wrapper) + problems.push("the launcher check runs before release:wrapper builds the launcher"); + if (exposes !== -1 && checksLauncher > exposes) + problems.push( + "the launcher check runs after a step that uploads, moves the image or publishes the release", + ); + unconditional(steps[checksLauncher]!); + } + + for (const step of steps) + for (const { move, check } of movers) { + const moved = step.run.findIndex((line) => move.test(command(line))); + if (moved === -1) continue; + if (!runsCheck(step, check, moved)) + problems.push( + `the step "${step.name}" runs \`${command(step.run[moved]!).trim()}\` without checking its channel first: add \`node scripts/releases/${check}\` on a line of its own before it`, + ); + unconditional(step, allowedConditions.get(check)); + } + + const report = steps.find((step) => + step.run.some((line) => line.includes("publication-report.ts")), + ); + if (report === undefined) problems.push("no step runs the report"); + else { + if (report.fields.get("if") !== "failure()") + problems.push("the report does not run only after a failure"); + if (report.fields.get("continue-on-error") !== "true") + problems.push("the report can fail the job"); + const limit = report.fields.get("timeout-minutes"); + if (limit === undefined || Number(limit) > 5) + problems.push("the report has no short time limit"); + } + return problems; +}; + +layer(NodeServices.layer)( + "the publish job runs the guard and the report where they protect", + (it) => { + const workflow = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return yield* fs.readFileString(".github/workflows/release-artifacts.yml"); + }); + + it.effect("the workflow checks every channel before moving it and bounds the report", () => + Effect.gen(function* () { + expect(wiring(yield* workflow)).toEqual([]); + }), + ); + + it.effect( + "each change that would leave a channel or the job's result unguarded is caught", + () => + Effect.gen(function* () { + const text = yield* workflow; + const once = (pattern: RegExp, replacement: string) => { + const changed = text.replace(pattern, replacement); + expect(changed, String(pattern)).not.toBe(text); + return changed; + }; + const preflightLine = + /^( {8}run: node scripts\/releases\/channel-guard\.ts preflight)$/mu; + const preflightName = /^( {6}- name: Refuse to publish over a newer release)$/mu; + const afterPreflight = (step: string) => once(preflightLine, `$1\n${step}`); + const cases: ReadonlyArray = [ + // Deleting a check, a step or a field. + [ + "the image's own check deleted", + once(/^ +node scripts\/releases\/channel-guard\.ts image\n/mu, ""), + "imagetools create", + ], + [ + "every step's own check deleted", + text.replace( + /^ +node scripts\/releases\/channel-guard\.ts (image|npm|homebrew [^\n]*|desktop-feed)\n/gmu, + "", + ), + "without checking its channel first", + ], + [ + "the preflight step deleted", + once( + /^ {6}# Before anything moves:[\s\S]*?run: node scripts\/releases\/channel-guard\.ts preflight\n/mu, + "", + ), + "no step runs `node scripts/releases/channel-guard.ts preflight`", + ], + [ + "the report's continue-on-error deleted", + once( + /(name: Report what was and was not published\n(?: {8}.*\n)*?) {8}continue-on-error: true\n/u, + "$1", + ), + "the report can fail the job", + ], + [ + "the report's time limit deleted", + once( + /(name: Report what was and was not published\n(?: {8}.*\n)*?) {8}timeout-minutes: \d+\n/u, + "$1", + ), + "the report has no short time limit", + ], + // Disarming a check while its line stays. + [ + "the npm check commented out", + once(/^( +)(node scripts\/releases\/channel-guard\.ts npm)$/mu, "$1# $2"), + "add `node scripts/releases/channel-guard.ts npm`", + ], + [ + "the feed check made to pass with || true", + once(/^( +node scripts\/releases\/channel-guard\.ts desktop-feed)$/mu, "$1 || true"), + "add `node scripts/releases/channel-guard.ts desktop-feed`", + ], + [ + "the Homebrew check inside if false", + once( + /^( +)(node scripts\/releases\/channel-guard\.ts homebrew "\$RUNNER_TEMP\/tap")$/mu, + "$1if false; then\n$1 $2\n$1fi", + ), + 'add `node scripts/releases/channel-guard.ts homebrew "$RUNNER_TEMP/tap"`', + ], + [ + "the preflight made to pass with || true", + once(preflightLine, "$1 || true"), + "no step runs `node scripts/releases/channel-guard.ts preflight`", + ], + [ + "the preflight replaced by an echo", + once(preflightLine, " run: echo channel-guard.ts preflight"), + "no step runs `node scripts/releases/channel-guard.ts preflight`", + ], + [ + "the preflight given if: false", + once(preflightName, "$1\n if: false"), + 'the step "Refuse to publish over a newer release" has if: false', + ], + [ + "the preflight given continue-on-error", + once(preflightName, "$1\n continue-on-error: true"), + 'the step "Refuse to publish over a newer release" has continue-on-error', + ], + // The launcher check deleted, moved after the upload, or allowed to fail. + [ + "the launcher check deleted", + once( + /^ {6}# npm never accepts a version twice[\s\S]*?channel-guard\.ts launcher [^\n]*\n/mu, + "", + ), + "no step runs `node scripts/releases/channel-guard.ts launcher", + ], + [ + "the launcher check moved after the upload", + (() => { + const step = + /^ {6}- name: Refuse a version npm holds with another archive\n(?: {8}.*\n)+/mu; + const moved = text.match(step)?.[0]; + expect(moved).toBeDefined(); + return text + .replace(step, "") + .replace( + /^( {6}- name: Publish the tested Docker manifest and channel)$/mu, + `${moved}$1`, + ); + })(), + "the launcher check runs after a step that uploads", + ], + [ + "the launcher check given continue-on-error", + once( + /^( {6}- name: Refuse a version npm holds with another archive)$/mu, + "$1\n continue-on-error: true", + ), + 'the step "Refuse a version npm holds with another archive" has continue-on-error', + ], + // A new step that moves a channel without its check. + [ + "a new step moving npm's tag with npm dist-tag", + afterPreflight( + ' - name: Tag it\n run: npm dist-tag add "executor@$VERSION" beta', + ), + 'the step "Tag it" runs `npm dist-tag add "executor@$VERSION" beta` without checking its channel first', + ], + [ + "a new step moving the image's channel with a second imagetools create", + afterPreflight( + ' - name: Retag\n run: docker buildx imagetools create --tag "$IMAGE:$CHANNEL" "$IMAGE:$VERSION"', + ), + 'the step "Retag" runs', + ], + [ + "a new step moving the image's channel with crane", + afterPreflight( + ' - name: Crane\n run: crane tag "$IMAGE:$VERSION" "$CHANNEL"', + ), + 'the step "Crane" runs `crane tag', + ], + ]; + for (const [change, changed, problem] of cases) + expect(wiring(changed).join("\n"), change).toContain(problem); + }), + ); + }, +); diff --git a/e2e/tests/run-retention.spec.ts b/e2e/tests/run-retention.spec.ts index 95e95ca20e..27f6c4f5cd 100644 --- a/e2e/tests/run-retention.spec.ts +++ b/e2e/tests/run-retention.spec.ts @@ -6,10 +6,10 @@ */ import { expect, layer } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { Effect, FileSystem, Path, Schedule, Stream } from "effect"; +import { Effect, FileSystem, Option, Path, Schedule, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { randomBytes } from "node:crypto"; -import { pruneRuns } from "../sdk/run-retention.ts"; +import { keepAllMarker, pruneRuns, shrunkMarker } from "../sdk/run-retention.ts"; const hour = 60 * 60_000; const day = 24 * hour; @@ -120,6 +120,14 @@ const writeRun = ( } for (const [file, text] of Object.entries(files)) yield* fs.writeFileString(path.join(directory, file), text); + // The run ended a minute after its name says it started. + const [, day, hours, minutes, seconds] = + /^(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2}\.\d{3})Z/.exec(name) ?? []; + const started = Date.parse(`${day}T${hours}:${minutes}:${seconds}Z`); + if (Number.isFinite(started) && outcome !== "none" && outcome !== "unreadable") { + const ended = new Date(Math.min(started + 60_000, Date.now())); + yield* fs.utimes(path.join(directory, "evidence.json"), ended, ended); + } return name; }); @@ -424,3 +432,211 @@ layer(NodeServices.layer, { excludeTestServices: true })("e2e run folder retenti }).pipe(Effect.scoped), ); }); + +/** A run folder's bulk and the files that must outlive a shrink. */ +const bulky = { + "local/coverage/0123456789abcdef/lcov.info": "TN:\n", + "local/data/diagnostics/motel.sqlite": "db", + "local/scenarios/0123456789abcdef0123456789abcdef/data/executor.pglite/base": "db", + "local/scenarios/0123456789abcdef0123456789abcdef/engines-abc/data/state": "db", + "cloud/data/diagnostics/motel-1/motel/data.sqlite": "db", +}; +const lasting = { + "local/scenarios/0123456789abcdef0123456789abcdef/server.log": "log\n", + "local/scenarios/0123456789abcdef0123456789abcdef/scenario.json": "{}", + "local/scenarios/0123456789abcdef0123456789abcdef/engines-abc/server.log": "log\n", + "local/report/evidence/abc/result.json": "{}", + "local/report/evidence/abc/trace.zip": "zip", + "local/report/evidence/abc/raw.webm": "webm", + "local/report/evidence/abc/checkpoint-1.png": "png", + "cloud/cloud.log": "log\n", +}; + +/** A finished run, `ago` old by its name and by when its `evidence.json` was written. */ +const shrinkable = (runs: string, ago: number, outcome: "passed" | "failed", suffix: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const name = runName(ago, "local", suffix); + const directory = path.join(runs, name); + for (const [file, text] of Object.entries({ ...bulky, ...lasting })) { + yield* fs.makeDirectory(path.dirname(path.join(directory, file)), { recursive: true }); + yield* fs.writeFileString(path.join(directory, file), text); + } + yield* writeRun(runs, name, outcome); + const ended = new Date(Date.now() - ago + 60_000); + yield* fs.utimes(path.join(directory, "evidence.json"), ended, ended); + return name; + }); + +/** Which of the bulk and lasting files a run folder still holds. */ +const holds = (runs: string, name: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const has = (file: string) => fs.exists(path.join(runs, name, file)); + const bulk: Array = []; + for (const file of Object.keys(bulky)) if (yield* has(file)) bulk.push(file); + const kept: Array = []; + for (const file of Object.keys(lasting)) if (yield* has(file)) kept.push(file); + return { + bulk, + lasting: kept.length === Object.keys(lasting).length, + marker: yield* has(shrunkMarker), + }; + }); +const whole = { bulk: Object.keys(bulky), lasting: true, marker: false }; +const shrunk = { bulk: [], lasting: true, marker: true }; + +layer(NodeServices.layer, { excludeTestServices: true })("e2e run folder shrinking", (it) => { + it.effect( + "the newest passed run stays whole, older passed runs and day-old failed runs lose only their bulk and say so", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { repo, runs } = yield* workspace; + yield* fs.makeDirectory(runs, { recursive: true }); + const newestPassed = yield* shrinkable(runs, 2 * hour, "passed", "a00001"); + const olderPassed = yield* shrinkable(runs, 3 * hour, "passed", "a00002"); + const freshFailed = yield* shrinkable(runs, 23 * hour, "failed", "a00003"); + const dayOldFailed = yield* shrinkable(runs, 26 * hour, "failed", "a00004"); + const start = Date.now(); + yield* pruneRuns({ workspace: repo, current: "" }); + expect(yield* holds(runs, newestPassed)).toEqual(whole); + expect(yield* holds(runs, freshFailed)).toEqual(whole); + expect(yield* holds(runs, olderPassed)).toEqual(shrunk); + expect(yield* holds(runs, dayOldFailed)).toEqual(shrunk); + const record = JSON.parse( + yield* fs.readFileString(path.join(runs, dayOldFailed, shrunkMarker)), + ) as { shrunkAt: string; reason: string; removed: Array }; + expect(record.removed.sort()).toEqual([ + "cloud/data", + "local/coverage", + "local/data", + "local/scenarios/0123456789abcdef0123456789abcdef/data", + "local/scenarios/0123456789abcdef0123456789abcdef/engines-abc/data", + ]); + expect(record.reason).toMatch(/24 hours/); + expect(Date.parse(record.shrunkAt)).toBeGreaterThanOrEqual(start - 1000); + // Shrunk once: a second start leaves its record as it was. + const before = yield* fs.readFileString(path.join(runs, olderPassed, shrunkMarker)); + yield* pruneRuns({ workspace: repo, current: "" }); + expect(yield* fs.readFileString(path.join(runs, olderPassed, shrunkMarker))).toBe(before); + }).pipe(Effect.scoped), + ); + + it.effect("keep-all-runs stops removal and shrinking", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { repo, runs } = yield* workspace; + yield* fs.makeDirectory(runs, { recursive: true }); + yield* shrinkable(runs, 2 * hour, "passed", "b00001"); + const older = yield* shrinkable(runs, 3 * hour, "passed", "b00002"); + const aged = yield* shrinkable(runs, 5 * day, "failed", "b00003"); + yield* fs.writeFileString(path.join(runs, keepAllMarker), ""); + yield* pruneRuns({ workspace: repo, current: "" }); + expect(yield* holds(runs, older)).toEqual(whole); + expect(yield* holds(runs, aged)).toEqual(whole); + }).pipe(Effect.scoped), + ); + + it.effect( + "a run in progress is not shrunk, and a bulk folder that is a link is not followed", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { repo, runs, outside } = yield* workspace; + yield* fs.makeDirectory(runs, { recursive: true }); + yield* shrinkable(runs, 1 * hour, "passed", "c00001"); + const writing = yield* shrinkable(runs, 30 * hour, "failed", "c00002"); + yield* fs.writeFileString( + path.join(runs, writing, `in-progress-${process.pid}-abcdef.json`), + JSON.stringify({ pid: process.pid }), + ); + const linked = yield* shrinkable(runs, 3 * hour, "passed", "c00003"); + // A scenario whose `data` is a link to a directory outside the workspace. + yield* fs.makeDirectory(path.join(outside, "data")); + yield* fs.writeFileString(path.join(outside, "data/keep.txt"), "outside\n"); + const scenario = path.join( + runs, + linked, + "local/scenarios/fedcba9876543210fedcba9876543210", + ); + yield* fs.makeDirectory(scenario, { recursive: true }); + yield* fs.symlink(path.join(outside, "data"), path.join(scenario, "data")); + yield* pruneRuns({ workspace: repo, current: "" }); + expect(yield* holds(runs, writing)).toEqual(whole); + expect(yield* holds(runs, linked)).toEqual(shrunk); + expect(yield* fs.exists(path.join(outside, "data/keep.txt"))).toBe(true); + // The link itself stays too: only real directories are bulk. + expect(Option.isSome(yield* Effect.option(fs.readLink(path.join(scenario, "data"))))).toBe( + true, + ); + }).pipe(Effect.scoped), + ); + + /** Seven passed runs, one scenario each, two minutes apart, the newest ending `newestAgo` ago. */ + const series = (runs: string, newestAgo: number) => + Effect.forEach([6, 5, 4, 3, 2, 1, 0], (step) => + shrinkable(runs, newestAgo + step * 2 * 60_000 + 60_000, "passed", `5e0${step}00`), + ); + const evidence = "local/report/evidence/abc/lookups.json"; + + it.effect( + "a series of quick passed runs read together keeps every run whole while it is recent", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { repo, runs } = yield* workspace; + yield* fs.makeDirectory(runs, { recursive: true }); + const names = yield* series(runs, 15 * 60_000); + for (const name of names) yield* fs.writeFileString(path.join(runs, name, evidence), "{}"); + // An eighth run in the series starts. + yield* pruneRuns({ workspace: repo, current: "" }); + for (const name of names) { + expect(yield* holds(runs, name), name).toEqual(whole); + expect(yield* fs.exists(path.join(runs, name, evidence)), name).toBe(true); + } + }).pipe(Effect.scoped), + ); + + it.effect( + "once the series is two hours old, the count takes passed runs past the newest five and the rest shrink", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const { repo, runs } = yield* workspace; + yield* fs.makeDirectory(runs, { recursive: true }); + const names = yield* series(runs, 3 * hour); + yield* pruneRuns({ workspace: repo, current: "" }); + const left = (yield* fs.readDirectory(runs)).sort(); + // `names` runs oldest first. + expect(left).toEqual(names.slice(2).sort()); + expect(yield* holds(runs, names[6]!)).toEqual(whole); + for (const name of names.slice(2, 6)) + expect(yield* holds(runs, name), name).toEqual(shrunk); + }).pipe(Effect.scoped), + ); + + it.effect("past the byte cap even recent runs go, oldest first, to protect the disk", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { repo, runs } = yield* workspace; + yield* fs.makeDirectory(runs, { recursive: true }); + const names = yield* Effect.forEach([4, 3, 2, 1], (step) => + writeRun(runs, runName(step * 5 * 60_000, "local", `ca0${step}00`), "passed", { + "big.bin": "x".repeat(2 * mebibyte), + }), + ); + // Four recent runs of about 2 MiB against a 5 MiB cap: the two oldest go. + yield* pruneRuns({ workspace: repo, current: "", keepBytes: 5 * mebibyte }); + expect((yield* fs.readDirectory(runs)).sort()).toEqual(names.slice(2).sort()); + expect(yield* fs.exists(path.join(runs, names[3]!, "big.bin"))).toBe(true); + }).pipe(Effect.scoped), + ); +}); diff --git a/e2e/tests/runtime-connect-repeat.spec.ts b/e2e/tests/runtime-connect-repeat.spec.ts new file mode 100644 index 0000000000..2f3e4e580e --- /dev/null +++ b/e2e/tests/runtime-connect-repeat.spec.ts @@ -0,0 +1,441 @@ +/** + * An exchange with the local app runtime whose connection never formed sent nothing, so the host + * repeats it once, at once, and the span records the repeat and how it ended. On Windows a loopback + * connect that is not answered in about a quarter of a second fails that way while the runtime is + * healthy. Anything that may have reached the runtime is never repeated: a session that opened, a + * request whose bytes were written. A call whose only exchange with the app never left the host + * says so and may be repeated; one that reached the app's code earlier in the same call may have + * written, and says that instead. + * + * Each case runs `packages/sdk/scripts/runtime-connect-fixture.ts` against a stub on 127.0.0.1, + * under Node and under Bun (the packaged CLI's runtime), composed as `workerd-apps.ts` composes its + * exchanges. `local-app-runtime-restart.spec.ts` checks the same repeat through the product itself. + */ +import { expect, layer } from "@effect/vitest"; +import { Clock, Duration, Effect, Schema, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { scenarios } from "../test-plan.ts"; +import { TestLive, withCase } from "../support/case.ts"; +import { Evidence } from "../support/evidence.ts"; + +const Attributes = Schema.Record( + Schema.String, + Schema.Union([Schema.String, Schema.Number, Schema.Boolean]), +); +const Probe = Schema.Struct({ + outcome: Schema.optional(Schema.Literals(["answered", "failed"])), + failure: Schema.optional(Schema.Unknown), + connections: Schema.optional(Schema.Number), + attempts: Schema.optional(Schema.Number), + invocations: Schema.optional(Schema.Number), + checks: Schema.optional(Schema.Array(Schema.Number)), + value: Schema.optional(Schema.String), + reason: Schema.optional(Schema.String), + decisions: Schema.optional(Schema.Record(Schema.String, Schema.Boolean)), + wire: Schema.optional( + Schema.Struct({ + values: Schema.Array( + Schema.Struct({ + label: Schema.String, + failure: Schema.optional(Schema.String), + known: Schema.Boolean, + notSent: Schema.Boolean, + }), + ), + read: Schema.Array(Schema.Struct({ label: Schema.String, decoded: Schema.Boolean })), + partlyRunDropsNotSent: Schema.Boolean, + }), + ), + notSent: Schema.optional(Schema.Boolean), + mayHaveWritten: Schema.optional(Schema.Boolean), + retryable: Schema.optional(Schema.NullOr(Schema.Boolean)), + action: Schema.optional(Schema.NullOr(Schema.String)), + attributes: Attributes, +}); + +/** + * Each probe's own bound, so a hung probe fails as itself and not as the scenario's time-out. + * Measured on CI (17 Nov 2026, CI run 44037352539, `e2e-self-host (1/1)` on + * blacksmith-16vcpu-ubuntu-2404 at 07:17Z): 30 probes, median 0.31 s, longest 1.28 s (the call-path + * probes); the scenario took 13.1 s. On minisforum the longest was 1.8 s. Whole self-host shards + * run on 8 vCPU, and weekday afternoons are up to about 1.5× slower (AS1's runner-speed report, + * 16 Nov), so 20 s is about 15× the longest measured probe. It is below the fixture's own 30 s, and + * a hung probe plus the rest stays well inside the scenario's time-out. + */ +const probeBound = Duration.seconds(20); +/** How much of a probe's output a time-out reports. */ +const lastOutputChars = 2_000; + +class ProbeTimedOut extends Schema.TaggedError()("ProbeTimedOut", { + probe: Schema.String, + ranMs: Schema.Number, + boundMs: Schema.Number, + lastOutput: Schema.String, +}) { + get message() { + const output = this.lastOutput.trim(); + return `Probe ${this.probe} ran for ${this.ranMs} ms without finishing (its bound is ${this.boundMs} ms); its last output: ${output === "" ? "(none)" : output}`; + } +} + +/** + * Run one probe process under its own bound and return its stdout. Its duration is recorded as a + * measurement; past the bound it is stopped and fails with its name, how long it ran and the end + * of its stdout and stderr. + */ +const runProbe = (label: string, command: string, args: ReadonlyArray) => + Effect.scoped( + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner, + evidence = yield* Evidence; + const started = yield* Clock.currentTimeMillis; + let stdout = ""; + let last = ""; + const keep = (text: string) => { + last = (last + text).slice(-lastOutputChars); + }; + const child = yield* processes.spawn( + ChildProcess.make(command, [...args], { + env: { PATH: process.env.PATH ?? "", HOME: process.env.HOME ?? "" }, + extendEnv: false, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }), + ); + yield* Effect.all( + [ + child.stdout.pipe( + Stream.decodeText(), + Stream.runForEach((text) => + Effect.sync(() => { + stdout += text; + keep(text); + }), + ), + ), + child.stderr.pipe( + Stream.decodeText(), + Stream.runForEach((text) => Effect.sync(() => keep(text))), + ), + ], + { concurrency: "unbounded", discard: true }, + ).pipe( + Effect.timeoutOrElse({ + duration: probeBound, + orElse: () => + Effect.gen(function* () { + const ranMs = (yield* Clock.currentTimeMillis) - started; + return yield* new ProbeTimedOut({ + probe: label, + ranMs, + boundMs: Duration.toMillis(probeBound), + lastOutput: last, + }); + }), + }), + ); + yield* evidence.measure(`Probe ${label}`, (yield* Clock.currentTimeMillis) - started); + return stdout; + }), + ); + +/** One case of the probe under `runtime` (`node` or `bun`). */ +const probe = (runtime: "node" | "bun", name: string) => + Effect.gen(function* () { + const evidence = yield* Evidence; + const output = yield* runProbe(`${runtime} ${name}`, runtime, [ + "packages/sdk/scripts/runtime-connect-fixture.ts", + "--case", + name, + ]); + const result = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Probe))( + output.trim().split("\n").at(-1) ?? "", + ); + yield* evidence.json(`runtime-connect-${runtime}-${name}.json`, result); + return result; + }); + +/** A case of the SDK call-path probe: a tool call through the real call path over a stub transport. */ +const CallPath = Schema.Struct({ + asked: Schema.optional(Schema.String), + failure: Schema.Struct({ + tag: Schema.String, + reason: Schema.String, + notSent: Schema.Boolean, + mayHaveWritten: Schema.Boolean, + }), + operations: Schema.Array(Schema.String), +}); +const callPath = (name: string) => + Effect.gen(function* () { + const evidence = yield* Evidence; + const output = yield* runProbe(`node call-path ${name}`, "node", [ + "packages/sdk/scripts/call-path-fixture.ts", + "--case", + name, + ]); + const result = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(CallPath))( + output.trim().split("\n").at(-1) ?? "", + ); + yield* evidence.json(`call-path-${name}.json`, result); + return result; + }); +const notSentSentence = + "Executor could not connect to the app runtime, so it did not send this tool call."; +const partlyRunSentence = + "Executor could not reach the app runtime to finish this tool call. An earlier step of the call reached the app, so the call may have partly run."; + +const rpc = "executor.runtime.rpc."; +const http = "executor.runtime.http."; + +layer(TestLive, { excludeTestServices: true })("Runtime connect repeat", (it) => { + it.effect(scenarios.runtimeConnectRepeat.title, (context) => + withCase( + context, + Effect.gen(function* () { + const evidence = yield* Evidence; + for (const runtime of ["node", "bun"] as const) { + yield* evidence.step( + `${runtime}: an RPC session whose first socket never opened is opened once more and answers`, + Effect.gen(function* () { + const rescued = yield* probe(runtime, "rpc-rescued"); + expect(rescued).toMatchObject({ outcome: "answered", connections: 2 }); + expect(rescued.attributes).toMatchObject({ + [`${rpc}connect_attempts`]: 2, + [`${rpc}first_attempt.error_code`]: "ECONNRESET", + [`${rpc}connect_outcome`]: "opened", + }); + expect(typeof rescued.attributes[`${rpc}first_attempt.failed_ms`]).toBe("number"); + // The plain attributes describe the attempt that decided: it opened and answered, so + // none are written; the first attempt's facts are under first_attempt. + expect( + Object.keys(rescued.attributes).filter( + (key) => !key.includes("connect_") && !key.includes("first_attempt."), + ), + ).toEqual([]); + }), + ); + + yield* evidence.step( + `${runtime}: a real Cap'n Web session whose first socket never opened runs its call once`, + Effect.gen(function* () { + const rescued = yield* probe(runtime, "rpc-capnweb-rescued"); + expect(rescued).toMatchObject({ + outcome: "answered", + value: "ran", + invocations: 1, + connections: 2, + }); + expect(rescued.attributes).toMatchObject({ + [`${rpc}connect_attempts`]: 2, + [`${rpc}connect_outcome`]: "opened", + [`${rpc}first_attempt.error_code`]: "ECONNRESET", + }); + expect(rescued.attributes).not.toHaveProperty(`${rpc}opened`); + expect(rescued.attributes).not.toHaveProperty(`${rpc}error_code`); + }), + ); + + yield* evidence.step( + `${runtime}: an RPC session whose socket never opens twice fails as unreachable, nothing sent`, + Effect.gen(function* () { + const unreachable = yield* probe(runtime, "rpc-unreachable"); + expect(unreachable).toMatchObject({ + outcome: "failed", + failure: { tag: "RuntimeProtocolFailed", reason: "disconnected", notSent: true }, + }); + expect(unreachable.attributes).toMatchObject({ + [`${rpc}connect_attempts`]: 2, + [`${rpc}first_attempt.error_code`]: "ECONNREFUSED", + [`${rpc}connect_outcome`]: "failed", + [`${rpc}opened`]: false, + [`${rpc}sent`]: 0, + "executor.runtime.failure": "unreachable", + }); + // A listener that closes every connection before its upgrade sees exactly two. + const neverOpens = yield* probe(runtime, "rpc-never-opens"); + expect(neverOpens).toMatchObject({ + outcome: "failed", + connections: 2, + failure: { tag: "RuntimeProtocolFailed", reason: "disconnected", notSent: true }, + }); + expect(neverOpens.attributes).toMatchObject({ + [`${rpc}connect_attempts`]: 2, + [`${rpc}first_attempt.error_code`]: "ECONNRESET", + [`${rpc}connect_outcome`]: "failed", + }); + }), + ); + + yield* evidence.step( + `${runtime}: an RPC session that opened is never repeated, whether or not it sent`, + Effect.gen(function* () { + for (const [name, sent] of [ + ["rpc-closed-after-open", 0], + ["rpc-sent-then-closed", 1], + ] as const) { + const result = yield* probe(runtime, name); + expect(result, name).toMatchObject({ + outcome: "failed", + connections: 1, + failure: { tag: "closed" }, + }); + expect(result.attributes, name).toMatchObject({ + [`${rpc}opened`]: true, + [`${rpc}sent`]: sent, + }); + expect( + Object.keys(result.attributes).filter((key) => key.includes("connect_")), + name, + ).toEqual([]); + } + }), + ); + + yield* evidence.step( + `${runtime}: an HTTP exchange refused at connect is run once more from its start`, + Effect.gen(function* () { + const rescued = yield* probe(runtime, "http-rescued"); + expect(rescued).toMatchObject({ outcome: "answered", attempts: 2, connections: 1 }); + expect(rescued.attributes).not.toHaveProperty(`${http}stage`); + expect(rescued.attributes).toMatchObject({ + [`${http}connect_attempts`]: 2, + [`${http}first_attempt.error_code`]: "ECONNREFUSED", + [`${http}connect_outcome`]: "answered", + }); + const unreachable = yield* probe(runtime, "http-unreachable"); + expect(unreachable).toMatchObject({ outcome: "failed", attempts: 2 }); + expect(unreachable.attributes).toMatchObject({ + [`${http}connect_attempts`]: 2, + [`${http}connect_outcome`]: "failed", + [`${http}stage`]: "request", + }); + }), + ); + + yield* evidence.step( + `${runtime}: an HTTP request whose bytes were written is never repeated`, + Effect.gen(function* () { + for (const name of ["http-reset-after-write", "http-closed-after-write"]) { + const result = yield* probe(runtime, name); + expect(result, name).toMatchObject({ + outcome: "failed", + attempts: 1, + connections: 1, + }); + expect( + Object.keys(result.attributes).filter((key) => key.includes("connect_")), + name, + ).toEqual([]); + } + }), + ); + } + + yield* evidence.step( + "A call whose only exchange with the app never left the host may be repeated; one that reached the app earlier may have written", + Effect.gen(function* () { + expect(yield* probe("node", "call-first-exchange")).toMatchObject({ + notSent: true, + mayHaveWritten: false, + retryable: true, + action: "Try the call again.", + reason: + "Executor could not connect to the app runtime, so it did not send this tool call.", + }); + // The catalog read for the call's kind reached the app's code first: it may have written. + expect(yield* probe("node", "call-second-exchange")).toMatchObject({ + notSent: false, + mayHaveWritten: true, + retryable: false, + reason: + "Executor could not reach the app runtime to finish this tool call. An earlier step of the call reached the app, so the call may have partly run.", + }); + // A call the app confirms only reads stays safe to repeat. + expect(yield* probe("node", "call-second-exchange-read")).toMatchObject({ + notSent: true, + mayHaveWritten: false, + }); + }), + ); + + yield* evidence.step( + "Through the SDK's call path, a call says it was not sent only when its never-opened exchange was its only hand-off to the app", + Effect.gen(function* () { + // The call names its kind: one hand-off, which never connected. + expect(yield* callPath("call-once")).toMatchObject({ + operations: ["call"], + failure: { + tag: "ToolCallFailed", + reason: notSentSentence, + notSent: true, + mayHaveWritten: false, + }, + }); + // The catalog read for its kind reached the app first. + expect(yield* callPath("call-after-kind-read")).toMatchObject({ + operations: ["inspect", "call"], + failure: { reason: partlyRunSentence, notSent: false, mayHaveWritten: true }, + }); + // The approval request reached the app: its setup and approval check ran. + expect(yield* callPath("call-resumed")).toMatchObject({ + asked: "approval-required", + operations: ["call", "call"], + failure: { reason: partlyRunSentence, notSent: false, mayHaveWritten: true }, + }); + }), + ); + + yield* evidence.step( + "A host's check runs before every connect, the repeat's included, and a session's own reason is kept with notSent", + Effect.gen(function* () { + expect(yield* probe("node", "rpc-check-before-each-connect")).toMatchObject({ + checks: [0, 1], + connections: 2, + }); + expect(yield* probe("node", "rpc-never-opens-own-reason")).toMatchObject({ + connections: 2, + failure: { tag: "RuntimeProtocolFailed", reason: "overloaded", notSent: true }, + }); + }), + ); + + yield* evidence.step( + "A never-opened exchange's failure carries only values the previous release reads, and this release reads that release's answers", + Effect.gen(function* () { + const { wire } = yield* probe("node", "wire-previous-release"); + expect(wire?.values).toHaveLength(3); + for (const value of wire?.values ?? []) { + // The previous release's closed list has the value; the fact is in `notSent`. + expect(value, value.label).toMatchObject({ + failure: "disconnected", + known: true, + notSent: true, + }); + } + expect(wire?.read.filter(({ decoded }) => !decoded)).toEqual([]); + expect(wire?.read).toHaveLength(6); + expect(wire?.partlyRunDropsNotSent).toBe(true); + }), + ); + + yield* evidence.step( + "An HTTP failure is repeated only when the platform's own fields prove the connect failed; Bun's ETIMEDOUT is not", + Effect.gen(function* () { + const { decisions } = yield* probe("node", "connect-decision-shapes"); + expect(decisions).toEqual({ + bunRefused: true, + bunTimedOut: false, + bunReset: false, + nodeConnectTimedOut: true, + nodeReadReset: false, + undiciConnectTimeout: true, + notTransport: false, + }); + }), + ); + }), + ), + ); +}); diff --git a/e2e/tests/runtime-hop-shape.spec.ts b/e2e/tests/runtime-hop-shape.spec.ts new file mode 100644 index 0000000000..1f54c6e2ac --- /dev/null +++ b/e2e/tests/runtime-hop-shape.spec.ts @@ -0,0 +1,266 @@ +/** + * A failed exchange between the local product and its app runtime (workerd) records how far it got, + * so a failure that passes on rerun can be told apart: a reply lost as its connection ended, a + * runtime that never answered, a request cut off while it was written, an RPC session that never + * opened or closed before or after it carried anything, a runtime just replaced. The recording can + * never change a failure: a value built to break it leaves the same error, classified the same, and + * the span says the recording failed. Only shapes are kept: a supervisor's context outside its own + * keys, or with a value that is not a number, a boolean or a short word, is dropped. + * + * Each case runs `packages/sdk/scripts/runtime-hop-fixture.ts` against a stub on 127.0.0.1, with + * the same recording the runtime's hops use. `local-app-runtime-restart.spec.ts` checks the same + * recording through the product itself, on a call that loses its runtime. + */ +import { expect, layer } from "@effect/vitest"; +import { Effect, Schema } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { scenarios } from "../test-plan.ts"; +import { TestLive, withCase } from "../support/case.ts"; +import { Evidence } from "../support/evidence.ts"; + +const Attributes = Schema.Record( + Schema.String, + Schema.Union([Schema.String, Schema.Number, Schema.Boolean]), +); +const Probe = Schema.Struct({ + outcome: Schema.optional(Schema.Literals(["answered", "failed"])), + first: Schema.optional(Schema.Literals(["answered", "failed"])), + sameErrorHttp: Schema.optional(Schema.Boolean), + sameErrorRpc: Schema.optional(Schema.Boolean), + classified: Schema.optional(Schema.Boolean), + attributes: Attributes, + rpcAttributes: Schema.optional(Attributes), + longAttributes: Schema.optional(Attributes), +}); + +/** One case of the probe: what the exchange recorded. */ +const probe = (name: string) => + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner, + evidence = yield* Evidence; + const output = yield* processes.string( + ChildProcess.make("node", ["packages/sdk/scripts/runtime-hop-fixture.ts", "--case", name], { + env: { PATH: process.env.PATH ?? "" }, + extendEnv: false, + }), + ); + const result = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Probe))( + output.trim().split("\n").at(-1) ?? "", + ); + yield* evidence.json(`runtime-hop-${name}.json`, result); + return result; + }); + +/** The keys a recording wrote under `prefix`. */ +const under = (attributes: Record, prefix: string) => + Object.keys(attributes).filter((key) => key.startsWith(prefix)); + +layer(TestLive, { excludeTestServices: true })("Runtime hop shape", (it) => { + it.effect(scenarios.runtimeHopShape.title, (context) => + withCase( + context, + Effect.gen(function* () { + const evidence = yield* Evidence; + + yield* evidence.step( + "An HTTP exchange records how far it got: no headers, headers then a reset, a reset while writing", + Effect.gen(function* () { + const unanswered = yield* probe("http-unanswered"); + expect(unanswered.outcome).toBe("failed"); + expect(unanswered.attributes).toMatchObject({ + "executor.runtime.http.route": "/workflow", + "executor.runtime.http.method": "POST", + "executor.runtime.http.stage": "request", + "executor.runtime.http.error": "HttpClientError", + "executor.runtime.http.reason": "TransportError", + "executor.runtime.http.code": "UND_ERR_SOCKET", + "executor.runtime.host.restarts": 0, + }); + expect(unanswered.attributes).not.toHaveProperty("executor.runtime.http.headers_ms"); + // The stub closed 120 ms after the request arrived. + expect(unanswered.attributes["executor.runtime.http.failed_ms"]).toBeGreaterThanOrEqual( + 100, + ); + + const reset = yield* probe("http-answer-reset"); + expect(reset.outcome).toBe("failed"); + expect(reset.attributes).toMatchObject({ + "executor.runtime.http.stage": "response-body", + "executor.runtime.http.status": 200, + "executor.runtime.http.reason": "DecodeError", + "executor.runtime.http.code": "ECONNRESET", + "executor.runtime.http.syscall": "read", + }); + expect(typeof reset.attributes["executor.runtime.http.headers_ms"]).toBe("number"); + expect(typeof reset.attributes["executor.runtime.http.errno"]).toBe("number"); + + const halfWritten = yield* probe("http-reset-half-written"); + expect(halfWritten.outcome).toBe("failed"); + expect(halfWritten.attributes).toMatchObject({ + "executor.runtime.http.stage": "request", + "executor.runtime.http.reason": "TransportError", + "executor.runtime.http.code": "ECONNRESET", + }); + expect(halfWritten.attributes).not.toHaveProperty("executor.runtime.http.headers_ms"); + }), + ); + + yield* evidence.step( + "A failure records how long the runtime's origin was idle and how recently the runtime was replaced", + Effect.gen(function* () { + const idle = yield* probe("http-idle"); + expect(idle.first).toBe("answered"); + expect(idle.attributes["executor.runtime.http.idle_ms"]).toBeGreaterThanOrEqual(140); + const restarted = yield* probe("http-restarted"); + expect(restarted.attributes).toMatchObject({ "executor.runtime.host.restarts": 2 }); + expect( + restarted.attributes["executor.runtime.host.restarted_ms"], + ).toBeGreaterThanOrEqual(75); + }), + ); + + yield* evidence.step( + "An exchange that succeeds records nothing", + Effect.gen(function* () { + // The stub answers before reading the request body; this host still reads the reply. + const { outcome, attributes } = yield* probe("http-unread-body"); + expect(outcome).toBe("answered"); + expect(attributes).toEqual({}); + }), + ); + + yield* evidence.step( + "A value built to break the recording leaves the same error, classified the same", + Effect.gen(function* () { + for (const name of ["guard-throwing-getter", "guard-proxy"]) { + const result = yield* probe(name); + expect(result, name).toMatchObject({ + sameErrorHttp: true, + sameErrorRpc: true, + classified: true, + }); + expect(result.attributes, name).toEqual({ "executor.runtime.recording": "failed" }); + } + const context = yield* probe("guard-throwing-context"); + expect(context).toMatchObject({ + sameErrorHttp: true, + sameErrorRpc: true, + classified: true, + }); + expect(context.attributes).toMatchObject({ + "executor.runtime.probe.recording": "failed", + "executor.runtime.http.stage": "request", + }); + const socket = yield* probe("guard-throwing-socket"); + expect(socket).toMatchObject({ + sameErrorHttp: true, + sameErrorRpc: true, + classified: true, + }); + expect(socket.rpcAttributes).toEqual({ "executor.runtime.recording": "failed" }); + }), + ); + + yield* evidence.step( + "A supervisor's context keeps only its own keys with short values, and an odd error tag is other", + Effect.gen(function* () { + const { attributes } = yield* probe("context-checked"); + expect(under(attributes, "executor.runtime.probe.").sort()).toEqual([ + "executor.runtime.probe.in_flight", + "executor.runtime.probe.outcome", + "executor.runtime.probe.sequence", + ]); + expect(attributes).toMatchObject({ + "executor.runtime.probe.sequence": 7, + "executor.runtime.probe.outcome": "timeout", + "executor.runtime.probe.in_flight": true, + "executor.runtime.http.route": "/workflow", + "executor.runtime.http.error": "other", + "executor.runtime.http.code": "other", + "executor.runtime.http.syscall": "other", + }); + expect( + Object.keys(attributes).some((key) => !key.startsWith("executor.runtime.")), + ).toBe(false); + expect(attributes).not.toHaveProperty("executor.runtime.forged"); + expect(attributes).not.toHaveProperty("executor.runtime.http.forged"); + }), + ); + + yield* evidence.step( + "A platform code as long as Node's and undici's longer ones is recorded whole; a longer one is other", + Effect.gen(function* () { + const { attributes, rpcAttributes, longAttributes } = yield* probe("code-long"); + expect(longAttributes?.["executor.runtime.http.code"]).toBe("other"); + expect(attributes["executor.runtime.http.code"]).toBe("ERR_STREAM_PREMATURE_CLOSE"); + expect(rpcAttributes?.["executor.runtime.http.code"]).toBe( + "UND_ERR_RESPONSE_STATUS_CODE", + ); + }), + ); + + yield* evidence.step( + "An RPC socket that never opens: refused, and accepted then closed before the upgrade", + Effect.gen(function* () { + const refused = yield* probe("ws-refused"); + expect(refused.attributes).toMatchObject({ + "executor.runtime.rpc.opened": false, + "executor.runtime.rpc.close_code": 1006, + "executor.runtime.rpc.error_code": "ECONNREFUSED", + "executor.runtime.rpc.sent": 0, + "executor.runtime.rpc.received": 0, + }); + expect(refused.attributes).not.toHaveProperty("executor.runtime.rpc.open_ms"); + const beforeUpgrade = yield* probe("ws-closed-before-upgrade"); + expect(beforeUpgrade.attributes).toMatchObject({ + "executor.runtime.rpc.opened": false, + "executor.runtime.rpc.close_code": 1006, + "executor.runtime.rpc.error_code": "ECONNRESET", + }); + }), + ); + + yield* evidence.step( + "An RPC socket closed with a code before any message, and one closed after a message each way", + Effect.gen(function* () { + const before = yield* probe("ws-close-before-message"); + expect(before.attributes).toMatchObject({ + "executor.runtime.rpc.opened": true, + "executor.runtime.rpc.close_code": 4001, + "executor.runtime.rpc.close_reason": "none", + "executor.runtime.rpc.sent": 0, + "executor.runtime.rpc.received": 0, + }); + expect(before.attributes).not.toHaveProperty("executor.runtime.rpc.last_received_ms"); + const after = yield* probe("ws-close-after-message"); + expect(after.attributes).toMatchObject({ + "executor.runtime.rpc.opened": true, + "executor.runtime.rpc.close_code": 1011, + "executor.runtime.rpc.sent": 1, + "executor.runtime.rpc.received": 1, + }); + // The client asked the stub to close at least 30 ms after it received the reply. + const failedMs = Number(after.attributes["executor.runtime.rpc.failed_ms"]); + const lastReceivedMs = Number( + after.attributes["executor.runtime.rpc.last_received_ms"], + ); + expect(failedMs).toBeGreaterThanOrEqual(30); + expect(after.attributes["executor.runtime.rpc.closed_ms"]).toBeGreaterThanOrEqual(30); + expect(lastReceivedMs).toBeGreaterThanOrEqual(30); + expect(lastReceivedMs).toBeLessThanOrEqual(failedMs); + }), + ); + + yield* evidence.step( + "An RPC session records only a transport failure: not an app's failure, a success or an interruption", + Effect.gen(function* () { + for (const name of ["rpc-app-failure", "rpc-success", "rpc-interrupt"]) { + const { attributes } = yield* probe(name); + expect(under(attributes, "executor.runtime."), name).toEqual([]); + } + }), + ); + }), + ), + ); +}); diff --git a/e2e/tests/search-statements.spec.ts b/e2e/tests/search-statements.spec.ts new file mode 100644 index 0000000000..0a001ccad9 --- /dev/null +++ b/e2e/tests/search-statements.spec.ts @@ -0,0 +1,400 @@ +/** + * MCP `search` reads every app the caller can reach before it ranks their kept tool listings, and + * each SQL statement is a network round trip from Cloud's session object to the database. Search + * reads what discovery needs for all of the request's apps together, so the statements under + * `mcp.search` do not grow with the number of apps: the regression guard is the count with a few + * apps against the count with several times as many, of each kind discovery treats differently (an + * app without accounts, one with a linked key account, one whose requirement has no linked account, + * and two whose OAuth account must reconnect, one the caller's and one the app's own). Each item + * still comes from its own app, and each app whose account must reconnect is still reported as + * unavailable, naming whose account it is as resolving it would. + */ +import { expect, layer } from "@effect/vitest"; +import { Effect, Schedule, Schema } from "effect"; +import { FetchHttpClient, HttpClient } from "effect/http"; +import { randomUUID } from "node:crypto"; +import { Actors } from "../support/actors.ts"; +import { Api, body } from "../support/api.ts"; +import { appsManifest } from "../support/apps-release.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { connectionsPath, linkedAccountIds } from "../support/app-accounts.ts"; +import { App, Resource } from "../support/contracts.ts"; +import { Evidence, Telemetry } from "../support/evidence.ts"; +import { McpClient } from "../support/mcp-client.ts"; +import { oauthSetupIssuer } from "../support/oauth-setup-issuer.ts"; +import { scenarios } from "../test-plan.ts"; + +/** Every tool names its app's marker, so an item shows which app it came from. */ +const tool = (marker: string, account: string) => + `probe: query({ description: "Search statements probe ${marker}.", input: object({}), output: string() }, async () => ${account})`; + +const accountlessSource = ( + marker: string, +) => `import { defineApp, object, string, query, router } from "apps"; +export default defineApp({}, { tools: router({ ${tool(marker, '"none"')} }) });`; + +const keySource = ( + marker: string, +) => `import { defineApp, defineProvider, secrets, object, string, query, router } from "apps"; +const service = defineProvider({ name: "Search statements key", auth: { + key: secrets({ label: "API key", fields: object({ token: string() }) }) +} }); +export default defineApp({ user: { accounts: { service } } }, async ctx => ({ + tools: router({ ${tool(marker, "ctx.user.accounts.service.id")} }), +}));`; + +const unlinkedSource = ( + marker: string, +) => `import { defineApp, defineProvider, secrets, object, string, query, router } from "apps"; +const service = defineProvider({ name: "Search statements unlinked", auth: { + key: secrets({ label: "API key", fields: object({ token: string() }) }) +} }); +export default defineApp({ user: { accounts: { service: service.many() } } }, async ctx => ({ + tools: router({ ${tool(marker, "String(ctx.user.accounts.service.length)")} }), +}));`; + +/** An OAuth app whose `service` account is the caller's (`user`) or the app's own (`app`). */ +const oauthSource = ( + discover: string, + scope: "user" | "app", + marker: string, +) => `import { defineApp, defineProvider, oauth2, object, string, query, router } from "apps"; +const service = defineProvider({ name: "Search statements OAuth", auth: { + oauth: oauth2({ discover: ${JSON.stringify(discover)} }) +} }); +export default defineApp({ ${scope}: { accounts: { service } } }, async ctx => ({ + tools: router({ ${tool(marker, `ctx.${scope}.accounts.service.fields.access_token`)} }), +}));`; + +const Token = Schema.Struct({ key: Schema.RedactedFromValue(Schema.String), id: Schema.String }); +const SignIn = Schema.Struct({ authorizationUrl: Schema.String }); +const SearchPage = Schema.Struct({ + items: Schema.Array(Schema.Struct({ path: Schema.String, description: Schema.String })), + unavailableApps: Schema.optional( + Schema.Array(Schema.Struct({ app: Schema.String, reason: Schema.String })), + ), +}); + +layer(HostedLive, { excludeTestServices: true })("Search statements", (it) => { + it.effect( + scenarios.searchStatements.title, + (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors, + evidence = yield* Evidence, + telemetry = yield* Telemetry, + mcp = yield* McpClient, + http = yield* HttpClient.HttpClient, + issuer = yield* oauthSetupIssuer; + const organization = actors.organization.id; + const prefix = `/api/organizations/${organization}`; + const accounts: string[] = []; + yield* Effect.addFinalizer(() => + Effect.forEach( + accounts, + (account) => + api + .request(actors.owner, "DELETE", `${prefix}/accounts/${account}`) + .pipe(Effect.map((response) => expect(response.status).toBe(200))), + { discard: true }, + ).pipe(Effect.orDie), + ); + /** Each deployed app, by the marker its tool names. */ + const deployed = new Map< + string, + { readonly id: string; readonly slug: string; readonly path: string } + >(); + const deploy = (kind: string, source: (marker: string) => string) => + Effect.gen(function* () { + const marker = `${kind}-${randomUUID().slice(0, 8)}`; + const response = yield* api.request(actors.owner, "POST", `${prefix}/apps/deploy`, { + name: `Search statements ${marker}`, + files: [{ path: "index.ts", content: source(marker) }, appsManifest], + }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + const app = yield* body(App, response); + const path = `${prefix}/apps/${app.id}`; + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", path).pipe( + Effect.map((removed) => expect(removed.status).toBe(200)), + Effect.orDie, + ), + ); + deployed.set(marker, { id: app.id, slug: app.slug, path }); + return { app, path, marker }; + }); + /** Connecting an account from the app links it for the owner, or for the app. */ + const connection = (path: string, scope: "user" | "app" = "user") => + api.request(actors.owner, "POST", connectionsPath(path, scope, "service"), {}).pipe( + Effect.tap((pending) => Effect.sync(() => expect(pending.status).toBe(200))), + Effect.flatMap((pending) => body(Resource, pending)), + ); + const linked = (path: string, account: string, scope: "user" | "app" = "user") => + Effect.gen(function* () { + accounts.push(account); + expect(yield* linkedAccountIds(actors.owner, path, scope)).toEqual([account]); + }); + const keyApp = Effect.gen(function* () { + const { path } = yield* deploy("key", keySource); + const pending = yield* connection(path); + const saved = yield* api.request( + actors.owner, + "POST", + `${prefix}/connections/${pending.id}/submit`, + { method: "key", label: "Synthetic", fields: { token: "synthetic-search" } }, + ); + expect(saved.status, JSON.stringify(saved.body)).toBe(200); + yield* linked(path, (yield* body(Resource, saved)).id); + }); + /** Apps of every kind but the OAuth one, `count` of each. */ + const deployKinds = (count: number) => + Effect.forEach( + Array.from({ length: count }), + () => + Effect.all([ + deploy("accountless", accountlessSource), + keyApp, + deploy("unlinked", unlinkedSource), + ]), + { discard: true }, + ); + + // Two OAuth apps whose service ends the grant, one using the caller's account and one + // the app's own: each next use marks its account for reconnecting. + yield* issuer.configure({ expiresIn: 1 }); + const signedIn = (scope: "user" | "app") => + Effect.gen(function* () { + const app = yield* deploy(`oauth${scope}`, (marker) => + oauthSource(`${issuer.origin}/mcp`, scope, marker), + ); + const pending = yield* connection(app.path, scope); + const started = yield* api.request( + actors.owner, + "POST", + `${prefix}/connections/${pending.id}/oauth/start`, + { method: "oauth", label: `Synthetic OAuth ${scope}` }, + ); + expect(started.status, JSON.stringify(started.body)).toBe(200); + const { authorizationUrl } = yield* body(SignIn, started); + const callbackUrl = yield* Effect.scoped( + Effect.gen(function* () { + const consent = yield* HttpClient.withScope(http).get(authorizationUrl); + expect(consent.status).toBe(302); + const location = consent.headers.location; + if (location === undefined) + return yield* Effect.die("Issuer did not return a callback"); + return location; + }), + ).pipe(Effect.provideService(FetchHttpClient.RequestInit, { redirect: "manual" })); + const completed = yield* api.request( + actors.owner, + "POST", + `${prefix}/connections/${pending.id}/oauth/complete`, + { callbackUrl }, + ); + expect(completed.status, JSON.stringify(completed.body)).toBe(200); + yield* linked(app.path, (yield* body(Resource, completed)).id, scope); + return app; + }); + const oauth = yield* signedIn("user"); + const appOauth = yield* signedIn("app"); + yield* Effect.sleep("1500 millis"); + yield* issuer.configure({ + tokenError: { status: 400, body: { error: "invalid_grant" } }, + }); + yield* Effect.forEach( + [oauth, appOauth], + (app) => + api + .request(actors.owner, "POST", `${app.path}/tools/call`, { + tool: "probe", + kind: "query", + input: {}, + }) + .pipe( + Effect.map((ended) => expect(ended.status, JSON.stringify(ended.body)).toBe(409)), + ), + { discard: true }, + ); + + /** An MCP connection with an API key of `actor`. */ + const connect = (actor: typeof actors.owner, name: string) => + Effect.gen(function* () { + const created = yield* api.request(actor, "POST", "/api/auth/api-key/create", { + name, + }); + expect(created.status).toBe(200); + const token = yield* body(Token, created); + yield* Effect.addFinalizer(() => + api + .request(actor, "POST", "/api/auth/api-key/delete", { keyId: token.id }) + .pipe(Effect.orDie), + ); + return yield* mcp.connect(token.key, "pat", { organization }); + }); + const client = yield* connect(actors.owner, "Search statements"); + type Connection = typeof client; + + /** One MCP search, its page and the SQL statements made under `mcp.search`. */ + const search = (operation: string, client: Connection) => + Effect.gen(function* () { + const called = yield* client.use(operation, (client, signal) => + client.callTool( + { name: "search", arguments: { query: "search statements probe", limit: 100 } }, + undefined, + { signal }, + ), + ); + const page = yield* Schema.decodeUnknownEffect(SearchPage)(called.structuredContent); + const request = (yield* evidence.requests) + .filter((request) => request.path === "/mcp") + .at(-1); + if (request === undefined) + return yield* Effect.die(new Error("MCP request evidence missing")); + const traced = yield* telemetry.query(request.traceId).pipe( + Effect.flatMap((result) => { + const root = result.data.find( + (entry) => entry.span.operationName === "mcp.search", + ); + const complete = result.data.some( + (entry) => entry.span.tags["http.response.status_code"] === "200", + ); + if (root === undefined || !complete) + return Effect.fail( + new Error("The completed server trace must reach the collector"), + ); + const byId = new Map(result.data.map(({ span }) => [span.spanId, span])); + const path = (spanId: string | null) => { + const names: string[] = []; + const visited = new Set(); + let parent = spanId; + while (parent !== null && !visited.has(parent)) { + if (parent === root.span.spanId) return names; + visited.add(parent); + const span = byId.get(parent); + names.push(span?.operationName ?? ""); + parent = span?.parentSpanId ?? null; + } + return undefined; + }; + const statements = result.data.flatMap(({ span }) => { + if (span.operationName !== "sql.execute") return []; + const names = path(span.parentSpanId); + if (names === undefined) return []; + return [ + { + text: String(span.tags["db.query.text"] ?? "") + .replace(/\s+/g, " ") + .slice(0, 100), + under: names.slice(0, 6).join(" < "), + }, + ]; + }); + return statements.length === 0 + ? Effect.fail(new Error("SQL descendants must reach the collector")) + : Effect.succeed(statements); + }), + Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 40 }), + Effect.timeout("25 seconds"), + ); + return { traceId: request.traceId, page, statements: traced }; + }); + + yield* deployKinds(1); + // The first search keeps each new app's listing as search rows; the next one reads them. + yield* search("Keep the listings of a few apps", client); + const few = yield* search("Search a few apps", client); + yield* deployKinds(4); + yield* search("Keep the listings of several times as many apps", client); + const many = yield* search("Search several times as many apps", client); + + // A member may use only the apps shared with them: the apps without accounts are shared + // with everyone, and the owner's other apps stay private. + const shared = [...deployed].filter(([marker]) => marker.startsWith("accountless-")); + yield* Effect.forEach( + shared, + ([, app]) => + Effect.gen(function* () { + const access = yield* body( + Schema.Struct({ revision: Schema.String }), + yield* api.request(actors.owner, "GET", `${app.path}/access`), + ); + const changed = yield* api.request(actors.owner, "PATCH", `${app.path}/access`, { + revision: access.revision, + audience: { kind: "everyone" }, + }); + expect(changed.status, JSON.stringify(changed.body)).toBe(200); + }), + { discard: true }, + ); + const memberClient = yield* connect(actors.member, "Search statements member"); + const member = yield* search("Search as a member", memberClient); + yield* evidence.json("search-statements.json", { + apps: { few: 4, many: 16 }, + few: { count: few.statements.length, ...few }, + many: { count: many.statements.length, ...many }, + member: { count: member.statements.length, ...member }, + }); + + /** The markers of the probes a page found; other apps, such as the built-in one, may match. */ + const probesOf = (label: string, result: typeof few) => + result.page.items.flatMap((item) => { + const marker = /Search statements probe ([a-z]+-[0-9a-f]{8})\./.exec( + item.description, + )?.[1]; + if (marker === undefined) return []; + const app = deployed.get(marker); + expect(app, `${label}: ${item.description}`).toBeDefined(); + expect(item.path, `${label}: an item is its own app's tool`).toBe( + `tools[${JSON.stringify(app?.slug)}].probe`, + ); + return [marker]; + }); + expect( + probesOf("member", member).sort(), + "A member finds the shared apps' tools and none of the owner's private apps'", + ).toEqual(shared.map(([marker]) => marker).sort()); + + for (const [label, result, apps] of [ + ["few", few, 3], + ["many", many, 15], + ] as const) { + const markers = probesOf(label, result); + expect( + markers.every((marker) => !marker.startsWith("oauth")), + `${label}: the apps whose account must reconnect list no tools`, + ).toBe(true); + expect(new Set(markers).size, `${label}: one item per app that lists tools`).toBe(apps); + const reason = (app: { readonly marker: string }) => + result.page.unavailableApps?.find( + (unavailable) => unavailable.app === deployed.get(app.marker)?.id, + )?.reason; + expect( + reason(oauth), + `${label}: the app whose caller's account must reconnect is reported`, + ).toContain("OAuthReconnectRequired"); + expect( + reason(oauth), + `${label}: the caller is told to reconnect their own account`, + ).not.toContain("someone who can edit"); + expect( + reason(appOauth), + `${label}: the app whose own account must reconnect is reported as the app's`, + ).toContain("someone who can edit the app must reconnect it"); + } + expect( + many.statements.length, + "Search reads every app's discovery state in statements that do not grow with the apps", + ).toBe(few.statements.length); + expect( + few.statements.length, + "The fork carries access with its app list, avoiding two repeated access reads; links, target snapshots, kept rows, rechecks, grants and ranking still stay fixed as apps grow", + ).toBe(10); + }).pipe(Effect.provide(McpClient.layer)), + ), + { timeout: 180_000 }, + ); +}); diff --git a/e2e/tests/seat-only-billing.spec.ts b/e2e/tests/seat-only-billing.spec.ts index 15f84d665e..81c7ef3455 100644 --- a/e2e/tests/seat-only-billing.spec.ts +++ b/e2e/tests/seat-only-billing.spec.ts @@ -106,6 +106,36 @@ layer(HostedLive, { excludeTestServices: true })("Seat-only billing", (it) => { }), )(result.structuredContent); expect(completed.execution.value).toEqual({ message: "seat-only", receipt }); + // The owner's agent reads the same plans through the Executor app. + const agentBilling = yield* client.use( + "Read billing through the Executor app", + (client, signal) => + client.callTool( + { + name: "execute", + arguments: { + code: `return await tools.executor.billing.overview(${JSON.stringify({ path: { organization: actors.organization.id } })});`, + }, + }, + undefined, + { signal }, + ), + ); + expect(agentBilling.isError).not.toBe(true); + const agentOverview = yield* Schema.decodeUnknownEffect( + Schema.Struct({ + status: Schema.Literal("completed"), + execution: Schema.Struct({ + ok: Schema.Literal(true), + value: Schema.Struct({ plans: Schema.Array(Schema.Struct({ name: Schema.String })) }), + }), + }), + )(agentBilling.structuredContent); + expect(agentOverview.execution.value.plans.map((plan) => plan.name).sort()).toEqual([ + "Enterprise", + "Free", + "Team", + ]); }).pipe(Effect.provide(Layer.mergeAll(McpOAuth.layer, McpClient.layer))), ), ); diff --git a/e2e/tests/self-host-native-telemetry.spec.ts b/e2e/tests/self-host-native-telemetry.spec.ts index 65359ac589..30cdc3168f 100644 --- a/e2e/tests/self-host-native-telemetry.spec.ts +++ b/e2e/tests/self-host-native-telemetry.spec.ts @@ -136,7 +136,7 @@ it.live(scenarios.selfHostNativeTelemetry.title, () => Effect.scoped( Effect.gen(function* () { const response = yield* collectorRequest(host.collectorSocket, "/v1/traces", body); - // Motel explains a refused export in its body, such as a full ingest queue. + // The collector explains a refused export in its body, such as a full ingest queue. expect(response.status, response.text.slice(0, 500)).toBe(200); const result = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Ingested))( response.text, @@ -177,7 +177,7 @@ it.live(scenarios.selfHostNativeTelemetryLifecycle.title, () => ChildProcess.make("pgrep", [ "-f", "--", - `--directory-path=motel-data=${host.motelDirectory.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`, + `--data=${host.motelDirectory.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`, ]), ) .pipe(Effect.map((lines) => lines.filter((line) => line.trim() !== "").map(Number))); diff --git a/e2e/tests/skill-editor.spec.ts b/e2e/tests/skill-editor.spec.ts index 2917f0456d..d095330f90 100644 --- a/e2e/tests/skill-editor.spec.ts +++ b/e2e/tests/skill-editor.spec.ts @@ -167,6 +167,11 @@ layer(HostedLive, { excludeTestServices: true })("Skill editor", (it) => { yield* Effect.addFinalizer(() => Effect.ignore(release)); yield* browser.login(actors.owner); + // A new app's first read of its working source creates its repository. On a deployed + // stage that can outlast the time the server's render waits for a read, and the page then + // leaves the skill to the browser after hydration, which the next step holds. Create the + // repository first, as for any app whose source was opened before. + expect(skillFile(yield* workspace())).toBe(skill); // The server sends the skill as the reader, and the editor replaces it once the page has // hydrated. Hold the page's scripts to see the server's page, then let it hydrate: the // skill's text, from its first heading to its last block, must stay where it was. One @@ -195,11 +200,22 @@ layer(HostedLive, { excludeTestServices: true })("Skill editor", (it) => { return scripts.then(() => route.fallback()); }) .then(() => page.goto(skillUrl, { waitUntil: "commit" })) + // The parser reaches the end of the server's document before any held script runs, + // so the page it has parsed then is all the server sent. .then(() => - Promise.all([ - held, - page.getByRole("cell", { name: "Full-text search", exact: true }).waitFor(), - ]), + Promise.all([held, page.waitForFunction(() => document.readyState !== "loading")]), + ) + // A part that streamed late arrives in a hidden container, and an inline script in + // the document moves it into place up to a few hundred milliseconds later. Check what + // the server sent wherever it sits, then wait for it to be in place. + .then(() => + page + .getByRole("cell", { name: "Full-text search", exact: true, includeHidden: true }) + .count(), + ) + .then((cells) => expect(cells, "the server's page shows the skill").toBe(1)) + .then(() => + page.getByRole("cell", { name: "Full-text search", exact: true }).waitFor(), ) // Measure with the final fonts; a late font load shifts the text by a pixel. .then(() => page.evaluate(() => document.fonts.ready.then(() => undefined))) diff --git a/e2e/tests/telemetry-backpressure.spec.ts b/e2e/tests/telemetry-backpressure.spec.ts index 1c8af0a2a3..0a4c5859d1 100644 --- a/e2e/tests/telemetry-backpressure.spec.ts +++ b/e2e/tests/telemetry-backpressure.spec.ts @@ -75,7 +75,7 @@ layer(TestLive, { excludeTestServices: true })("Telemetry backpressure", (it) => Effect.retry({ schedule: Schedule.spaced("200 millis"), times: 25 }), ); - // Motel's answer when its ingest queue is full. + // The collector's answer when its ingest queue is full. const full = yield* report([{ status: 429, retryAfter: () => "1" }]); expect(full.status, "a report the full queue refused is sent again").toBe(202); expect(full.arrivals.map(({ status }) => status)).toEqual([429, 200]); @@ -100,7 +100,7 @@ layer(TestLive, { excludeTestServices: true })("Telemetry backpressure", (it) => ); yield* stored(dated.traceId); - // OTLP throttles with 503 too, as Motel does while its storage is unavailable. + // OTLP throttles with 503 too, as the collector does while it stops. const unavailable = yield* report([{ status: 503, retryAfter: () => "2" }]); expect(unavailable.status, "a throttled 503 is sent again after its delay").toBe(202); expect(unavailable.arrivals.map(({ status }) => status)).toEqual([503, 200]); diff --git a/e2e/tests/telemetry-partial-trace.spec.ts b/e2e/tests/telemetry-partial-trace.spec.ts new file mode 100644 index 0000000000..ed7ca30d26 --- /dev/null +++ b/e2e/tests/telemetry-partial-trace.spec.ts @@ -0,0 +1,148 @@ +/** + * A trace the collector answers in part, because ordering it would take more than the collector's + * skeleton ceiling (its README, "Readers"), is never read by a check as the whole trace: the + * suite's `Telemetry.query` fails, without retrying, while `Telemetry.stored`, which failure + * evidence reads, keeps the spans returned and the collector's marker beside them. + */ +import { expect, layer } from "@effect/vitest"; +import { Cause, Effect, Exit, FileSystem, Layer } from "effect"; +import { HttpBody, HttpClient } from "effect/http"; +import { randomBytes } from "node:crypto"; +import { TestLive, withCase } from "../support/case.ts"; +import { Telemetry } from "../support/evidence.ts"; +import { ownCollectorSpans, serveOtlpCollector } from "../support/otlp-collector.ts"; +import { Target } from "../support/platform.ts"; +import { scenarios } from "../test-plan.ts"; + +/** + * Spans that each name a parent that never arrives. The collector counts 640 bytes of skeleton for + * such a span against a 48 MiB ceiling, so it answers at most 78,643 of them (README, "Readers"). + */ +const spans = 80_000; +const perExport = 10_000; + +layer(TestLive, { excludeTestServices: true })("Telemetry partial trace", (it) => { + it.effect(scenarios.telemetryPartialTrace.title, (context) => + withCase( + context, + Effect.gen(function* () { + const target = yield* Target, + fs = yield* FileSystem.FileSystem, + http = yield* HttpClient.HttpClient; + // A collector of the scenario's own, so no other scenario's reads meet this trace. + const directory = yield* fs.makeTempDirectory({ + directory: target.directory, + prefix: "partial", + }); + const collector = yield* serveOtlpCollector(directory); + const traceId = randomBytes(16).toString("hex"); + const start = BigInt(Date.now()) * 1_000_000n; + for (let first = 0; first < spans; first += perExport) { + const batch = Array.from({ length: perExport }, (_, offset) => { + const index = BigInt(first + offset); + return { + traceId, + spanId: randomBytes(8).toString("hex"), + parentSpanId: randomBytes(8).toString("hex"), + name: "orphan", + kind: 1, + startTimeUnixNano: String(start + index), + endTimeUnixNano: String(start + index + 1n), + }; + }); + const { status } = yield* Effect.scoped( + http.post(new URL("/v1/traces", collector).href, { + body: HttpBody.jsonUnsafe({ + resourceSpans: [ + { + resource: { + attributes: [{ key: "service.name", value: { stringValue: "partial" } }], + }, + scopeSpans: [{ scope: { name: "e2e.partial" }, spans: batch }], + }, + ], + }), + }), + ); + expect(status).toBe(200); + } + // The suite's own reader, pointed at this collector (`collector.json` in its directory). + const reader = Telemetry.layer.pipe( + Layer.provide(Layer.succeed(Target, { ...target, directory })), + ); + const { query, stored, searchWholeEnvironment, logs, logsWholeEnvironment } = + yield* Effect.gen(function* () { + return yield* Telemetry; + }).pipe(Effect.provide(reader)); + + const kept = yield* stored(traceId); + expect(kept.meta?.truncated, "the collector marks the answer as part of the trace").toBe( + true, + ); + expect(kept.meta?.spanCount).toBe(spans); + expect(kept.meta?.returned).toBeLessThan(spans); + // Each returned span comes with a placeholder for its missing parent. + expect(kept.data.length).toBe(2 * kept.meta!.returned!); + + const read = yield* Effect.exit(query(traceId)); + expect(Exit.isFailure(read), "a check cannot read the trace as whole").toBe(true); + if (Exit.isFailure(read)) { + expect(Cause.hasDies(read.cause), "the read is a defect, never retried").toBe(true); + expect(Cause.pretty(read.cause)).toContain( + `The collector answered trace ${traceId} in part (${kept.meta!.returned} of ${spans} spans)`, + ); + } + + // A span search the collector cut off at its limit fails the same way: 80,000 spans match. + const searched = yield* Effect.exit(searchWholeEnvironment("orphan", {})); + expect(Exit.isFailure(searched), "a check cannot read a cut-off search as all").toBe(true); + if (Exit.isFailure(searched)) { + expect(Cause.hasDies(searched.cause)).toBe(true); + expect(Cause.pretty(searched.cause)).toContain("answered only those"); + } + const windowed = yield* Effect.exit( + ownCollectorSpans(collector, { + from: Date.now() - 60_000, + to: Date.now(), + operation: "orphan", + }), + ); + expect(Exit.isFailure(windowed), "nor a cut-off search in a time window").toBe(true); + if (Exit.isFailure(windowed)) expect(Cause.hasDies(windowed.cause)).toBe(true); + + // Log lists are read whole: 601 logs of one trace, more than one page, the oldest holding a + // marker a privacy check looks for. Both log readers must see it. + const logTrace = randomBytes(16).toString("hex"); + const common = `partial-logs-${randomBytes(4).toString("hex")}`; + const forbidden = `forbidden-${randomBytes(4).toString("hex")}`; + const records = Array.from({ length: 601 }, (_, index) => ({ + timeUnixNano: String(start + BigInt(index) * 1_000_000n), + severityNumber: 9, + severityText: "INFO", + body: { stringValue: index === 0 ? `${common} ${forbidden}` : `${common} ${index}` }, + traceId: logTrace, + })); + const { status: logged } = yield* Effect.scoped( + http.post(new URL("/v1/logs", collector).href, { + body: HttpBody.jsonUnsafe({ + resourceLogs: [ + { + resource: { + attributes: [{ key: "service.name", value: { stringValue: "partial" } }], + }, + scopeLogs: [{ scope: { name: "e2e.partial" }, logRecords: records }], + }, + ], + }), + }), + ); + expect(logged).toBe(200); + const traceLogs = yield* logs(logTrace); + expect(traceLogs, "a trace's logs are read past the first page").toContain(forbidden); + expect(traceLogs.split(`"${common} `).length - 1).toBe(601); + const found = yield* logsWholeEnvironment(common); + expect(found, "a log search is read past the first page").toContain(forbidden); + }), + ), + ); +}); diff --git a/e2e/tests/telemetry-store-bound.spec.ts b/e2e/tests/telemetry-store-bound.spec.ts index a40a569346..3991f4f1d9 100644 --- a/e2e/tests/telemetry-store-bound.spec.ts +++ b/e2e/tests/telemetry-store-bound.spec.ts @@ -1,9 +1,11 @@ /** * The bundled collector, once its store is full, keeps the traces it holds while an idle server - * trickles in small exports, instead of evicting the oldest for every export. + * trickles in small exports, instead of evicting the oldest for every export. It evicts in steps + * after it answers an export, so the scenario waits on its own status (`/api/ingest`'s + * `store.maintenancePending` and `store.draining`) for each eviction to finish, never a fixed time. */ import { expect, layer } from "@effect/vitest"; -import { Effect, FileSystem, Schema } from "effect"; +import { Effect, FileSystem, Schedule, Schema } from "effect"; import { HttpBody, HttpClient } from "effect/http"; import { randomBytes } from "node:crypto"; import { TestLive, withCase } from "../support/case.ts"; @@ -19,6 +21,12 @@ const idleExports = 20; const Evictions = Schema.Struct({ evictions: Schema.Struct({ passes: Schema.Number, traces: Schema.Number }), }); +/** Whether retention has finished: no eviction or free pages left to do. */ +const Retention = Schema.Struct({ + store: Schema.Struct({ maintenancePending: Schema.Boolean, draining: Schema.Boolean }), +}); +/** How long retention may take to finish after an export: a step is bounded in rows. */ +const retentionFinishes = "20 seconds"; const attribute = (key: string, value: string) => ({ key, value: { stringValue: value } }); @@ -72,9 +80,7 @@ layer(TestLive, { excludeTestServices: true })("Telemetry store bound", (it) => directory: target.directory, prefix: "collector", }); - const collector = yield* serveOtlpCollector(directory, { - MOTEL_OTEL_MAX_DB_SIZE_MB: String(boundMegabytes), - }); + const collector = yield* serveOtlpCollector(directory, { maxStoreMb: boundMegabytes }); const send = (traceIds: ReadonlyArray, spans: number) => Effect.scoped( http.post(new URL("/v1/traces", collector).href, { @@ -93,13 +99,30 @@ layer(TestLive, { excludeTestServices: true })("Telemetry store bound", (it) => Effect.map(({ evictions }) => evictions), ); const newId = () => randomBytes(16).toString("hex"); + // The collector's status once retention has finished. Its first status read after an + // export's 200 already says whether that export left work to do. + const settled = ingestStatus.pipe( + Effect.flatMap((status) => + Schema.decodeUnknownEffect(Retention)(status).pipe( + Effect.flatMap(({ store }) => + store.maintenancePending || store.draining + ? Effect.fail("Retention has not finished") + : Effect.succeed(status), + ), + ), + ), + Effect.retry({ schedule: Schedule.spaced("100 millis") }), + Effect.timeout(retentionFinishes), + ); - // Fill the store, two 40-span traces an export, until it evicts its first trace. + // Fill the store, two 40-span traces an export, until it evicts its first trace, and let + // that eviction finish. const sent: Array = []; while (sent.length === 0 || (yield* stored(sent[0]!))) { const traceIds = [newId(), newId()]; expect(yield* send(traceIds, 40)).toBe(200); sent.push(...traceIds); + yield* settled; } // Traces are evicted oldest first, so the ones still stored are a suffix of those sent. let low = 0; @@ -110,21 +133,21 @@ layer(TestLive, { excludeTestServices: true })("Telemetry store bound", (it) => else low = middle + 1; } const oldest = sent[low]!; - const filled = yield* ingestStatus; + const filled = yield* settled; // An idle server's exports, about half a MiB in all, after the store first evicted. const idle = Array.from({ length: idleExports }, newId); for (const traceId of idle) expect(yield* send([traceId], 10)).toBe(200); + const afterIdle = yield* settled; expect(yield* stored(oldest), "The oldest stored trace is still stored").toBe(true); for (const traceId of idle) expect(yield* stored(traceId)).toBe(true); // The export that crossed the bound evicted down to the low-water mark in one pass. const passes = yield* evictions(filled); expect(passes.passes, "Filling the store took one eviction pass").toBe(1); - expect( - yield* evictions(yield* ingestStatus), - "No idle export started an eviction pass", - ).toEqual(passes); + expect(yield* evictions(afterIdle), "No idle export started an eviction pass").toEqual( + passes, + ); }), ), ); diff --git a/e2e/tests/telemetry-viewer.spec.ts b/e2e/tests/telemetry-viewer.spec.ts new file mode 100644 index 0000000000..8b8f4a9e70 --- /dev/null +++ b/e2e/tests/telemetry-viewer.spec.ts @@ -0,0 +1,231 @@ +/** + * The local server's telemetry collector serves the trace viewer at the URL `collector.json` + * names: the trace list, a trace, a reload on a trace's own address, an unknown trace and the + * logs page's filters, all from the collector's own answers, and never the AI routes Motel's + * viewer called. + */ +import { expect, layer } from "@effect/vitest"; +import { Effect, FileSystem, Schema } from "effect"; +import { HttpBody, HttpClient } from "effect/http"; +import { randomBytes } from "node:crypto"; +import { Browser } from "../support/browser.ts"; +import { TestLive, withCase } from "../support/case.ts"; +import { Collector } from "../support/contracts.ts"; +import { Target } from "../support/platform.ts"; +import { scenarios } from "../test-plan.ts"; + +const attribute = (key: string, value: string) => ({ key, value: { stringValue: value } }); + +layer(TestLive, { excludeTestServices: true })("Telemetry viewer", (it) => { + it.effect(scenarios.telemetryViewer.title, (context) => + withCase( + context, + Effect.gen(function* () { + const target = yield* Target, + fs = yield* FileSystem.FileSystem, + http = yield* HttpClient.HttpClient, + browser = yield* Browser; + const { url } = yield* fs + .readFileString(`${target.directory}/data/diagnostics/collector.json`) + .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(Collector)))); + const marker = randomBytes(6).toString("hex"); + const service = `viewer-${marker}`; + const operation = `viewer.root.${marker}`; + const child = `viewer.child.${marker}`; + const traceId = randomBytes(16).toString("hex"); + const root = randomBytes(8).toString("hex"); + const now = BigInt(Date.now()) * 1_000_000n; + const post = (path: string, body: unknown) => + Effect.scoped( + http.post(new URL(path, url).href, { body: HttpBody.jsonUnsafe(body) }), + ).pipe(Effect.map(({ status }) => status)); + const resource = { attributes: [attribute("service.name", service)] }; + expect( + yield* post("/v1/traces", { + resourceSpans: [ + { + resource, + scopeSpans: [ + { + scope: { name: "e2e.viewer" }, + spans: [ + { + traceId, + spanId: root, + name: operation, + kind: 2, + startTimeUnixNano: String(now - 20_000_000n), + endTimeUnixNano: String(now), + status: { code: 1 }, + }, + { + traceId, + spanId: randomBytes(8).toString("hex"), + parentSpanId: root, + name: child, + kind: 1, + startTimeUnixNano: String(now - 15_000_000n), + endTimeUnixNano: String(now - 5_000_000n), + status: { code: 1 }, + }, + ], + }, + ], + }, + ], + }), + ).toBe(200); + const log = (severityText: string, severityNumber: number, text: string) => ({ + timeUnixNano: String(now), + severityNumber, + severityText, + body: { stringValue: text }, + traceId, + spanId: root, + }); + const infoBody = `viewer info ${marker}`; + const errorBody = `viewer error ${marker}`; + expect( + yield* post("/v1/logs", { + resourceLogs: [ + { + resource, + scopeLogs: [ + { + scope: { name: "e2e.viewer" }, + logRecords: [log("INFO", 9, infoBody), log("ERROR", 17, errorBody)], + }, + ], + }, + ], + }), + ).toBe(200); + + // Every request the viewer makes, to show none goes to the AI routes it no longer has. + const requested: Array = []; + // Every script and stylesheet the viewer's pages load, and how each was answered. + const assets: Array<{ + readonly path: string; + readonly type: string; + readonly status: number; + }> = []; + yield* browser.use("Record the viewer's requests", (page) => + Promise.resolve( + page + .on("request", (request) => { + requested.push(new URL(request.url()).pathname); + }) + .on("response", (response) => { + const type = response.request().resourceType(); + if (type === "script" || type === "stylesheet") + assets.push({ + path: new URL(response.url()).pathname, + type, + status: response.status(), + }); + }) + .on("requestfailed", (request) => { + const type = request.resourceType(); + if (type === "script" || type === "stylesheet") + assets.push({ path: new URL(request.url()).pathname, type, status: 0 }); + }), + ), + ); + yield* browser.use("Open the viewer at the collector's address", (page) => + page.goto(new URL("/", url).href), + ); + yield* browser.use("Find the trace in the list", (page) => + page.getByText(operation, { exact: true }).waitFor(), + ); + expect( + yield* browser.use("Read the address the root redirected to", (page) => + Promise.resolve(new URL(page.url()).pathname), + ), + ).toBe("/traces"); + yield* browser.checkpoint("Trace list"); + yield* browser.use("Open the trace", (page) => + page + .getByText(operation, { exact: true }) + .click() + .then(() => page.getByRole("heading", { level: 1, name: operation }).waitFor()), + ); + expect( + yield* browser.use("Read the trace's address", (page) => + Promise.resolve(new URL(page.url()).pathname), + ), + ).toBe(`/trace/${traceId}`); + yield* browser.use("Find the child span in the waterfall", (page) => + page + .getByLabel(/Trace waterfall/) + .getByText(child, { exact: true }) + .waitFor(), + ); + // The stylesheet laid the page out: a span's bar has the height only it gives. + expect( + yield* browser.use("Measure a span's bar", (page) => + page + .locator('[data-span-bar="true"]') + .first() + .boundingBox() + .then((box) => box?.height ?? 0), + ), + "the waterfall's bars are drawn", + ).toBeGreaterThan(0); + yield* browser.checkpoint("Trace"); + // The collector serves the viewer's page at the trace's own address, so a reload stays. + yield* browser.use("Reload the trace", (page) => + page + .reload() + .then(() => page.getByRole("heading", { level: 1, name: operation }).waitFor()), + ); + const unknown = randomBytes(16).toString("hex"); + yield* browser.use("Open a trace the collector does not hold", (page) => + page + .goto(new URL(`/trace/${unknown}`, url).href) + .then(() => page.getByText("Trace not found").waitFor()), + ); + yield* browser.checkpoint("Unknown trace"); + + // The logs page: the body search finds both of this trace's logs, the severity filter one. + yield* browser.use("Open the logs", (page) => + page + .goto(new URL("/logs", url).href) + .then(() => page.getByPlaceholder("Search log body...").fill(marker)) + .then(() => page.getByPlaceholder("Search log body...").press("Enter")) + .then(() => page.getByText(infoBody, { exact: true }).waitFor()) + .then(() => page.getByText(errorBody, { exact: true }).waitFor()), + ); + yield* browser.use("Filter the logs to errors", (page) => + page + .getByRole("button", { name: "ERROR", exact: true }) + .click() + .then(() => page.getByText(infoBody, { exact: true }).waitFor({ state: "detached" })) + .then(() => page.getByText(errorBody, { exact: true }).waitFor()), + ); + expect( + yield* browser.use("Read the filtered address", (page) => + Promise.resolve(Object.fromEntries(new URL(page.url()).searchParams)), + ), + ).toEqual({ body: marker, severity: "ERROR" }); + yield* browser.checkpoint("Logs filtered to errors"); + + expect(requested.length, "the viewer made requests").toBeGreaterThan(0); + // Its pages loaded a script and a stylesheet, every one of them whole. + expect(assets.map(({ type }) => type).sort()).toEqual( + expect.arrayContaining(["script", "stylesheet"]), + ); + expect( + assets.filter(({ status }) => status !== 200), + "every script and stylesheet the viewer requested was served", + ).toEqual([]); + expect( + requested.filter((path) => path.startsWith("/api/ai")), + "the viewer asks no AI route", + ).toEqual([]); + // Its data came from the collector's own query routes. + expect(requested).toContain(`/api/traces/${traceId}`); + expect(requested).toContain("/api/logs"); + }), + ), + ); +}); diff --git a/e2e/tests/test-stage-branch.spec.ts b/e2e/tests/test-stage-branch.spec.ts new file mode 100644 index 0000000000..14499f7c7a --- /dev/null +++ b/e2e/tests/test-stage-branch.spec.ts @@ -0,0 +1,361 @@ +/** + * A deployed E2E stage's teardown removes its PlanetScale branch by its exact name when Alchemy's + * destroy left it (a create that never became ready is "not owned by this stack"), and keeps the + * stage's lease unless PlanetScale then answers 404 (`test-stage-branch.ts`). These cases run the + * removal as a process (`apps/hosted/cloud/scripts/test-stage-branch.ts`, the same + * `removeStageBranch` the destroy calls) against a local server standing in for PlanetScale's API. + * The stand-in holds the stage's branch beside branches that must never be touched: a production + * branch, the test database's index-lock branch, and names that only start with or contain the + * stage's. Every request is recorded; anything but a read or delete of one branch is answered 500. + */ +import { expect, layer } from "@effect/vitest"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { Effect, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; + +const organization = "synthetic-org"; +const database = "executor-test"; +const branchPath = (name: string) => + `/v1/organizations/${organization}/databases/${database}/branches/${name}`; + +/** How the stand-in answers for one branch. */ +type Held = { + readonly production: boolean; + readonly ready: boolean; + /** What a delete answers; a 2xx removes the branch unless `stays`. */ + readonly deleteStatus: number; + readonly stays?: boolean; + /** The name the branch's read answers with, when it is not its own. */ + readonly answersAs?: string; + /** Reads answer 200 this many times, then 404: the destroy's own delete landing. */ + readonly goneAfterReads?: number; + /** A delete gets no answer: the connection is dropped. */ + readonly dropDelete?: boolean; + /** A read answers this status with the branch's body. */ + readonly readStatus?: number; + /** A read never answers. */ + readonly hangRead?: boolean; +}; + +/** Branches no teardown may touch, whatever the slug. */ +const bystanders: Record = { + main: { production: true, ready: true, deleteStatus: 500 }, + "idx-lock-38132096340-1": { production: true, ready: true, deleteStatus: 500 }, + "test-e2e-stuck1-copy": { production: false, ready: true, deleteStatus: 500 }, + "xtest-e2e-stuck1": { production: false, ready: true, deleteStatus: 500 }, + "test-apps-v2-w5": { production: false, ready: true, deleteStatus: 500 }, +}; + +type Seen = { method: string; path: string; authorization: string | undefined }; + +const planetscale = (seen: Seen[], held: Record) => + Effect.acquireRelease( + Effect.callback<{ origin: string; close: () => void }>((resume) => { + const branches = new Map(Object.entries({ ...bystanders, ...held })); + const readCounts = new Map(); + const server = createServer((request: IncomingMessage, response: ServerResponse) => { + const url = new URL(request.url ?? "/", "http://localhost"); + seen.push({ + method: request.method ?? "", + path: url.pathname, + authorization: request.headers.authorization, + }); + const json = (status: number, value?: unknown) => + response + .writeHead(status, { "content-type": "application/json" }) + .end(value === undefined ? undefined : JSON.stringify(value)); + const prefix = branchPath(""); + if (!url.pathname.startsWith(prefix)) + return json(500, { message: "the teardown asked for something it must not" }); + const name = decodeURIComponent(url.pathname.slice(prefix.length)); + const branch = branches.get(name); + if (request.method === "GET" && branch?.hangRead === true) return; + const reads = (readCounts.get(name) ?? 0) + 1; + if (request.method === "GET") readCounts.set(name, reads); + if ( + request.method === "GET" && + branch?.goneAfterReads !== undefined && + reads > branch.goneAfterReads + ) + branches.delete(name); + if (request.method === "GET" && branches.get(name) === undefined) + return json(404, { code: "not_found", message: "Not Found" }); + if (request.method === "GET") + return branch === undefined + ? json(404, { code: "not_found", message: "Not Found" }) + : json(branch.readStatus ?? 200, { + type: "Branch", + name: branch.answersAs ?? name, + production: branch.production, + ready: branch.ready, + }); + if (request.method === "DELETE") { + if (branch === undefined) return json(404, { code: "not_found" }); + if (branch.dropDelete === true) return request.socket.destroy(); + if (branch.deleteStatus >= 200 && branch.deleteStatus < 300 && branch.stays !== true) + branches.delete(name); + return branch.deleteStatus === 204 + ? response.writeHead(204).end() + : json(branch.deleteStatus, { code: "forbidden", message: "Forbidden" }); + } + return json(500, { message: "the teardown asked for something it must not" }); + }); + server.listen(0, () => { + const { port } = server.address() as AddressInfo; + resume(Effect.succeed({ origin: `http://localhost:${port}`, close: () => server.close() })); + }); + }), + ({ close }) => Effect.sync(close), + ); + +const environment = { + TEST_STAGE_DATABASE: database, + PLANETSCALE_ORGANIZATION: organization, + PLANETSCALE_API_TOKEN_ID: "synthetic-id", + PLANETSCALE_API_TOKEN: "synthetic-token", +}; + +/** Remove `slug`'s branch against the stand-in; the printed outcome and the exit code. */ +const remove = (slug: string, origin: string) => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make( + "node", + ["apps/hosted/cloud/scripts/test-stage-branch.ts", "--slug", slug], + { + env: { + PATH: process.env.PATH ?? "", + ...environment, + TEST_STAGE_PLANETSCALE_API: origin, + TEST_STAGE_BRANCH_POLL: "20 millis", + }, + extendEnv: false, + stdout: "pipe", + stderr: "pipe", + }, + ), + ); + const [stdout, stderr, code] = yield* Effect.all( + [ + Stream.mkString(Stream.decodeText(child.stdout)), + Stream.mkString(Stream.decodeText(child.stderr)), + child.exitCode, + ], + { concurrency: 3 }, + ); + expect(stderr).toBe(""); + return { + ...(JSON.parse(stdout) as { + outcome: { _tag: string; status?: number }; + message: string; + leaseKept: boolean; + }), + code: Number(code), + }; + }).pipe(Effect.scoped); + +const stuck = { production: false, ready: false, deleteStatus: 204 } satisfies Held; + +layer(NodeServices.layer, { excludeTestServices: true })("Test stage branch removal", (it) => { + it.effect( + "a create that never became ready: the delete goes to the exact name, and 404 is confirmed", + () => + Effect.gen(function* () { + const seen: Seen[] = []; + const { origin } = yield* planetscale(seen, { "test-e2e-stuck1": stuck }); + const run = yield* remove("e2e-stuck1", origin); + expect(run.code).toBe(0); + expect(run.outcome).toEqual({ _tag: "Deleted", status: 204 }); + expect(run.leaseKept).toBe(false); + expect(run.message).toBe( + "PlanetScale branch test-e2e-stuck1 was left by the destroy; a delete was sent (PlanetScale answered 204) and it is gone (404).", + ); + // A read, seven more while the destroy's own delete could land (one now, six spaced), one delete, one read. + const at = branchPath("test-e2e-stuck1"); + expect(seen.map(({ method, path }) => `${method} ${path}`)).toEqual([ + ...Array.from({ length: 8 }, () => `GET ${at}`), + `DELETE ${at}`, + `GET ${at}`, + ]); + for (const request of seen) + expect(request.authorization).toBe("synthetic-id:synthetic-token"); + }), + ); + + it.effect("a refused delete keeps the lease and says a delete was sent and refused", () => + Effect.gen(function* () { + const seen: Seen[] = []; + const { origin } = yield* planetscale(seen, { + "test-e2e-stuck2": { ...stuck, deleteStatus: 403 }, + }); + const run = yield* remove("e2e-stuck2", origin); + expect(run.code).toBe(1); + expect(run.outcome).toEqual({ _tag: "DeleteRefused", status: 403 }); + expect(run.leaseKept).toBe(true); + expect(run.message).toContain( + "PlanetScale branch test-e2e-stuck2 was left by the destroy. A delete was sent and PlanetScale refused it (403).", + ); + expect(run.message).toContain("Its stage lease is kept"); + expect(seen.map(({ method }) => method)).toEqual([...Array(8).fill("GET"), "DELETE"]); + }), + ); + + it.effect("a production (paid-size) branch is never sent a delete, and the lease is kept", () => + Effect.gen(function* () { + const seen: Seen[] = []; + const { origin } = yield* planetscale(seen, { + "test-e2e-paid1": { production: true, ready: true, deleteStatus: 204 }, + }); + const run = yield* remove("e2e-paid1", origin); + expect(run.code).toBe(1); + expect(run.outcome).toEqual({ _tag: "Production" }); + expect(run.leaseKept).toBe(true); + expect(run.message).toContain( + "is a production branch (created at a paid size). No delete was sent", + ); + expect(seen.map(({ method, path }) => `${method} ${path}`)).toEqual([ + `GET ${branchPath("test-e2e-paid1")}`, + ]); + }), + ); + + it.effect("a delete accepted but still listed keeps the lease and says how long it waited", () => + Effect.gen(function* () { + const seen: Seen[] = []; + const { origin } = yield* planetscale(seen, { + "test-e2e-slow1": { ...stuck, stays: true }, + }); + const run = yield* remove("e2e-slow1", origin); + expect(run.code).toBe(1); + expect(run.outcome._tag).toBe("StillThere"); + expect(run.leaseKept).toBe(true); + expect(run.message).toContain( + "A delete was sent and accepted (204), but PlanetScale still lists it after", + ); + // Eight reads before, one delete, then the first check and 24 more. + expect(seen.map(({ method }) => method)).toEqual([ + ...Array(8).fill("GET"), + "DELETE", + ...Array(25).fill("GET"), + ]); + }), + ); + + it.effect("a branch already gone needs no delete, and the lease may go", () => + Effect.gen(function* () { + const seen: Seen[] = []; + const { origin } = yield* planetscale(seen, {}); + const run = yield* remove("e2e-gone1", origin); + expect(run.code).toBe(0); + expect(run.outcome).toEqual({ _tag: "Absent" }); + expect(run.leaseKept).toBe(false); + expect(seen.map(({ method }) => method)).toEqual(["GET"]); + }), + ); + + it.effect("a read that answers for another branch sends no delete", () => + Effect.gen(function* () { + const seen: Seen[] = []; + const { origin } = yield* planetscale(seen, { + "test-e2e-odd1": { ...stuck, answersAs: "main" }, + }); + const run = yield* remove("e2e-odd1", origin); + expect(run.code).toBe(1); + expect(run.outcome).toEqual({ _tag: "Unreadable", status: 200 }); + expect(run.leaseKept).toBe(true); + expect(seen.map(({ method }) => method)).toEqual(["GET"]); + }), + ); + + it.effect("a branch the destroy is still deleting gets no second delete", () => + Effect.gen(function* () { + const seen: Seen[] = []; + const { origin } = yield* planetscale(seen, { + "test-e2e-owned1": { ...stuck, goneAfterReads: 3 }, + }); + const run = yield* remove("e2e-owned1", origin); + expect(run.code).toBe(0); + expect(run.outcome).toEqual({ _tag: "Absent" }); + expect(run.leaseKept).toBe(false); + expect(run.message).toContain("No delete was needed."); + expect(seen.map(({ method }) => method)).toEqual(["GET", "GET", "GET", "GET"]); + }), + ); + + it.effect("a delete that gets no answer says a delete was sent, and keeps the lease", () => + Effect.gen(function* () { + const seen: Seen[] = []; + const { origin } = yield* planetscale(seen, { + "test-e2e-drop1": { ...stuck, dropDelete: true }, + }); + const run = yield* remove("e2e-drop1", origin); + expect(run.code).toBe(1); + expect(run.outcome).toEqual({ _tag: "DeleteUnanswered" }); + expect(run.leaseKept).toBe(true); + expect(run.message).toContain("A delete was sent; no answer came back within 20 seconds."); + expect(run.message).not.toContain("No delete was sent"); + expect(seen.map(({ method }) => method)).toEqual([...Array(8).fill("GET"), "DELETE"]); + }), + ); + + it.effect("a read answering other than 200 or 404 sends no delete, whatever its body", () => + Effect.gen(function* () { + const seen: Seen[] = []; + const { origin } = yield* planetscale(seen, { + "test-e2e-err1": { ...stuck, readStatus: 500 }, + }); + const run = yield* remove("e2e-err1", origin); + expect(run.code).toBe(1); + expect(run.outcome).toEqual({ _tag: "Unreadable", status: 500 }); + expect(run.leaseKept).toBe(true); + expect(seen.map(({ method }) => method)).toEqual(["GET"]); + }), + ); + + it.effect( + "a read that never answers is given up after 20 s, with no delete and the lease kept", + () => + Effect.gen(function* () { + const seen: Seen[] = []; + const { origin } = yield* planetscale(seen, { + "test-e2e-hang1": { ...stuck, hangRead: true }, + }); + const run = yield* remove("e2e-hang1", origin); + expect(run.code).toBe(1); + expect(run.outcome).toEqual({ _tag: "Unreadable" }); + expect(run.leaseKept).toBe(true); + expect(run.message).toContain("could not be read (no answer within 20 seconds)"); + expect(seen.map(({ method }) => method)).toEqual(["GET"]); + }), + 60_000, + ); + + it.effect("a name that is not exactly a deployed E2E stage's is refused before any request", () => + Effect.gen(function* () { + const seen: Seen[] = []; + const { origin } = yield* planetscale(seen, { "test-e2e-stuck1": stuck }); + for (const slug of [ + "idx-lock-38132096340-1", + "apps-v2-w5", + "pr-4120", + "main", + "e2e-", + "e2e-STUCK1", + "e2e-stuck1-copy", + "e2e-abcdefghijklmn", + "e2e-stuck1/../main", + "", + ]) { + const run = yield* remove(slug, origin); + expect(run.code, slug).toBe(1); + expect(run.outcome._tag, slug).toBe("Refused"); + expect(run.leaseKept, slug).toBe(true); + expect(run.message, slug).toContain("No request was sent."); + } + expect(seen).toEqual([]); + }), + ); +}); diff --git a/e2e/tests/test-stage-unreadable.spec.ts b/e2e/tests/test-stage-unreadable.spec.ts index 2db3a75333..0f40d42e5e 100644 --- a/e2e/tests/test-stage-unreadable.spec.ts +++ b/e2e/tests/test-stage-unreadable.spec.ts @@ -71,6 +71,8 @@ interface World { /** Alchemy records by stage and logical ID: readable, undecryptable, or failing that many reads. */ state: Record>; faults: Fault[]; + /** The PlanetScale test database's branches: whether production, and what a delete answers. */ + planetscale?: Record; /** Pages of the bucket listing, by name, when the listing is paginated. */ bucketPages?: ReadonlyArray>; /** Listing results that report more pages than they return. */ @@ -265,6 +267,40 @@ const answer = (world: World, method: string, url: URL, body: string, authorizat return { status: 204, headers: {}, body: "" }; } + // PlanetScale: the test database's branches. + if ( + (match = + /^\/v1\/organizations\/synthetic-org\/databases\/executor-test\/branches(?:\/([^/]+))?$/u.exec( + path, + )) + ) { + const branches = world.planetscale ?? {}; + const name = match[1] === undefined ? undefined : decodeURIComponent(match[1]); + if (name === undefined && method === "GET") + return json({ + type: "list", + current_page: 1, + next_page: null, + data: Object.keys(branches).map((branch) => ({ + name: branch, + created_at: "2026-11-16T00:00:00Z", + })), + }); + const branch = name === undefined ? undefined : branches[name]; + if (branch === undefined) return missing; + if (method === "GET") return json({ name, production: branch.production, ready: false }); + if (method === "DELETE") { + if (branch.deleteStatus !== 204) + return { + status: branch.deleteStatus, + headers: {}, + body: JSON.stringify({ code: "forbidden" }), + }; + delete branches[name!]; + return { status: 204, headers: {}, body: "" }; + } + } + // Cloudflare. if (path === `${cf}/workers/scripts`) { if (method === "GET") @@ -552,6 +588,8 @@ const cli = ( TEST_STAGE_AXIOM_API: apis.origin, TEST_STAGE_STATE_STORE_URL: apis.origin, TEST_STAGE_STATE_STORE_TOKEN: "state-token", + // Only read when a case names a PlanetScale test database. + TEST_STAGE_PLANETSCALE_API: apis.origin, TEST_STAGE_REMOVAL_SETTLE: "10 millis", TEST_STAGE_API_BACKOFF: "10 millis", ...(options.registry === undefined @@ -1034,3 +1072,144 @@ describe("Removing a test stage by name, with its registry", () => { 180_000, ); }); + +/** + * A deployed E2E stage on PlanetScale whose deploy stopped before Alchemy recorded its branch: the + * stage's Alchemy destroy (a stand-in that succeeds and deletes nothing) leaves the branch, and + * `test-stage destroy` then removes it by its exact name, keeping the stage's lease unless + * PlanetScale answers 404 (`test-stage-branch.ts`). The same holds for scheduled cleanup. + */ +describe("Removing a branch a PlanetScale stage's destroy left", () => { + const leftWorld = (slug: string, deleteStatus: number): World => ({ + workers: [], + zones: [testZone, appZone, otherZone], + routes: {}, + domains: [], + workflows: [], + buckets: {}, + dns: {}, + destinations: [], + hyperdrive: [], + certificates: [], + artifacts: {}, + branches: [], + monitors: [], + tokens: [], + state: { [`test-${slug}`]: { Api: "readable", PreviewDatabase: "readable" } }, + faults: [], + planetscale: { + [`test-${slug}`]: { production: false, deleteStatus }, + // Never touched: production, the index-lock branch, a name that starts with the stage's. + main: { production: true, deleteStatus: 500 }, + "idx-lock-38132096340-1": { production: true, deleteStatus: 500 }, + [`test-${slug}-copy`]: { production: false, deleteStatus: 500 }, + }, + requests: [], + }); + const planetscaleEnv = (bin: string) => ({ + PATH: `${bin}:${process.env.PATH ?? ""}`, + TEST_STAGE_DATABASE: "executor-test", + PLANETSCALE_ORGANIZATION: "synthetic-org", + PLANETSCALE_API_TOKEN_ID: "synthetic-id", + PLANETSCALE_API_TOKEN: "synthetic-token", + TEST_STAGE_BRANCH_POLL: "10 millis", + }); + /** A directory whose `alchemy` succeeds and does nothing, as a destroy that left the branch. */ + const quietAlchemy = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "executor-stage-branch-" }); + const bin = path.join(directory, "bin"); + yield* fs.makeDirectory(bin); + yield* fs.writeFileString(path.join(bin, "alchemy"), "#!/bin/sh\nexit 0\n"); + yield* fs.chmod(path.join(bin, "alchemy"), 0o755); + return bin; + }); + const planetscaleRequests = (world: World) => + world.requests.filter((request) => request.includes("/v1/organizations/")); + const leaseCount = (psql: (sql: string) => Effect.Effect, slug: string) => + psql(`select count(*) from executor_test_stage_leases where slug = '${slug}'`); + /** A temporary PlanetScale lease created at `createdAt`; it expires 3 hours later. */ + const planetscaleLease = (slug: string, createdAt: string) => + `insert into executor_test_stage_leases (slug, owner, created_at, expires_at, database_provider, retention, background) + values ('${slug}', 'CI', '${createdAt}', '${createdAt}'::timestamptz + interval '3 hours', 'planetscale', 'temporary', 'active')`; + + it.live( + "destroy deletes the left branch by its exact name, and only then removes the lease", + () => + Effect.gen(function* () { + const { url, psql } = yield* registry; + const world = leftWorld("e2e-left1", 204); + yield* prepareRegistry(world, url); + yield* psql(planetscaleLease("e2e-left1", "2099-01-01T00:00:00Z")); + const bin = yield* quietAlchemy; + const result = yield* cli(world, ["destroy", "e2e-left1", "--no-input", "--yes"], { + registry: url, + env: planetscaleEnv(bin), + }); + expect(result.code, result.output).toBe(0); + expect(result.output).toContain( + "PlanetScale branch test-e2e-left1 was left by the destroy; a delete was sent (PlanetScale answered 204) and it is gone (404).", + ); + const branch = + "/v1/organizations/synthetic-org/databases/executor-test/branches/test-e2e-left1"; + expect( + planetscaleRequests(world).filter((request) => request.startsWith("DELETE ")), + ).toEqual([`DELETE ${branch}`]); + expect(Object.keys(world.planetscale!).toSorted()).toEqual([ + "idx-lock-38132096340-1", + "main", + "test-e2e-left1-copy", + ]); + expect(yield* leaseCount(psql, "e2e-left1")).toBe("0\n"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + 180_000, + ); + + it.live( + "destroy keeps the lease when the delete is refused, and says so", + () => + Effect.gen(function* () { + const { url, psql } = yield* registry; + const world = leftWorld("e2e-left2", 403); + yield* prepareRegistry(world, url); + yield* psql(planetscaleLease("e2e-left2", "2099-01-01T00:00:00Z")); + const bin = yield* quietAlchemy; + const result = yield* cli(world, ["destroy", "e2e-left2", "--no-input", "--yes"], { + registry: url, + env: planetscaleEnv(bin), + }); + expect(result.code, result.output).not.toBe(0); + expect(result.output).toContain( + "PlanetScale branch test-e2e-left2 was left by the destroy. A delete was sent and PlanetScale refused it (403). Its stage lease is kept", + ); + expect(yield* leaseCount(psql, "e2e-left2")).toBe("1\n"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + 180_000, + ); + + it.live( + "scheduled cleanup retries a due lease's left branch and keeps the lease it could not finish", + () => + Effect.gen(function* () { + const { url, psql } = yield* registry; + const world = leftWorld("e2e-left3", 403); + yield* prepareRegistry(world, url); + // Past its cleanup deadline. + yield* psql(planetscaleLease("e2e-left3", "2026-11-16T01:00:00Z")); + const bin = yield* quietAlchemy; + const result = yield* cli(world, ["cleanup"], { registry: url, env: planetscaleEnv(bin) }); + expect(result.code, result.output).not.toBe(0); + expect(result.output).toContain( + "Cleanup failed for e2e-left3; its lease remains for retry. PlanetScale branch test-e2e-left3 was left by the destroy. A delete was sent and PlanetScale refused it (403).", + ); + expect( + planetscaleRequests(world).filter((request) => request.startsWith("DELETE ")), + ).toEqual([ + "DELETE /v1/organizations/synthetic-org/databases/executor-test/branches/test-e2e-left3", + ]); + expect(yield* leaseCount(psql, "e2e-left3")).toBe("1\n"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + 180_000, + ); +}); diff --git a/e2e/tests/tool-call-batch.spec.ts b/e2e/tests/tool-call-batch.spec.ts index c3ba47befe..ab50c07623 100644 --- a/e2e/tests/tool-call-batch.spec.ts +++ b/e2e/tests/tool-call-batch.spec.ts @@ -251,7 +251,7 @@ layer(HostedLive, { excludeTestServices: true })("Tool call batch", (it) => { const trace = yield* traceOf(request.traceId); return { execution, trace }; }); - /** The last MCP tool call of a trace, once its whole trace has reached Motel. */ + /** The last MCP tool call of a trace, once its whole trace has reached the collector. */ const traceOf = (traceId: string) => telemetry.query(traceId).pipe( Effect.flatMap((result) => { @@ -263,7 +263,9 @@ layer(HostedLive, { excludeTestServices: true })("Tool call batch", (it) => { (entry) => entry.span.tags["http.response.status_code"] === "200", ); if (root === undefined || !complete) - return Effect.fail(new Error("The completed server trace must reach Motel")); + return Effect.fail( + new Error("The completed server trace must reach the collector"), + ); const byId = new Map(result.data.map(({ span }) => [span.spanId, span])); /** The span names between a span and the tool call, or undefined outside it. */ const path = (spanId: string | null) => { @@ -296,7 +298,7 @@ layer(HostedLive, { excludeTestServices: true })("Tool call batch", (it) => { // later than the call's own statements. return statements.length === 0 || executed.length === statements.length ? Effect.fail( - new Error("The tool call's statements and log write must reach Motel"), + new Error("The tool call's statements and log write must reach the collector"), ) : Effect.succeed({ batches: inside.filter(({ span }) => span.operationName === "sql.pipeline") diff --git a/e2e/tests/tool-call-log.spec.ts b/e2e/tests/tool-call-log.spec.ts index f563ce1b72..242ae23bb0 100644 --- a/e2e/tests/tool-call-log.spec.ts +++ b/e2e/tests/tool-call-log.spec.ts @@ -315,6 +315,9 @@ const mcpTool = ( return result.structuredContent; }); +/** The bodies of a trace's delivered log records. */ +const LogBodies = Schema.Struct({ data: Schema.Array(Schema.Struct({ body: Schema.String })) }); + /** The fields of an error other than the reference, which must not repeat it. */ const withoutReference = (error: object) => JSON.stringify(Object.fromEntries(Object.entries(error).filter(([key]) => key !== "reference"))); @@ -331,7 +334,9 @@ const slowWrite = ( ) => Effect.gen(function* () { const api = yield* Api, - actors = yield* Actors; + actors = yield* Actors, + evidence = yield* Evidence, + telemetry = yield* Telemetry; if (!/^[A-Za-z0-9_-]+$/.test(actors.organization.id)) return yield* Effect.die("Expected an organization ID"); // Every row this organization writes takes 1.5 s, three times a failed call's bound. @@ -352,7 +357,6 @@ const slowWrite = ( const agent = yield* api.session(); const call = (tool: string) => Effect.gen(function* () { - const started = performance.now(); const response = yield* api.request( agent, "POST", @@ -360,8 +364,41 @@ const slowWrite = ( { tool, input: { text: "x" } }, key.bearer, ); - return { response, ms: performance.now() - started }; + const request = (yield* evidence.requests).at(-1); + if (request === undefined) + return yield* Effect.die(new Error("The call request is missing")); + return { response, traceId: request.traceId }; }); + /** + * When the call's answer was sent and when its log row was written, both on the server's own + * clock and from the call's own trace: the end of its `http.server` span and of its + * `product.tool_call_log.write` span. Their order says whether the call waited for its row, + * however slow the runner or the stage; no time is read in the test. + */ + const answeredAndWritten = (traceId: string, label: string) => + telemetry + .settled( + traceId, + (spans) => + spans.some((span) => span.operationName.startsWith("http.server ")) && + spans.some((span) => span.operationName === "product.tool_call_log.write"), + ) + .pipe( + Effect.tap((spans) => evidence.json(`${label}-spans.json`, spans)), + Effect.map((spans) => { + const end = (name: (operation: string) => boolean) => { + const span = spans.find((candidate) => name(candidate.operationName)); + return span === undefined ? Number.NaN : Date.parse(span.startTime) + span.durationMs; + }; + const answered = end((name) => name.startsWith("http.server ")); + const written = end((name) => name === "product.tool_call_log.write"); + return { + answered, + written, + order: `answered ${answered - written} ms after its row was written`, + }; + }), + ); // The app's first use starts its Worker, which can take longer than the bound on a busy // runner. Listing its tools, which writes no row, starts it before the calls are timed. @@ -371,7 +408,11 @@ const slowWrite = ( // The call that succeeded answered before its row was written; the row follows. const succeeded = yield* call("echo"); expect(succeeded.response.status).toBe(200); - expect(succeeded.ms, "a successful call does not wait for its log write").toBeLessThan(1000); + const success = yield* answeredAndWritten(succeeded.traceId, "succeeded-call"); + expect( + success.answered, + `a successful call does not wait for its log write: it ${success.order}`, + ).toBeLessThan(success.written); const logged = yield* readLogWhen( actors.owner, { app: app.id, range: "24h" }, @@ -384,16 +425,37 @@ const slowWrite = ( const failed = yield* call("explode"); const failure = yield* body(Failure, failed.response); expect(failure._tag).toBe("ToolCallFailed"); + const failedOrder = yield* answeredAndWritten(failed.traceId, "failed-call"); if (cloud) { - // A failed call waits for its row at most its bound, then answers without a reference. + // A failed call waits for its row at most its bound, then answers without a reference: it + // answered before the row was written, and the wait ended where the product bounds it. expect(failure.reference).toBeUndefined(); - expect(failed.ms, "a failed call waits at most its bound").toBeLessThan(1500); + expect( + failedOrder.answered, + `a failed call waits at most its bound, not for its row: it ${failedOrder.order}`, + ).toBeLessThan(failedOrder.written); + const bounded = yield* telemetry.logs(failed.traceId).pipe( + Effect.map((logs) => + Schema.decodeUnknownSync(Schema.fromJsonString(LogBodies))(logs).data.some( + (record) => record.body === "The tool call log write outlasted its caller's wait", + ), + ), + Effect.filterOrFail( + (found) => found, + () => new Error("The failed call's bounded wait has not reached the collector"), + ), + Effect.retry({ schedule: Schedule.spaced("500 millis"), times: 40 }), + ); + expect(bounded, "the failed call's wait for its row was cut off by its bound").toBe(true); } else { // Self-host's database runs each statement on the server's own thread, so nothing can // stop waiting for it: the failed call answers once its row is written, with a reference // that leads to it. expect(failure.reference).toMatch(reference); - expect(failed.ms).toBeGreaterThanOrEqual(1500); + expect( + failedOrder.answered, + `a failed call on self-host answers once its row is written: it ${failedOrder.order}`, + ).toBeGreaterThanOrEqual(failedOrder.written); expect(yield* findCall(actors.owner, failure.reference ?? "")).toMatchObject({ tool: "explode", status: "failed", diff --git a/e2e/tests/tool-call-read-ahead.spec.ts b/e2e/tests/tool-call-read-ahead.spec.ts index 000279baf2..349140e43e 100644 --- a/e2e/tests/tool-call-read-ahead.spec.ts +++ b/e2e/tests/tool-call-read-ahead.spec.ts @@ -265,13 +265,15 @@ layer(HostedLive, { excludeTestServices: true })("Tool call read ahead", (it) => span.operationName.startsWith("http.server") && span.tags["url.path"] === "/mcp", ); if (!served) - return Effect.fail(new Error("The MCP request's server span must reach Motel")); + return Effect.fail( + new Error("The MCP request's server span must reach the collector"), + ); const root = result.data .filter(({ span }) => span.operationName === "mcp.tool.call") .toSorted((a, b) => Date.parse(a.span.startTime) - Date.parse(b.span.startTime)) .at(-1); if (root === undefined) - return Effect.fail(new Error("The tool call must reach Motel")); + return Effect.fail(new Error("The tool call must reach the collector")); const found: Statement[] = []; for (const { span } of result.data) { if (span.operationName !== "sql.execute" && span.operationName !== "sql.pipeline") @@ -307,7 +309,7 @@ layer(HostedLive, { excludeTestServices: true })("Tool call read ahead", (it) => } return found.some(isPatCheck) ? Effect.succeed(found.toSorted((a, b) => a.start - b.start)) - : Effect.fail(new Error("The tool call's statements must reach Motel")); + : Effect.fail(new Error("The tool call's statements must reach the collector")); }), Effect.retry({ schedule: Schedule.spaced("250 millis"), times: 80 }), ); diff --git a/e2e/tests/tool-call-statements.spec.ts b/e2e/tests/tool-call-statements.spec.ts index 72ab79c0a7..3f1e177ec0 100644 --- a/e2e/tests/tool-call-statements.spec.ts +++ b/e2e/tests/tool-call-statements.spec.ts @@ -200,7 +200,9 @@ layer(HostedLive, { excludeTestServices: true })("Tool call statements", (it) => (entry) => entry.span.tags["http.response.status_code"] === "200", ); if (root === undefined || !complete) - return Effect.fail(new Error("The completed server trace must reach Motel")); + return Effect.fail( + new Error("The completed server trace must reach the collector"), + ); const byId = new Map(result.data.map(({ span }) => [span.spanId, span])); const path = (spanId: string | null) => { const names: string[] = []; @@ -239,7 +241,9 @@ layer(HostedLive, { excludeTestServices: true })("Tool call statements", (it) => path(span.parentSpanId) !== undefined, ); return reads.length === 0 || write === undefined || logWrites.length === 0 - ? Effect.fail(new Error("SQL descendants and the log's write must reach Motel")) + ? Effect.fail( + new Error("SQL descendants and the log's write must reach the collector"), + ) : Effect.succeed({ reads, logWrites, diff --git a/e2e/tests/tools-error-state.spec.ts b/e2e/tests/tools-error-state.spec.ts index b32da655d9..0a6d0b6aef 100644 --- a/e2e/tests/tools-error-state.spec.ts +++ b/e2e/tests/tools-error-state.spec.ts @@ -187,6 +187,60 @@ export default defineApp({}, async () => ({ page.getByRole("alert").waitFor({ state: "hidden" }), ); yield* browser.checkpoint("Tools-error-recovered"); + + // A Worker failure whose text matches a resource limit, written by the app's own code: one + // retry, then stop retrying unchanged and investigate, rebuilding only an app on an older + // apps version. Executor has not confirmed the limit, so nothing says a repeat proves it. + const limited = yield* api.request(actors.owner, "POST", `${prefix}/apps/deploy`, { + name: "Sample limited tools", + files: [ + { + path: "index.ts", + content: `import { defineApp, query, object, router } from "apps"; +const json = Response.prototype.json; +export default defineApp({}, async () => { + Response.prototype.json = function () { + Response.prototype.json = json; + return Promise.reject(new Error("Worker exceeded CPU time limit.")); + }; + return { tools: router({ ping: query({ input: object({}) }, async () => "pong") }) }; +});`, + }, + appsManifest, + ], + }); + expect(limited.status).toBe(200); + const limitedApp = yield* body(App, limited); + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", `${prefix}/apps/${limitedApp.id}`).pipe(Effect.orDie), + ); + yield* openThroughBrowser( + "Open the Tools page of an app stopped at a matched limit", + `/org/${actors.organization.slug}/apps/${limitedApp.id}?view=tools`, + ); + yield* browser.use("The limit's recovery is shown", (page) => + page.getByRole("alert", { name: title, exact: true }).waitFor(), + ); + const limitCopy = yield* browser.use("Read the limit's error state", (page) => + page.getByRole("alert", { name: title, exact: true }).innerText(), + ); + expect(limitCopy).toContain( + "Its error matches this cause, which Executor has not confirmed", + ); + expect(limitCopy).toContain( + "Try again once. If it fails the same way, stop retrying it unchanged and investigate. If the app was built on an older apps version than this host’s, rebuild it on this host’s version and deploy it again", + ); + expect(limitCopy).not.toContain("will not help"); + yield* browser.checkpoint("Tools-error-matched-limit"); + const limitPrompt = yield* browser.use("Copy the limit's fix prompt", (page) => + page + .getByRole("button", { name: "Copy fix prompt", exact: true }) + .click() + .then(() => page.evaluate(() => navigator.clipboard.readText())), + ); + expect(limitPrompt).toContain("Retry the unchanged request at most once."); + expect(limitPrompt).toContain("If the app's version is older"); + expect(limitPrompt).not.toContain("not transient"); }), ), ); diff --git a/e2e/tests/tools-index-cache.spec.ts b/e2e/tests/tools-index-cache.spec.ts index 20074172be..a1edbf065a 100644 --- a/e2e/tests/tools-index-cache.spec.ts +++ b/e2e/tests/tools-index-cache.spec.ts @@ -90,15 +90,20 @@ const toolDetail = (actor: Session, app: string, name: string) => return request; }); -/** How a request read the tool listing, and whether it invoked the app, from its trace. */ -const listingRead = (traceId: string, label: string) => +/** + * How a request read the tool listing, and whether it invoked the app, from its trace. A trace + * arrives in pieces: the app's spans (`app.dispatch`) are exported by the app's Worker, apart from + * the host's. So the read waits until every span in `reads` has arrived, as `Telemetry.settled` + * asks: name `app.dispatch` there when the check expects the app to have been invoked. + */ +const listingRead = (traceId: string, label: string, reads: ReadonlyArray) => Effect.gen(function* () { const evidence = yield* Evidence, telemetry = yield* Telemetry; const spans = yield* telemetry.query(traceId).pipe( Effect.map((result) => result.data.map((row) => row.span)), Effect.flatMap((spans) => - spans.some((span) => span.operationName === "sdk.tools.listing") + reads.every((name) => spans.some((span) => span.operationName === name)) ? Effect.succeed(spans) : Effect.fail(new Error(`The ${label} trace has not arrived`)), ), @@ -130,7 +135,9 @@ layer(HostedLive, { excludeTestServices: true })("Tools index cache", (it) => { const kept = yield* indexTools(actors.owner, app.id, "kept-index"); expect(cold.names.length).toBeGreaterThan(0); expect(kept.names).toEqual(cold.names); - const keptIndex = yield* listingRead(kept.request.traceId, "kept-index"); + const keptIndex = yield* listingRead(kept.request.traceId, "kept-index", [ + "sdk.tools.listing", + ]); expect(keptIndex.cache).toBe("hit"); expect(keptIndex.dispatched).toBe(false); @@ -172,7 +179,10 @@ layer(HostedLive, { excludeTestServices: true })("Tools index cache", (it) => { expect(yield* linkedAccountIds(actors.owner, appPath)).toEqual([secondKey.id]); const revised = yield* indexTools(actors.owner, app.id, "revised-index"); expect(revised.names).toEqual(cold.names); - const revisedListing = yield* listingRead(revised.request.traceId, "revised-listing"); + const revisedListing = yield* listingRead(revised.request.traceId, "revised-listing", [ + "sdk.tools.listing", + "app.dispatch", + ]); expect(revisedListing.dispatched).toBe(true); const [first, second, third] = yield* Effect.all( @@ -203,7 +213,10 @@ layer(HostedLive, { excludeTestServices: true })("Tools index cache", (it) => { yield* linkAccount(actors.owner, appPath, "service", account); expect(yield* linkedAccountIds(actors.owner, appPath)).toEqual([account]); const tool = yield* toolDetail(actors.owner, app.id, cold.names[0] ?? ""); - const keptTool = yield* listingRead(tool.traceId, "kept-tool"); + const keptTool = yield* listingRead(tool.traceId, "kept-tool", [ + "sdk.tools.listing", + "sdk.tools.get", + ]); expect(keptTool.cache).toBe("hit"); expect(keptTool.source).toBe("listing"); expect(keptTool.dispatched).toBe(false); diff --git a/e2e/tests/webhook-source-order.spec.ts b/e2e/tests/webhook-source-order.spec.ts new file mode 100644 index 0000000000..7c800d8fd1 --- /dev/null +++ b/e2e/tests/webhook-source-order.spec.ts @@ -0,0 +1,207 @@ +/** + * Pending webhook sources are served first in, first out: a source whose intent is saved again, by + * linking it again or configuring it, is served in its turn while new sources keep arriving; sources + * returned to the queue by a failure outside their attempt, or linked again before every sweep, + * keep no newer source waiting. And the lease is untouched: a source linked again while a sweep + * holds its lease is neither claimed by a second sweep nor forgotten, and two sweeps that both read + * the same pending source serve it once. The pending query keeps its order from its index. + * + * The cases run in `packages/sdk/scripts/webhook-sources-fixture.ts`, the SDK's real webhook-source + * reconciliation on PGlite in memory, with apps that have no deployment so that serving a source + * needs no external call. Each case has a database of its own, so its queue holds only the sources + * it created, whatever runs beside the scenario; all run in one process, started once. The cases + * count sweeps and read stored state; none depends on timing. + * `app-account-webhooks.spec.ts` checks the same through the deployed product. + */ +import { expect, layer } from "@effect/vitest"; +import { Effect, Schema } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { scenarios } from "../test-plan.ts"; +import { TestLive, withCase } from "../support/case.ts"; +import { Evidence } from "../support/evidence.ts"; + +const Starvation = Schema.Struct({ + before: Schema.optional(Schema.String), + pendingAfterSave: Schema.optional(Schema.String), + servedInRound: Schema.NullOr(Schema.Number), + rounds: Schema.Number, + limit: Schema.Number, +}); +const Fairness = Schema.Struct({ + servedInSweep: Schema.NullOr(Schema.Number), + rounds: Schema.Number, + limit: Schema.Number, + maxOldClaims: Schema.optional(Schema.Number), +}); +const SetupCompleted = Schema.Struct({ + pendingAfterSetup: Schema.String, + servedInRound: Schema.NullOr(Schema.Number), + rounds: Schema.Number, + limit: Schema.Number, +}); +const WaitingCheck = Schema.Struct({ + waitingChecks: Schema.Number, + status: Schema.String, + retries: Schema.Number, + pushedAhead: Schema.Boolean, +}); +const Race = Schema.Struct({ readsBeforeClaims: Schema.Number, claims: Schema.Number }); +const PlanSummary = Schema.Struct({ + nodes: Schema.Array(Schema.String), + index: Schema.NullOr(Schema.String), + rowsRead: Schema.Number, + sorted: Schema.Boolean, +}); +const Plan = Schema.Struct({ + pending: Schema.Number, + leaseUntil: PlanSummary, + createdAt: PlanSummary, +}); +const Lease = Schema.Struct({ + claimsWhileHeld: Schema.Number, + afterFirst: Schema.optional(Schema.String), + claimsAfter: Schema.Number, + finalStatus: Schema.optional(Schema.String), +}); + +const Cases = Schema.Struct({ + "relink-while-new-arrive": Starvation, + "configure-while-new-arrive": Starvation, + "relink-during-lease": Lease, + "outside-failures-then-new": Fairness, + "relinks-before-each-sweep-then-new": Fairness, + "two-sweeps-race": Race, + "intent-plan": Plan, + "setup-completed-while-new-arrive": SetupCompleted, + "waiting-push-after-setup": WaitingCheck, +}); + +/** Every case of the probe, each on a database of its own that it opened empty, in one process. */ +const probe = Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner, + evidence = yield* Evidence; + const output = yield* processes.string( + ChildProcess.make("node", ["packages/sdk/scripts/webhook-sources-fixture.ts", "--all"], { + env: { PATH: process.env.PATH ?? "", HOME: process.env.HOME ?? "" }, + extendEnv: false, + }), + ); + const run = yield* Schema.decodeUnknownEffect( + Schema.fromJsonString( + Schema.Struct({ + rowsAtOpen: Schema.Record(Schema.String, Schema.Number), + cases: Schema.Unknown, + }), + ), + )(output.trim().split("\n").at(-1) ?? ""); + yield* evidence.json("webhook-source-rows-at-open.json", run.rowsAtOpen); + // Every case opened a database of its own, holding no row of another case. + expect(run.rowsAtOpen).toEqual( + Object.fromEntries(Object.keys(Cases.fields).map((name) => [name, 0])), + ); + const results = yield* Schema.decodeUnknownEffect(Cases)(run.cases); + for (const [name, result] of Object.entries(results)) + yield* evidence.json(`webhook-source-${name}.json`, result); + return results; +}); + +layer(TestLive, { excludeTestServices: true })("Webhook source order", (it) => { + it.effect(scenarios.webhookSourceOrder.title, (context) => + withCase( + context, + Effect.gen(function* () { + const evidence = yield* Evidence; + const results = yield* probe; + + yield* evidence.step( + "A source linked again, or configured again, is served in its turn while new sources keep arriving", + Effect.sync(() => { + for (const name of ["relink-while-new-arrive", "configure-while-new-arrive"] as const) { + const result = results[name]; + // Served once, its intent saved again: pending. + expect(result, name).toMatchObject({ + before: "needs-setup", + pendingAfterSave: "pending", + }); + // Every sweep also finds a full sweep's worth of new sources; the older source is + // served in the first of them, not left behind. + expect(result.servedInRound, name).toBe(1); + } + }), + ); + + yield* evidence.step( + "A source linked again while a sweep holds its lease is not claimed by a second sweep, and is served again afterwards", + Effect.sync(() => { + expect(results["relink-during-lease"]).toEqual({ + // Only the sweep holding the lease reconciled it. + claimsWhileHeld: 1, + // That sweep saw the new link and left the source pending. + afterFirst: "pending", + // The next sweep served it again. + claimsAfter: 2, + finalStatus: "needs-setup", + }); + }), + ); + + yield* evidence.step( + "Sources returned to the queue by a failure outside their attempt, or linked again before every sweep, keep no newer source waiting", + Effect.sync(() => { + // Eight sources fail outside their attempt at every run; a sweep serves four. They go + // to the back of the queue, so the source that joined after them is served by the + // third sweep, and none of them runs more than twice meanwhile. + const failing = results["outside-failures-then-new"]; + expect(failing.servedInSweep).not.toBeNull(); + expect(failing.servedInSweep).toBeLessThanOrEqual(3); + expect(failing.maxOldClaims).toBeLessThanOrEqual(2); + // Four sources linked again before every sweep join after the new one. + expect(results["relinks-before-each-sweep-then-new"].servedInSweep).toBe(1); + }), + ); + + yield* evidence.step( + "A source whose setup a person completes joins the queue then, though it held a retry time an hour ahead, and the waiting check does not push it back", + Effect.sync(() => { + const setup = results["setup-completed-while-new-arrive"]; + expect(setup.pendingAfterSetup).toBe("pending"); + // Every sweep also finds a full sweep's worth of new sources. + expect(setup.servedInRound).toBe(1); + // The setup is completed while the waiting check of the same source runs, between its + // read and its conditional write; that write must not push the source a delay ahead. + expect(results["waiting-push-after-setup"]).toEqual({ + waitingChecks: 1, + status: "pending", + retries: 0, + pushedAhead: false, + }); + }), + ); + + yield* evidence.step( + "Two sweeps that both read the same pending source before either claims it: exactly one serves it", + Effect.sync(() => { + expect(results["two-sweeps-race"]).toEqual({ + // Each sweep's two queue reads came before either claimed. + readsBeforeClaims: 4, + claims: 1, + }); + }), + ); + + yield* evidence.step( + "The pending query takes its order from the existing partial index, reading only what it serves", + Effect.sync(() => { + const plan = results["intent-plan"]; + expect(plan.leaseUntil).toMatchObject({ + index: "executor_webhook_sources_intent", + sorted: false, + }); + // A sweep's worth of rows out of 2,000 pending. + expect(plan.leaseUntil.rowsRead).toBeLessThanOrEqual(4); + }), + ); + }), + ), + ); +}); diff --git a/e2e/tests/webhook-sweep-waits.spec.ts b/e2e/tests/webhook-sweep-waits.spec.ts new file mode 100644 index 0000000000..d26956b8ef --- /dev/null +++ b/e2e/tests/webhook-sweep-waits.spec.ts @@ -0,0 +1,540 @@ +/** + * What the webhook waits conclude from the sweep's passes, and the deployed preparation's wait for + * its start-up queue. The waits run their real loop against a simulated product, sweeper and + * collector, on a clock the case controls: sleeping moves it on at once, so a case about a 60 s + * bound takes milliseconds and its times are exact. + */ +import { expect, it } from "@effect/vitest"; +import { Cause, Clock, Duration, Effect, Exit } from "effect"; +import { attachedStage, axiomQueryMs } from "../support/axiom.ts"; +import { collectorReadMs } from "../support/evidence.ts"; +import { + deployedPace as deployedSweepPace, + localPace, + managedCloudPace, + sweepPaceFor, +} from "../support/webhook-sources.ts"; +import { + drainBound, + type DrainPace, + deployedPace as deployedDrainPace, + drainWebhookQueue, + type QueueDrain, +} from "../sdk/webhook-queue.ts"; +import { + type Delivered, + type PendingRead, + skippingPass, + type SweepPace, + waitOnSweep, +} from "../support/webhook-sweep.ts"; + +const t0 = Date.parse("2026-11-16T12:00:00.000Z"); +const trace = (name: string) => Buffer.from(name).toString("hex").padEnd(32, "0").slice(0, 32); + +/** A clock the case controls: sleeping moves it on by the time slept, at once. */ +const virtualClock = () => { + let now = t0; + const nanos = () => BigInt(now) * 1_000_000n; + const clock: Clock.Clock = { + currentTimeMillisUnsafe: () => now, + currentTimeMillis: Effect.sync(() => now), + currentTimeNanosUnsafe: nanos, + currentTimeNanos: Effect.sync(nanos), + monotonicTimeNanosUnsafe: nanos, + monotonicTimeNanos: Effect.sync(nanos), + sleep: (duration) => + Effect.sync(() => { + now += Duration.toMillis(duration); + }), + }; + return { clock, now: () => now }; +}; + +const span = ( + name: string, + operation: string, + start: number, + durationMs: number, + tags: Record, +): Delivered => ({ + traceId: trace(name), + span: { + spanId: trace(name).slice(0, 16), + parentSpanId: null, + operationName: operation, + serviceName: "executor", + startTime: new Date(start).toISOString(), + durationMs, + status: "ok", + events: [], + tags, + }, +}); +/** A pass of the sweeper on server `instance` (none on Cloud). */ +const pass = (name: string, start: number, taken: number, instance?: string, durationMs = 200) => + span(name, "sdk.webhooks.sweep", start, durationMs, { + "executor.webhooks.sweep.taken": String(taken), + "executor.webhooks.sweep.limit": "8", + ...(instance === undefined ? {} : { "executor.webhooks.instance": instance }), + }); +/** A reconciliation of the source in pass `name`'s trace. */ +const reconcile = (name: string, start: number, durationMs = 100) => + span(name, "sdk.webhooks.reconcileSource", start, durationMs, {}); +const read = (start: number, end: number, instance?: string): PendingRead => ({ + start, + end, + instance, +}); + +it("a pass of the read's own server with room after a pending read, that reconciled nothing of the source, skipped it", () => { + const reads = [read(t0, t0 + 10, "a")]; + const judge = (passes: Delivered[], served: Delivered[] = []) => + skippingPass({ + passes, + served, + reads, + servedReadAt: t0 + 60_000, + deliveryMs: 5_000, + servers: "shared", + }); + const room = pass("room", t0 + 1_000, 3, "a"); + expect(judge([pass("full", t0 + 500, 8, "a"), room])).toBe(room); + // Full passes leave it queued; a pass from before the read proves nothing. + expect(judge([pass("full", t0 + 500, 8, "a"), pass("early", t0 - 1_000, 0, "a")])).toBe( + undefined, + ); + // A pass that reconciled it took it. + expect(judge([room], [reconcile("room", t0 + 1_050)])).toBe(undefined); + // Another server's pass, sharing the collector, says nothing of this server's queue. + expect(judge([pass("other", t0 + 1_000, 0, "b")])).toBe(undefined); + // A pass is judged only once every reconciliation before it has surely been delivered. + expect( + skippingPass({ + passes: [room], + served: [], + reads, + servedReadAt: t0 + 1_200 + 4_999, + deliveryMs: 5_000, + servers: "shared", + }), + ).toBe(undefined); +}); + +it("where each collector holds one coordinator's passes, a pass with room skips the source though another Worker answered the read", () => { + // On Cloud the read is answered by the API Worker and the pass run by the coordinator, each with + // its own webhook source service. + const room = pass("room", t0 + 1_000, 0, "coordinator"); + const judge = (servers: "own" | "shared") => + skippingPass({ + passes: [room], + served: [], + reads: [read(t0, t0 + 10, "api")], + servedReadAt: t0 + 60_000, + deliveryMs: 5_000, + servers, + }); + expect(judge("own")).toBe(room); + expect(judge("shared")).toBe(undefined); +}); + +it("a source served and queued again between two reads that showed it pending was not skipped by the pass between", () => { + // Reads at 0 s and 1 s show it pending. A pass at 1.05 s serves it (ready at 1.2 s); a pass with + // room at 1.22 s finds it ready; it is queued again at 1.24 s, before the next read. + const reads = [read(t0, t0 + 10), read(t0 + 1_000, t0 + 1_010)]; + const judged = skippingPass({ + passes: [pass("served", t0 + 1_050, 1), pass("between", t0 + 1_220, 0)], + served: [reconcile("served", t0 + 1_100, 100)], + reads, + servedReadAt: t0 + 60_000, + deliveryMs: 5_000, + servers: "own", + }); + expect(judged).toBe(undefined); + // A read after that reconciliation, then a pass with room that does not take it: skipped. + const later = pass("later", t0 + 2_000, 0); + expect( + skippingPass({ + passes: [pass("served", t0 + 1_050, 1), pass("between", t0 + 1_220, 0), later], + served: [reconcile("served", t0 + 1_100, 100)], + reads: [...reads, read(t0 + 1_500, t0 + 1_510)], + servedReadAt: t0 + 60_000, + deliveryMs: 5_000, + servers: "own", + }), + ).toBe(later); +}); + +const pace: SweepPace = { + readSpacingMs: 250, + spanSpacingMs: 2_000, + deliveryMs: 5_000, + queryMs: 5_000, + reportMarginMs: 3_000, + servers: "shared", +}; + +/** + * One app with one pending source, as the product, its sweeper and the collector show it over + * time: `readyAt` is when the product reads it ready (never, if undefined), `passes` and `served` + * what the sweeper did, each readable once it has ended. `spans: "down"` fails every span read. + */ +const world = (input: { + readonly readyAt?: number; + readonly passes?: ReadonlyArray; + readonly served?: ReadonlyArray; + readonly instance?: string; + /** "down" fails every span read; "slow" makes each take the reader's whole time-out. */ + readonly spans?: "down" | "slow"; +}) => { + const { clock, now } = virtualClock(); + const reads = new Map(); + let count = 0; + const delivered = (all: ReadonlyArray = []) => + Effect.sync(() => + all.filter(({ span }) => Date.parse(span.startTime) + span.durationMs <= now()), + ); + const spans =
(effect: Effect.Effect): Effect.Effect => + input.spans === "down" + ? Effect.fail(new Error("collector unreachable")) + : input.spans === "slow" + ? Effect.sleep(pace.queryMs).pipe(Effect.andThen(effect)) + : effect; + const wait = waitOnSweep({ + what: "Webhook sources", + deadline: t0 + 60_000, + pace, + reads: { + attempt: Effect.sync(() => { + const ready = input.readyAt !== undefined && now() >= input.readyAt; + return { done: ready, value: now(), shown: JSON.stringify({ ready }) }; + }), + pending: Effect.sync(() => { + const name = `read-${count++}`; + reads.set( + trace(name), + span( + name, + "sdk.webhooks.sources", + now(), + 10, + input.instance === undefined ? {} : { "executor.webhooks.instance": input.instance }, + ), + ); + return { + accounts: [{ account: "acc_team", label: "team-key" }], + shown: [{ account: "acc_team", status: "pending" }], + trace: trace(name), + }; + }), + passes: () => spans(delivered(input.passes)), + served: () => spans(delivered(input.served)), + readSpan: (id) => + spans(delivered([...reads.values()].filter((r) => r.traceId === id))).pipe( + Effect.map((found) => found[0]?.span), + ), + }, + }).pipe(Effect.provideService(Clock.Clock, clock), Effect.exit); + return { wait, now }; +}; +/** Full passes every second from `from` for `count` passes, on server `instance`. */ +const fullPasses = (from: number, count: number, instance?: string) => + Array.from({ length: count }, (_, n) => + pass(`full-${instance}-${n}`, from + n * 1_000, 8, instance), + ); +const failure = (exit: Exit.Exit) => + Exit.isFailure(exit) ? Cause.pretty(exit.cause) : ""; + +it.effect("a source behind full passes is waited for until it is served", () => + Effect.gen(function* () { + const { wait, now } = world({ + instance: "a", + readyAt: t0 + 40_300, + passes: [...fullPasses(t0 + 100, 40, "a"), pass("takes", t0 + 40_100, 1, "a")], + served: [reconcile("takes", t0 + 40_150)], + }); + const exit = yield* wait; + expect(Exit.isSuccess(exit)).toBe(true); + expect(now()).toBeGreaterThanOrEqual(t0 + 40_300); + }), +); + +it.effect( + "a pass with room that does not take a pending source fails the wait at once, naming it", + () => + Effect.gen(function* () { + const { wait, now } = world({ + instance: "a", + passes: [...fullPasses(t0 + 100, 5, "a"), pass("skips", t0 + 6_000, 0, "a")], + }); + const exit = yield* wait; + const text = failure(exit); + expect(text).toContain("the webhook source sweep skipped team-key"); + expect(text).toContain(`its pass at ${new Date(t0 + 6_000).toISOString()}`); + expect(text).toContain(trace("skips")); + expect(text).toContain("took 0 of its 8"); + // As soon as every reconciliation before the pass has surely been delivered, long before + // the scenario's bound. + expect(now()).toBeLessThan(t0 + 6_200 + pace.deliveryMs + pace.spanSpacingMs + 1_000); + }), +); + +it.effect("a pass with room from another server sharing the collector is not taken as a skip", () => + Effect.gen(function* () { + const { wait } = world({ + instance: "a", + readyAt: t0 + 30_300, + passes: [ + ...fullPasses(t0 + 100, 30, "a"), + ...fullPasses(t0 + 150, 30, "b").map((full, n) => + pass(`other-${n}`, Date.parse(full.span.startTime), 0, "b"), + ), + pass("takes", t0 + 30_100, 1, "a"), + ], + served: [reconcile("takes", t0 + 30_150)], + }); + expect(Exit.isSuccess(yield* wait)).toBe(true); + }), +); + +it.effect("with the collector unreadable, a source that is then served passes", () => + Effect.gen(function* () { + const { wait } = world({ instance: "a", readyAt: t0 + 40_000, spans: "down" }); + expect(Exit.isSuccess(yield* wait)).toBe(true); + }), +); + +it.effect( + "with the collector unreadable, a source never served fails at the bound, saying the passes could not be read", + () => + Effect.gen(function* () { + const { wait, now } = world({ instance: "a", spans: "down" }); + const text = failure(yield* wait); + expect(text).toContain("did not settle before the scenario's time-out"); + expect(text).toContain("could not be read"); + expect(now()).toBe(t0 + 60_000 - pace.reportMarginMs); + }), +); + +it.effect( + "a source served and queued again between reads is waited for, not reported skipped", + () => + Effect.gen(function* () { + // Served at 1.1 s by a pass with room; a pass with room at 1.22 s finds it ready; it is queued + // again at 1.24 s and stays pending in every read until a full pass and then one that takes + // it again, at 20 s. + const { wait } = world({ + instance: "a", + readyAt: t0 + 20_300, + passes: [ + pass("served", t0 + 1_050, 1, "a"), + pass("between", t0 + 1_220, 0, "a"), + ...fullPasses(t0 + 1_300, 18, "a"), + pass("again", t0 + 20_100, 1, "a"), + ], + served: [reconcile("served", t0 + 1_100, 100), reconcile("again", t0 + 20_150)], + }); + expect(Exit.isSuccess(yield* wait)).toBe(true); + }), +); + +/** A sweep 100 times the deployed rate, so the bound is small; the clock is the case's own. */ +const drainPace: DrainPace = { + rate: 233, + deliveryMs: 300, + readSpacingMs: 50, + provisioningMs: 1_000, + idleWakeMs: 650, +}; + +/** + * Drain a queue of 500 against the passes `passes` gives (each readable once it has started). + * The stage's provisioning has 3 jobs left until `provisioningFor` ms in. + */ +const drain = ( + passes: ReadonlyArray, + provisioningFor = 0, + queue: { readonly pending: number; readonly pace: DrainPace } = { pending: 500, pace: drainPace }, +) => + Effect.gen(function* () { + const { clock, now } = virtualClock(); + let countedAt: number | undefined; + let recorded: QueueDrain | undefined; + const exit = yield* drainWebhookQueue({ + count: Effect.sync(() => { + countedAt = now(); + return { + pending: queue.pending, + provisioning: now() - t0 < provisioningFor ? 3 : 0, + at: now(), + }; + }), + passes: () => + Effect.sync(() => ({ + data: passes.filter(({ span }) => Date.parse(span.startTime) <= now()), + })), + record: (figures) => + Effect.sync(() => { + recorded = figures; + }), + pace: queue.pace, + }).pipe(Effect.provideService(Clock.Clock, clock), Effect.exit); + return { exit, recorded, countedAt, finished: now() }; + }); + +it.effect( + "preparation waits for provisioning to queue its sources, then until a pass after its count has room", + () => + Effect.gen(function* () { + // A pass with room during provisioning says nothing of the sources it has yet to queue. + const result = yield* drain( + [ + pass("before", t0 - 1_000, 2), + pass("provisioning", t0 + 100, 0), + ...Array.from({ length: 30 }, (_, n) => + pass(`full-${n}`, t0 + 600 + n * 33, 8, undefined, 30), + ), + pass("drained", t0 + 1_700, 4, undefined, 30), + ], + 400, + ); + expect(Exit.isSuccess(result.exit)).toBe(true); + // Counted at the first read after provisioning finished, at 400 ms. + expect(result.countedAt).toBeDefined(); + expect(result.recorded).toEqual({ + provisioningMs: 400, + queued: 500, + boundMs: drainBound(500, drainPace), + passes: 31, + passesWithRoom: 1, + drainMs: 1_730 - 400, + drainedBy: { + startTime: new Date(t0 + 1_700).toISOString(), + traceId: trace("drained"), + taken: 4, + limit: 8, + }, + }); + expect(result.finished).toBe(t0 + 1_700); + }), +); + +it.effect( + "provisioning that has not finished by its bound is recorded, and the queue is counted then", + () => + Effect.gen(function* () { + const result = yield* drain([pass("drained", t0 + 1_200, 1)], Number.POSITIVE_INFINITY); + expect(Exit.isSuccess(result.exit)).toBe(true); + expect(result.recorded).toMatchObject({ + provisioningMs: drainPace.provisioningMs, + provisioningOutstanding: 3, + queued: 500, + }); + }), +); + +it.effect( + "with the sweep stopped, preparation fails at the bound derived from its count, with the figures", + () => + Effect.gen(function* () { + // At the deployed rate: (500 + 2 × 8) / 2.33 s, then the coordinator's idle wake (60 s and + // 5 s for a late trigger), then 30 s for the last pass's span. + expect(drainBound(500)).toBe(Math.ceil((516 / 2.33) * 1000) + 65_000 + 30_000); + const bound = drainBound(500, drainPace); + const result = yield* drain([]); + expect(failure(result.exit)).toContain("did not drain within its bound"); + // The first read at or after the bound, at the drain's own spacing. + expect(result.finished).toBe( + t0 + Math.ceil(bound / drainPace.readSpacingMs) * drainPace.readSpacingMs, + ); + expect(result.recorded).toEqual({ + provisioningMs: 0, + queued: 500, + boundMs: bound, + passes: 0, + passesWithRoom: 0, + pendingAtBound: 500, + }); + }), +); + +it.effect("a sweep that only takes full passes fails the bound too, saying none had room", () => + Effect.gen(function* () { + const result = yield* drain( + Array.from({ length: 200 }, (_, n) => pass(`full-${n}`, t0 + 10 + n * 33, 8, undefined, 30)), + ); + expect(failure(result.exit)).toContain("did not drain within its bound"); + expect(result.recorded).toMatchObject({ queued: 500, passesWithRoom: 0 }); + expect(result.recorded?.passes).toBeGreaterThan(30); + }), +); + +it.effect("on a quiet stage, a drain with nothing queued finishes at once", () => + Effect.gen(function* () { + const result = yield* drain([], 0, { pending: 0, pace: deployedDrainPace }); + expect(Exit.isSuccess(result.exit)).toBe(true); + expect(result.finished).toBe(t0); + expect(result.recorded).toMatchObject({ queued: 0, drainMs: 0, passes: 0 }); + }), +); + +it.effect( + "a first pass 59 s after the count, on the idle coordinator's next wake, still drains a short queue", + () => + Effect.gen(function* () { + // The pass that took the last sources was already running at the count; the next comes on + // the coordinator's minute heartbeat. + const result = yield* drain([pass("running", t0 - 100, 8), pass("wake", t0 + 59_000, 0)], 0, { + pending: 3, + pace: deployedDrainPace, + }); + expect(Exit.isSuccess(result.exit)).toBe(true); + expect(result.recorded).toMatchObject({ queued: 3, drainedBy: { traceId: trace("wake") } }); + }), +); + +it.effect("no span read starts that could run past the report, however slow the reads", () => + Effect.gen(function* () { + // Every read takes the reader's whole time-out, and another server's passes with room keep + // the wait reading reconciliations and read spans too. + const { wait, now } = world({ + instance: "a", + spans: "slow", + passes: Array.from({ length: 60 }, (_, n) => + pass(`other-${n}`, t0 + 500 + n * 1_000, 0, "b"), + ), + }); + const text = failure(yield* wait); + expect(text).toContain("did not settle before the scenario's time-out"); + expect(now()).toBeGreaterThanOrEqual(t0 + 60_000 - pace.reportMarginMs); + expect(now()).toBeLessThan(t0 + 60_000); + }), +); + +it.effect("an attached stage's origin must name its stage, once, before any scenario", () => + Effect.gen(function* () { + expect(yield* attachedStage("https://e2e-abc123.executor-test.example")).toBe( + "test-e2e-abc123", + ); + const exit = yield* Effect.exit(attachedStage("https://Not_A_Stage.example")); + expect(failure(exit)).toContain("does not name its stage"); + }), +); + +it("the waits' paces follow the readers, Axiom's delay and its quota, and each target's collectors", () => { + // No span read may outlast the report: each pace's query time is its reader's own time-out. + expect(deployedSweepPace.queryMs).toBe(axiomQueryMs); + expect(localPace.queryMs).toBe(collectorReadMs); + // A skip is judged only after the slowest delivery measured on a test stage, 18 s. + expect(deployedSweepPace.deliveryMs).toBeGreaterThanOrEqual(18_000); + // 16 waiting workers read passes within a tenth of the organization's 1,800 queries a minute. + expect(16 * (60_000 / deployedSweepPace.spanSpacingMs)).toBeLessThanOrEqual(180); + // Each Cloud environment has its own collector, and a deployed stage is read by its name: any + // pass there is the read's own coordinator's. Self-hosted and local servers may share one. + expect(sweepPaceFor({ target: "cloud", mode: "attached" })).toBe(deployedSweepPace); + expect(sweepPaceFor({ target: "cloud", mode: "managed" })).toBe(managedCloudPace); + expect(managedCloudPace.servers).toBe("own"); + expect(deployedSweepPace.servers).toBe("own"); + expect(sweepPaceFor({ target: "self-host", mode: "managed" }).servers).toBe("shared"); + expect(sweepPaceFor({ target: "local", mode: "managed" }).servers).toBe("shared"); +}); diff --git a/e2e/tests/worker-sizes.spec.ts b/e2e/tests/worker-sizes.spec.ts index d578b4f786..89caa6f16b 100644 --- a/e2e/tests/worker-sizes.spec.ts +++ b/e2e/tests/worker-sizes.spec.ts @@ -8,6 +8,7 @@ import { expect, layer } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { Effect, FileSystem, Path, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import * as skipCheck from "../ci-skip-check.ts"; /** * One Worker's upload: its bytes, its budget, the sources the bytes come from, and the bytes its @@ -37,6 +38,10 @@ const check = (input: { readonly uploads: Readonly>; readonly base?: Readonly> | "missing"; readonly description?: string; + /** A push to main's merged pull requests, as worker-baseline.ts `merged` writes them. */ + readonly merged?: ReadonlyArray<{ readonly number: number; readonly body: string }>; + /** What the base is, as the report names it (scripts/ci/worker-baseline.ts's `label`). */ + readonly baseline?: string; }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -64,6 +69,8 @@ const check = (input: { yield* fs.writeFileString(summary, ""); if (input.description !== undefined) yield* fs.writeFileString(path.join(directory, "description"), input.description); + if (input.merged !== undefined) + yield* fs.writeFileString(path.join(directory, "merged"), JSON.stringify(input.merged)); const child = yield* processes.spawn( ChildProcess.make( "bun", @@ -76,6 +83,8 @@ const check = (input: { ...(input.description === undefined ? [] : ["--description", path.join(directory, "description")]), + ...(input.merged === undefined ? [] : ["--merged", path.join(directory, "merged")]), + ...(input.baseline === undefined ? [] : ["--baseline", input.baseline]), ], { env: { GITHUB_STEP_SUMMARY: summary }, extendEnv: true }, ), @@ -137,6 +146,42 @@ layer(NodeServices.layer, { excludeTestServices: true })("Worker upload growth", }), ); + it.effect( + "the error gives the line to add, and that line passes once its reason replaces the placeholder", + () => + Effect.gen(function* () { + const uploads = { ...main, api: { bytes: 5_450_001, budget: 6_000_000 } }; + const line = "api +50,001: "; + const failed = yield* check({ uploads, description: "Adds a sign-in method." }); + expect(failed.exitCode, failed.log).toBe(1); + expect(failed.errors).toEqual([ + expect.stringContaining( + `Trim it, or allow it: put the line \`${line}\` in a worker-growth block of the pull request description, with your reason in place of , then run this job again.`, + ), + ]); + expect(failed.log).toContain(`\`\`\`worker-growth\n${line}\n\`\`\``); + // Pasted as printed, the placeholder is not a reason. + const pasted = yield* check({ uploads, description: description(line) }); + expect(pasted.exitCode, pasted.log).toBe(1); + expect(pasted.summary).toContain( + `- \`${line}\` is not a Worker, a size and a reason (write the real reason in place of )`, + ); + const reasoned = yield* check({ + uploads, + description: description( + "api +50,001: the SAML library, which every enterprise sign-in loads", + ), + }); + expect(reasoned.exitCode, reasoned.log).toBe(0); + // A real reason may itself start and end with angle brackets. + const bracketed = yield* check({ + uploads, + description: description("api +50,001: around "), + }); + expect(bracketed.exitCode, bracketed.log).toBe(0); + }), + ); + it.effect("an allowance for at least what the Worker adds passes and shows its reason", () => Effect.gen(function* () { const result = yield* check({ @@ -235,7 +280,7 @@ layer(NodeServices.layer, { excludeTestServices: true })("Worker upload growth", }); expect(result.exitCode, result.log).toBe(0); expect(result.summary).toContain( - "Main's sizes for this base were not available, so no change is shown and growth is not checked.", + "The sizes of main were not available, so no change is shown and growth is not checked.", ); expect(result.summary).toContain("| api | 5,900,000 | n/a | 6,000,000 |"); }), @@ -370,3 +415,2222 @@ layer(NodeServices.layer, { excludeTestServices: true })("Worker static imports" }), ); }); + +/** + * A `gh` for scripts/ci/worker-baseline.ts that answers from `answers`: open pull requests by head + * branch (`open`), merge bases by branch (`merge`; a branch missing from it is not found), main's + * push runs by commit (`runs`), recorded sizes by artifact name (`artifacts`), the run each was + * recorded by (`runSources`; by default a CI run of this repository's pull request), the uploads + * each download writes (`downloads`, by artifact name) and the commit a recorded build names + * (`builtFrom`; by default the one in its name, null for none). For `above`: the pull requests on + * the branch (`uppers`), the CI runs on each head (`upperRuns`) and each run's record (`records`, + * shown after `after` reads). `failPullRequests` and `failCompare` fail for good; `flaky` fails a + * call containing a key that many times first. Each call is logged. + */ +interface Upper { + readonly number: number; + readonly headRefOid: string; + readonly isCrossRepository?: boolean; +} + +interface StubRecord { + readonly id?: number; + readonly name: string; + readonly text: string; + readonly after?: number; +} + +interface GitHubAnswers { + readonly open?: Readonly< + Record> + >; + readonly merge?: Readonly>; + readonly runs?: Readonly>; + readonly artifacts?: Readonly>; + readonly downloads?: Readonly>>>; + readonly builtFrom?: Readonly>; + readonly runSources?: Readonly< + Record + >; + readonly uppers?: ReadonlyArray; + /** The pull requests above as each successive read finds them; the last one stays. */ + readonly upperSnapshots?: ReadonlyArray>; + readonly upperRuns?: Readonly< + Record< + string, + ReadonlyArray<{ + id: number; + path?: string; + status?: string; + conclusion?: string | null; + repository?: string; + }> + > + >; + /** Each run's records, listed once the run's artifacts have been read more than `after` times. */ + readonly records?: Readonly>>; + readonly otherArtifacts?: Readonly>>; + /** A run's select job, for a run whose status is not completed. */ + readonly selectJobs?: Readonly>; + readonly failPullRequests?: boolean; + readonly failCompare?: ReadonlyArray; + /** + * The skip check's ancestry compare, by \`...\`: GitHub's status, or \`fail\`. + * Unlisted pairs answer 404, as a commit GitHub does not know would. + */ + readonly contains?: Readonly>; + readonly flaky?: Readonly>; + /** A push to main: the commits it adds, each one's merged pull requests, and their bodies. */ + readonly pushCommits?: ReadonlyArray; + readonly commitPulls?: Readonly>>; + readonly pullBodies?: Readonly>; + readonly failPush?: boolean; + /** A call containing a key never answers, that many times. */ + readonly hangs?: Readonly>; + /** A call containing a key answers that many milliseconds late. */ + readonly slow?: Readonly>; + /** Each commit's message, author, parents and changed files, by SHA. */ + readonly commits?: Readonly< + Record< + string, + { + message: string; + author: string; + parents: ReadonlyArray; + files: ReadonlyArray<{ name: string; patch: string | null }>; + } + > + >; +} + +const stubGh = `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const answers = JSON.parse(fs.readFileSync(path.join(__dirname, "answers.json"), "utf8")); +const args = process.argv.slice(2); +const command = args.join(" "); +fs.appendFileSync(path.join(__dirname, "calls"), command + "\\n"); +const value = (flag) => args[args.indexOf(flag) + 1]; +const fail = (message) => { process.stderr.write(message + "\\n"); process.exit(1); }; +// A call whose text contains a key of \`hangs\` never answers, that many times. +for (const [part, times] of Object.entries(answers.hangs || {})) { + if (!command.includes(part)) continue; + const count = path.join(__dirname, "hang-" + Buffer.from(part).toString("hex")); + const hung = fs.existsSync(count) ? Number(fs.readFileSync(count, "utf8")) : 0; + if (hung < times) { fs.writeFileSync(count, String(hung + 1)); Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0); } +} +// A call whose text contains a key of \`slow\` answers that many milliseconds late. +for (const [part, ms] of Object.entries(answers.slow || {})) + if (command.includes(part)) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +// A call whose text contains a key of \`flaky\` fails, as GitHub does for a moment, that many times. +for (const [part, times] of Object.entries(answers.flaky || {})) { + if (!command.includes(part)) continue; + const count = path.join(__dirname, "flaky-" + Buffer.from(part).toString("hex")); + const failed = fs.existsSync(count) ? Number(fs.readFileSync(count, "utf8")) : 0; + if (failed < times) { fs.writeFileSync(count, String(failed + 1)); fail("gh: Server Error (HTTP 502)"); } +} +const calls = (part) => fs.readFileSync(path.join(__dirname, "calls"), "utf8").split("\\n").filter((call) => call.includes(part)).length; +// A conditional REST read (\`gh api -i\`): an ETag of the body, and a 304, on which gh exits 1, when +// it is the one the caller sent. +const reply = (body) => { + const text = JSON.stringify(body); + const etag = 'W/"' + require("node:crypto").createHash("sha1").update(text).digest("hex") + '"'; + const sent = args.includes("-H") ? value("-H").replace(/^If-None-Match: /, "") : undefined; + if (sent === etag) { process.stdout.write("HTTP/2.0 304 Not Modified\\r\\nEtag: " + etag + "\\r\\n\\r\\n"); process.exit(1); } + process.stdout.write("HTTP/2.0 200 OK\\r\\nEtag: " + etag + "\\r\\n\\r\\n" + text); +}; +const at = (list, part) => list[Math.min(calls(part), list.length) - 1]; +const pathArg = args.find((arg) => arg.startsWith("repos/")) || ""; +if (args[0] === "pr" && args[1] === "list") { + if (answers.failPullRequests) fail("gh: Server Error (HTTP 502)"); + const open = (answers.open || {})[value("--head")] || []; + process.stdout.write(JSON.stringify(open.map((pull) => ({ isCrossRepository: false, ...pull })))); +} else if (args[0] === "api" && args[1] === "rate_limit") { + process.stdout.write("core 120 used of 1000, 880 left; graphql 0 used of 1000"); +} else if (args[0] === "api" && /\\/pulls\\?base=/.test(pathArg)) { + if (answers.failPullRequests) fail("gh: Server Error (HTTP 502)"); + const uppers = answers.upperSnapshots ? at(answers.upperSnapshots, "/pulls?base=") : answers.uppers || []; + reply(uppers.map((pull) => ({ number: pull.number, head: { sha: pull.headRefOid, repo: { full_name: pull.isCrossRepository ? "someone/fork" : "owner/repository" } } }))); +} else if (args[0] === "api" && /\\/actions\\/runs\\?/.test(pathArg)) { + const headSha = pathArg.replace(/^.*head_sha=/, "").replace(/&.*$/, ""); + const runs = ((answers.upperRuns || {})[headSha] || []).map((run) => ({ path: ".github/workflows/ci.yml", status: "completed", conclusion: "success", html_url: "https://github.com/owner/repository/actions/runs/" + run.id, head_repository: { full_name: run.repository || "owner/repository" }, ...run })); + reply({ workflow_runs: runs }); +} else if (args[0] === "api" && /\\/actions\\/runs\\/\\d+\\/artifacts/.test(pathArg)) { + const run = pathArg.replace(/^.*\\/runs\\//, "").replace(/\\/.*$/, ""); + const reads = calls("/runs/" + run + "/artifacts"); + const listed = [].concat((answers.records || {})[run] || []).filter((record) => reads > (record.after || 0)); + reply({ artifacts: [...listed.map((record, index) => ({ id: record.id || Number(run) * 10 + index, name: record.name })), ...((answers.otherArtifacts || {})[run] || [])] }); +} else if (args[0] === "api" && /\\/actions\\/runs\\/\\d+\\/jobs/.test(pathArg)) { + const run = pathArg.replace(/^.*\\/runs\\//, "").replace(/\\/.*$/, ""); + const select = (answers.selectJobs || {})[run] || { status: "in_progress" }; + reply({ jobs: [{ name: "checks / select", status: select.status, steps: [{ name: "Check that the layer above was computed on this layer", status: select.waitingAbove ? "in_progress" : "pending" }] }] }); +} else if (args[0] === "api" && /\\/compare\\//.test(args[1]) && value("--jq") === ".status") { + const pair = args[1].replace(/^.*\\/compare\\//, "").replace(/\\?.*$/, ""); + const status = (answers.contains || {})[pair]; + if (status === undefined) fail("gh: Not Found (HTTP 404)"); + if (status === "fail") fail("gh: Server Error (HTTP 502)"); + process.stdout.write(status + "\\n"); +} else if (args[0] === "api" && /\\/compare\\//.test(args[1]) && value("--jq") === "[.commits[].sha]") { + if (answers.failPush) fail("gh: Server Error (HTTP 502)"); + process.stdout.write(JSON.stringify(answers.pushCommits || [])); +} else if (args[0] === "api" && /\\/commits\\/[0-9a-f]+\\/pulls$/.test(args[1])) { + const commit = args[1].replace(/^.*\\/commits\\//, "").replace(/\\/pulls$/, ""); + process.stdout.write(JSON.stringify((answers.commitPulls || {})[commit] || [])); +} else if (args[0] === "api" && /\\/pulls\\/\\d+$/.test(args[1])) { + const number = args[1].replace(/^.*\\/pulls\\//, ""); + const body = (answers.pullBodies || {})[number]; + if (body === undefined) fail("gh: Not Found (HTTP 404)"); + process.stdout.write(body); +} else if (args[0] === "api" && /\\/compare\\//.test(args[1])) { + const branch = decodeURIComponent(args[1].replace(/^.*\\/compare\\//, "").replace(/\\.\\.\\..*$/, "")); + if ((answers.failCompare || []).includes(branch)) fail("gh: Server Error (HTTP 503)"); + const sha = (answers.merge || {})[branch]; + if (sha === undefined) fail("gh: Not Found (HTTP 404)"); + process.stdout.write(sha); +} else if (args[0] === "run" && args[1] === "list") { + // Only main's own push runs are answered: a pull request's run can name main as its branch. + process.stdout.write(value("--event") === "push" ? (answers.runs || {})[value("--commit")] || "" : ""); +} else if (args[0] === "api" && /\\/actions\\/artifacts\\?name=/.test(args[1])) { + const name = args[1].replace(/^.*name=/, "").replace(/&.*$/, ""); + process.stdout.write((answers.artifacts || {})[name] || "[]"); +} else if (args[0] === "api" && /\\/actions\\/runs\\/\\d+$/.test(args[1])) { + const source = (answers.runSources || {})[args[1].replace(/^.*\\/runs\\//, "")]; + process.stdout.write(JSON.stringify({ path: ".github/workflows/ci.yml", event: "pull_request", repository: "owner/repository", ...source })); +} else if (args[0] === "api" && /\\/commits\\/[0-9a-f]+$/.test(args[1])) { + const info = (answers.commits || {})[args[1].replace(/^.*\\/commits\\//, "")]; + if (info === undefined) fail("gh: Not Found (HTTP 404)"); + process.stdout.write(JSON.stringify(info)); +} else if (args[0] === "run" && args[1] === "download") { + const name = value("--name"); + fs.mkdirSync(value("--dir"), { recursive: true }); + if (name.startsWith("ci-tested-")) { + const record = Object.values(answers.records || {}).flat().find((each) => each.name === name); + if (record === undefined) fail("no artifact matches"); + fs.writeFileSync(path.join(value("--dir"), "record"), record.text); + process.exit(0); + } + const uploads = (answers.downloads || {})[name]; + if (uploads === undefined) fail("no artifact matches"); + for (const [worker, upload] of Object.entries(uploads)) + fs.writeFileSync(path.join(value("--dir"), worker + ".json"), JSON.stringify({ worker, budget: 10000000, sources: {}, ...upload })); + // A recorded build names the commit it was built from, unless \`builtFrom\` says otherwise. + const built = name.startsWith("worker-uploads-at-") ? ((answers.builtFrom || {})[name] === undefined ? name.slice("worker-uploads-at-".length) : answers.builtFrom[name]) : name === "worker-build" ? ((answers.builtFrom || {})[name] ?? null) : null; + if (built !== null) fs.writeFileSync(path.join(value("--dir"), name === "worker-build" ? "built-from.txt" : "baseline-commit.txt"), built + "\\n"); +} else fail("unexpected gh " + command); +`; + +/** One run of scripts/ci/worker-baseline.ts with `env`, against a stub `gh` answering `answers`. */ +const baseline = ( + command: ReadonlyArray, + env: Readonly>, + answers: GitHubAnswers = {}, +) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "executor-worker-baseline-" }); + const bin = path.join(directory, "bin"); + yield* fs.makeDirectory(bin); + yield* fs.writeFileString(path.join(bin, "gh"), stubGh); + yield* fs.chmod(path.join(bin, "gh"), 0o755); + yield* fs.writeFileString(path.join(bin, "answers.json"), JSON.stringify(answers)); + yield* fs.writeFileString(path.join(bin, "calls"), ""); + const output = path.join(directory, "output"); + yield* fs.writeFileString(output, ""); + const sizes = path.join(directory, "sizes"); + const file = path.join(directory, "merged.json"); + const child = yield* processes.spawn( + ChildProcess.make( + "node", + [ + "scripts/ci/worker-baseline.ts", + ...command.map((arg) => (arg === "" ? sizes : arg === "" ? file : arg)), + ], + { + env: { + PATH: `${bin}:${process.env["PATH"] ?? ""}`, + GITHUB_REPOSITORY: "owner/repository", + DEFAULT_BRANCH: "main", + GITHUB_OUTPUT: output, + RUNNER_TEMP: directory, + WORKER_BASELINE_RETRY_SECONDS: "0.01", + ...env, + }, + extendEnv: true, + }, + ), + ); + const [log, exitCode] = yield* Effect.all( + [child.all.pipe(Stream.decodeText(), Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + const outputs = Object.fromEntries( + (yield* fs.readFileString(output)) + .split("\n") + .filter((line) => line.includes("=")) + .map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1)]), + ); + return { + exitCode: Number(exitCode), + log, + outputs, + calls: (yield* fs.readFileString(path.join(bin, "calls"))).split("\n").filter(Boolean), + sizes: (yield* fs.exists(sizes)) ? yield* fs.readDirectory(sizes) : [], + file: (yield* fs.exists(file)) ? yield* fs.readFileString(file) : undefined, + }; + }).pipe(Effect.scoped); + +const sha = (character: string) => character.repeat(40); + +/** The files and diffs of a beta release commit, as 940e411b6 (beta.53) changed them. */ +const releaseFiles = (version: string) => { + const before = `2.0.0-beta.${Number(version) - 1}`; + const after = `2.0.0-beta.${version}`; + return [ + { + name: "apps/cli/CHANGELOG.md", + patch: `@@ -1,3 +1,9 @@\n # executor\n \n+## ${after}\n+`, + }, + { + name: "apps/cli/package.json", + patch: `@@ -1,6 +1,6 @@\n {\n "name": "executor",\n- "version": "${before}",\n+ "version": "${after}",\n "private": true,\n "description": "Run Executor locally and connect your agents to your apps.",\n "homepage": "https://executor.sh",`, + }, + { + name: "bun.lock", + patch: `@@ -40,7 +40,7 @@\n },\n "apps/cli": {\n "name": "executor",\n- "version": "${before}",\n+ "version": "${after}",\n },\n "apps/docs": {\n "name": "@executor-js/docs",`, + }, + ]; +}; +const head = sha("f"); + +layer(NodeServices.layer, { excludeTestServices: true })("Worker size baseline", (it) => { + it.effect("a stack that ends on main is compared with main, as before", () => + Effect.gen(function* () { + // A pull request into main: main at its base commit, without reading the stack. + const onMain = yield* baseline(["resolve"], { + BASE_REF: "main", + BASE_SHA: sha("1"), + HEAD_SHA: head, + }); + expect(onMain.outputs).toMatchObject({ + branch: "main", + commit: sha("1"), + id: `main@${sha("1")}`, + }); + expect(onMain.calls).toEqual([]); + // A push to main: the commit before it. + const push = yield* baseline(["resolve"], { BASE_REF: "", BASE_SHA: sha("2") }); + expect(push.outputs).toMatchObject({ branch: "main", commit: sha("2") }); + // An upper layer: its base is the head of an open pull request into main, two layers down. + const layered = yield* baseline( + ["resolve"], + { BASE_REF: "layer-2", BASE_SHA: sha("3"), HEAD_SHA: head }, + { + open: { "layer-2": [{ baseRefName: "layer-1" }], "layer-1": [{ baseRefName: "main" }] }, + merge: { main: sha("4") }, + }, + ); + expect(layered.exitCode, layered.log).toBe(0); + expect(layered.outputs).toMatchObject({ branch: "main", commit: sha("4"), note: "" }); + }), + ); + + it.effect( + "a stack that ends on another branch is compared with that branch at its merge base", + () => + Effect.gen(function* () { + // A pull request into the long-lived branch, and one stacked on such a pull request. + const answers: GitHubAnswers = { + open: { "eapps-layer": [{ baseRefName: "effect-apps" }] }, + merge: { "effect-apps": sha("5"), main: sha("6") }, + }; + for (const base of ["effect-apps", "eapps-layer"]) { + const run = yield* baseline( + ["resolve"], + { BASE_REF: base, BASE_SHA: sha("7"), HEAD_SHA: head }, + answers, + ); + expect(run.exitCode, run.log).toBe(0); + expect(run.outputs, base).toMatchObject({ + branch: "effect-apps", + commit: sha("5"), + id: `effect-apps@${sha("5")}`, + label: "effect-apps at 555555555", + }); + } + // The long-lived branch's own pull request into main is a stack on main: charged its total. + const merge = yield* baseline( + ["resolve"], + { BASE_REF: "main", BASE_SHA: sha("8"), HEAD_SHA: head }, + answers, + ); + expect(merge.outputs).toMatchObject({ branch: "main", commit: sha("8") }); + // Another repository's pull request from a branch of the same name is not this stack. + const fork = yield* baseline( + ["resolve"], + { BASE_REF: "effect-apps", BASE_SHA: sha("7"), HEAD_SHA: head }, + { + ...answers, + open: { "effect-apps": [{ baseRefName: "main", isCrossRepository: true }] }, + }, + ); + expect(fork.outputs).toMatchObject({ branch: "effect-apps" }); + }), + ); + + it.effect("whatever cannot be read, or could end on main, is compared with main", () => + Effect.gen(function* () { + const env = { BASE_REF: "effect-apps", BASE_SHA: sha("7"), HEAD_SHA: head }; + // The base branch was deleted, or force-pushed so it shares no history with the head. + const gone = yield* baseline(["resolve"], env, { merge: { main: sha("6") } }); + expect(gone.exitCode, gone.log).toBe(0); + expect(gone.outputs).toMatchObject({ branch: "main", commit: sha("6") }); + expect(gone.outputs.note).toContain("effect-apps could not be read"); + // The stack below could not be listed. + const unlisted = yield* baseline( + ["resolve"], + { ...env, BASE_REF: "eapps-layer" }, + { + failPullRequests: true, + merge: { main: sha("6"), "effect-apps": sha("5") }, + }, + ); + expect(unlisted.outputs).toMatchObject({ branch: "main", commit: sha("6") }); + expect(unlisted.outputs.note).toContain("could not be read"); + // A branch with open pull requests into main and into another branch could end on main. + const both = yield* baseline( + ["resolve"], + { ...env, BASE_REF: "shared" }, + { + open: { shared: [{ baseRefName: "effect-apps" }, { baseRefName: "main" }] }, + merge: { main: sha("6"), "effect-apps": sha("5") }, + }, + ); + expect(both.outputs).toMatchObject({ branch: "main", commit: sha("6") }); + // A branch whose stack could end on two branches, neither of them main: no one baseline. + const two = yield* baseline( + ["resolve"], + { ...env, BASE_REF: "shared" }, + { + open: { shared: [{ baseRefName: "effect-apps" }, { baseRefName: "other-branch" }] }, + merge: { main: sha("6"), "effect-apps": sha("5"), "other-branch": sha("4") }, + }, + ); + expect(two.outputs).toMatchObject({ branch: "main", commit: sha("6") }); + // --main resolves main whatever the stack ends on. + const forced = yield* baseline(["resolve", "--main"], env, { + merge: { main: sha("6"), "effect-apps": sha("5") }, + }); + expect(forced.outputs).toMatchObject({ branch: "main", commit: sha("6") }); + }), + ); + + it.effect( + "a branch's sizes come from a recorded build of that commit, else a build, else main's", + () => + Effect.gen(function* () { + const branch = { BASELINE_BRANCH: "effect-apps", BASELINE_COMMIT: sha("5") }; + const name = `worker-uploads-at-${sha("5")}`; + const recorded = yield* baseline(["fetch", ""], branch, { + artifacts: { [name]: JSON.stringify([{ expired: false, run: 77 }]) }, + downloads: { [name]: { api: { bytes: 5_500_000 } } }, + }); + expect(recorded.outputs).toEqual({ + found: "true", + build: "false", + label: "effect-apps at 555555555", + }); + expect(recorded.sizes.toSorted()).toEqual(["api.json", "baseline-commit.txt"]); + // Nothing recorded, or only an expired record (which GitHub may still serve): the job + // builds the commit. + for (const artifacts of [{}, { [name]: JSON.stringify([{ expired: true, run: 76 }]) }]) { + const missing = yield* baseline(["fetch", ""], branch, { + artifacts, + downloads: { [name]: { api: { bytes: 5_500_000 } } }, + }); + expect(missing.outputs).toMatchObject({ found: "false", build: "true" }); + expect(missing.sizes).toEqual([]); + } + // Main's sizes come from main's own CI run at the commit, never from a recorded build. + const main = yield* baseline( + ["fetch", ""], + { BASELINE_BRANCH: "main", BASELINE_COMMIT: sha("6") }, + { + runs: { [sha("6")]: "88" }, + downloads: { "worker-uploads": { api: { bytes: 5_400_000 } } }, + artifacts: { + [`worker-uploads-at-${sha("6")}`]: JSON.stringify([{ expired: false, run: 99 }]), + }, + }, + ); + expect(main.outputs).toMatchObject({ + found: "true", + build: "false", + label: "main at 666666666", + }); + expect(main.calls.some((call) => call.includes("actions/artifacts"))).toBe(false); + // Without the branch's sizes, the fallback compares with main's. + const fallback = yield* baseline( + ["fallback", ""], + { BASE_REF: "effect-apps", BASE_SHA: sha("7"), HEAD_SHA: head }, + { + merge: { main: sha("6") }, + runs: { [sha("6")]: "88" }, + downloads: { "worker-uploads": { api: { bytes: 5_400_000 } } }, + }, + ); + expect(fallback.outputs).toMatchObject({ branch: "main", commit: sha("6"), found: "true" }); + expect(fallback.sizes).toEqual(["api.json"]); + }), + ); + + it.effect( + "a release commit has no sizes of its own, so main's are those of the nearest commit before it that has them", + () => + Effect.gen(function* () { + const release = (version: string, parent: string) => ({ + message: `Release 2.0.0-beta.${version} [skip ci]`, + author: "github-actions[bot]", + parents: [parent], + files: releaseFiles(version), + }); + const sizes = { "worker-uploads": { api: { bytes: 5_400_000 } } }; + const at = (commit: string) => ({ BASELINE_BRANCH: "main", BASELINE_COMMIT: commit }); + // 9cbea0adc's shape: one release commit on top of a commit main's CI ran. + const one = yield* baseline(["fetch", ""], at(sha("a")), { + commits: { [sha("a")]: release("54", sha("b")) }, + runs: { [sha("b")]: "88" }, + downloads: sizes, + }); + expect(one.outputs).toMatchObject({ found: "true", label: "main at bbbbbbbbb" }); + expect(one.log).toContain( + `main's sizes for ${sha("a")} are those of ${sha("b")}, the nearest earlier commit with sizes: ${sha("a")} is a release commit, which changes no Worker.`, + ); + // Two release commits in a row are both stepped over. + const two = yield* baseline(["fetch", ""], at(sha("a")), { + commits: { [sha("a")]: release("54", sha("c")), [sha("c")]: release("53", sha("b")) }, + runs: { [sha("b")]: "88" }, + downloads: sizes, + }); + expect(two.outputs).toMatchObject({ found: "true", label: "main at bbbbbbbbb" }); + // A commit that is not a release commit is not stepped over: one with other files, another + // author, another title (or the title further in), more than one parent, no files, or a + // lockfile or manifest change beyond the CLI's version; its own sizes are simply not there + // yet, as before. + const files = releaseFiles("54"); + const changed = (name: string, patch: string | null) => + files.map((file) => (file.name === name ? { name, patch } : file)); + for (const other of [ + { + ...release("54", sha("b")), + files: [ + ...files, + { name: "apps/hosted/cloud/src/api.ts", patch: "@@ -1 +1 @@\n-a\n+b" }, + ], + }, + { ...release("54", sha("b")), author: "someone" }, + { ...release("54", sha("b")), message: "Fix the API [skip ci]" }, + { ...release("54", sha("b")), message: 'Revert "Release 2.0.0-beta.54 [skip ci]"' }, + { ...release("54", sha("b")), parents: [sha("b"), sha("c")] }, + { ...release("54", sha("b")), files: [] }, + { + ...release("54", sha("b")), + files: changed( + "bun.lock", + `${files[2]!.patch}\n@@ -900,7 +900,7 @@\n "effect": ["effect@4.1.4", "", {}, "sha512-a"],\n- "hono": ["hono@4.9.0", "", {}, "sha512-b"],\n+ "hono": ["hono@4.9.1", "", {}, "sha512-c"],`, + ), + }, + { + ...release("54", sha("b")), + files: changed( + "bun.lock", + files[2]!.patch!.replace('"apps/cli": {', '"apps/hosted/cloud": {'), + ), + }, + { + ...release("54", sha("b")), + files: changed( + "apps/cli/package.json", + files[1]!.patch!.replace( + ' "private": true,', + '- "private": true,\n+ "dependencies": { "effect": "4.2.0" },', + ), + ), + }, + { ...release("54", sha("b")), files: changed("bun.lock", null) }, + ]) { + const stopped = yield* baseline(["fetch", ""], at(sha("a")), { + commits: { [sha("a")]: other }, + runs: { [sha("b")]: "88" }, + downloads: sizes, + }); + expect(stopped.outputs, JSON.stringify(other)).toMatchObject({ + found: "false", + label: "main at aaaaaaaaa", + }); + expect(stopped.log).toContain( + `main's sizes for ${sha("a")} are not available yet, so no change is shown.`, + ); + } + // No sizes within five release commits: as before, and it says which it looked past. + const chain = Object.fromEntries( + ["a", "c", "d", "e", "1", "2"].map((name, index, names) => [ + sha(name), + release(String(54 - index), sha(names[index + 1] ?? "b")), + ]), + ); + const far = yield* baseline(["fetch", ""], at(sha("a")), { + commits: chain, + runs: { [sha("b")]: "88" }, + downloads: sizes, + }); + expect(far.outputs).toMatchObject({ found: "false" }); + expect(far.log).toContain( + `main's sizes for ${sha("a")} are not available yet, nor for the release commits before it (`, + ); + expect(far.calls.filter((call) => call.includes("/commits/"))).toHaveLength(5); + }), + ); + + /** + * Three pull requests into effect-apps whose descriptions allowed the branch's growth over main + * while adding nothing themselves: #4552, #4549 and #4538, with their real commits and the growth + * their runs measured against main. + */ + const effectApps = [ + { + pull: 4552, + base: "1179b8ca4265c1732417025f940ffbc157ba29fe", + head: "8bac4152b92066961b76ff939971213a4f44a9d7", + }, + { + pull: 4549, + base: "25fee749c5b43a26f15db7e19f6b6cf7fd5a1b16", + head: "dae54b308c893385ab6467395665265514f55f7e", + }, + { + pull: 4538, + base: "72546a2ddc63bed6f2787772228104e4a106559c", + head: "d65f5aac1c4c08a9933f896bf9e71f47d856bda2", + }, + ]; + const branchSizes = { + "mcp-server": { bytes: 3_520_000, budget: 3_700_000 }, + api: { bytes: 5_355_000, budget: 6_000_000 }, + "app-pages": { bytes: 2_578_000, budget: 2_700_000 }, + "app-data": { bytes: 723_000, budget: 750_000 }, + }; + const mainSizes = { + "mcp-server": { bytes: 3_379_000, budget: 3_700_000 }, + api: { bytes: 5_234_000, budget: 6_000_000 }, + "app-pages": { bytes: 2_460_000, budget: 2_700_000 }, + "app-data": { bytes: 664_000, budget: 750_000 }, + }; + + it.effect( + "the effect-apps pull requests that added nothing need no allowance against their branch", + () => + Effect.gen(function* () { + for (const { pull, base, head: commit } of effectApps) { + const resolved = yield* baseline( + ["resolve"], + { BASE_REF: "effect-apps", BASE_SHA: base, HEAD_SHA: commit }, + { merge: { "effect-apps": base, main: sha("6") } }, + ); + expect(resolved.outputs, `#${pull}`).toMatchObject({ + branch: "effect-apps", + commit: base, + }); + // Against effect-apps it adds nothing: no allowance. + const own = yield* check({ + uploads: branchSizes, + base: branchSizes, + description: "No worker-growth block.", + baseline: resolved.outputs.label!, + }); + expect(own.exitCode, `#${pull}: ${own.log}`).toBe(0); + // Against main, as before, it was charged the branch's lead. + const before = yield* check({ + uploads: branchSizes, + base: mainSizes, + description: "No worker-growth block.", + }); + expect(before.exitCode, `#${pull}`).toBe(1); + } + }), + ); + + it.effect( + "growth of a pull request's own still fails, named against its baseline, and budgets hold off main", + () => + Effect.gen(function* () { + // #4344: +60 KB of its own to api, into main, fails without its allowance. + const v1 = yield* check({ + uploads: { ...main, api: { bytes: 5_460_000, budget: 6_000_000 } }, + description: "Serves v1's MCP URLs.", + baseline: "main at 3f59c3bb0", + }); + expect(v1.exitCode).toBe(1); + expect(v1.errors).toEqual([ + expect.stringContaining( + "::error title=api Worker grew without an allowance::+60,000 bytes over main at 3f59c3bb0", + ), + ]); + // A pull request on effect-apps that grows a Worker 50 KB past the branch fails, naming the branch. + const grown = yield* check({ + uploads: { ...branchSizes, api: { bytes: 5_405_001, budget: 6_000_000 } }, + base: branchSizes, + description: "Adds a route.", + baseline: "effect-apps at 555555555", + }); + expect(grown.exitCode).toBe(1); + expect(grown.errors).toEqual([ + expect.stringContaining("+50,001 bytes over effect-apps at 555555555"), + ]); + expect(grown.summary).toContain("Change from effect-apps at 555555555"); + // effect-apps' own pull request into main is charged its total against main. + const merge = yield* check({ + uploads: branchSizes, + base: mainSizes, + description: "Merges apps 1.x.", + baseline: "main at 3f59c3bb0", + }); + expect(merge.exitCode).toBe(1); + expect(merge.errors.join("\n")).toContain("+141,000 bytes over main at 3f59c3bb0"); + // A budget fails whatever the baseline, with no growth at all. + const over = yield* check({ + uploads: { ...branchSizes, api: { bytes: 6_000_001, budget: 6_000_000 } }, + base: { ...branchSizes, api: { bytes: 6_000_001, budget: 6_000_000 } }, + description: "No growth.", + baseline: "effect-apps at 555555555", + }); + expect(over.exitCode).toBe(1); + expect(over.errors).toEqual([ + expect.stringContaining("title=api Worker over its upload budget"), + ]); + }), + ); + + it.effect( + "a run is never repeated across baselines, and a missing branch build falls back to main", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const checks = yield* fs.readFileString(".github/workflows/checks.yml"); + // The size gate's record names the baseline, in select (what a later run expects) and in + // site (what this run's gate read), so a run compared with one baseline is not reported for + // another. + expect(checks).toContain( + 'sized=$(node e2e/ci-key.ts record "$tree" "$base" sized "$WORKER_GROWTH" "$BASELINE")', + ); + expect(checks).toContain( + 'node e2e/ci-key.ts record "$tree" "$base" sized "$growth" "$BASELINE"', + ); + expect(checks).toContain( + "BASELINE: ${{ steps.main-baseline.outputs.id || steps.baseline.outputs.id }}", + ); + expect(checks).toContain( + 'key=$(node e2e/ci-key.ts record "$tree" "$base" "$SELECTION" "$WORKER_GROWTH")', + ); + // A branch build that fails falls back to main's sizes; nothing skips the comparison. + expect(checks).toContain( + "if: steps.baseline-sizes.outputs.build == 'true' && steps.baseline-build.outcome != 'success'", + ); + expect(checks).toContain( + "node scripts/ci/worker-baseline.ts fallback .local/worker-uploads-base", + ); + expect(checks).toContain(`--baseline "$BASELINE"`); + }), + ); + + it.effect( + "sizes are used only from a source this repository's own code wrote, for the commit asked for", + () => + Effect.gen(function* () { + // Main's sizes: only from main's push runs (the stub answers no other event). + const main = yield* baseline( + ["fetch", ""], + { BASELINE_BRANCH: "main", BASELINE_COMMIT: sha("6") }, + { runs: { [sha("6")]: "88" }, downloads: { "worker-uploads": { api: { bytes: 1 } } } }, + ); + expect(main.outputs).toMatchObject({ found: "true" }); + expect(main.calls.find((call) => call.startsWith("run list"))).toContain("--event push"); + // A branch's sizes: only from CI runs of this repository's own pull requests, and only + // when they name the commit they were built from. + const branch = { BASELINE_BRANCH: "effect-apps", BASELINE_COMMIT: sha("5") }; + const name = `worker-uploads-at-${sha("5")}`; + const recorded = { + artifacts: { [name]: JSON.stringify([{ expired: false, run: 77 }]) }, + downloads: { [name]: { api: { bytes: 5_500_000 } } }, + }; + const own = yield* baseline(["fetch", ""], branch, recorded); + expect(own.outputs).toMatchObject({ found: "true", build: "false" }); + expect(own.sizes.toSorted()).toEqual(["api.json", "baseline-commit.txt"]); + for (const [why, answers] of [ + ["a fork's run", { runSources: { "77": { repository: "someone/fork" } } }], + ["another workflow's run", { runSources: { "77": { path: ".github/workflows/x.yml" } } }], + ["a push run", { runSources: { "77": { event: "push" } } }], + ["sizes built from another commit", { builtFrom: { [name]: sha("4") } }], + ["sizes that name no commit", { builtFrom: { [name]: null } }], + ] as const) { + const refused = yield* baseline(["fetch", ""], branch, { + ...recorded, + ...answers, + }); + expect(refused.outputs, why).toMatchObject({ found: "false", build: "true" }); + expect(refused.sizes, why).toEqual([]); + } + // An older trusted record after a refused one is still used. + const second = yield* baseline(["fetch", ""], branch, { + ...recorded, + artifacts: { + [name]: JSON.stringify([ + { expired: false, run: 78 }, + { expired: false, run: 77 }, + ]), + }, + runSources: { "78": { repository: "someone/fork" } }, + }); + expect(second.outputs).toMatchObject({ found: "true" }); + expect(second.calls).toContain( + `run download 77 --repo owner/repository --name ${name} --dir ${second.calls.find((call) => call.startsWith("run download"))!.split(" --dir ")[1]}`, + ); + expect(second.calls.some((call) => call.startsWith("run download 78"))).toBe(false); + }), + ); + + it.effect( + "a read that fails for a moment is tried again, and a stack that still cannot be read is compared with main", + () => + Effect.gen(function* () { + const env = { BASE_REF: "eapps-layer", BASE_SHA: sha("7"), HEAD_SHA: head }; + const answers: GitHubAnswers = { + open: { "eapps-layer": [{ baseRefName: "effect-apps" }] }, + merge: { "effect-apps": sha("5"), main: sha("6") }, + }; + // Two failures, then an answer: the stack is read as if nothing had failed. + const flaky = yield* baseline(["resolve"], env, { + ...answers, + flaky: { "pr list": 2, "compare/effect-apps": 2 }, + }); + expect(flaky.exitCode, flaky.log).toBe(0); + expect(flaky.outputs).toMatchObject({ branch: "effect-apps", commit: sha("5"), note: "" }); + expect(flaky.calls.filter((call) => call.startsWith("pr list"))).toHaveLength(4); + // Still failing after the retries: compared with main, with a notice, never a failed job. + const down = yield* baseline(["resolve"], env, { ...answers, failPullRequests: true }); + expect(down.exitCode, down.log).toBe(0); + expect(down.outputs).toMatchObject({ branch: "main", commit: sha("6") }); + expect(down.log).toContain( + "::notice title=Worker sizes::the stack below eapps-layer could not be read, so this compares with main.", + ); + expect(down.calls.filter((call) => call.startsWith("pr list"))).toHaveLength(4); + // Not even main's commit can be read: main with no sizes, so no change is shown. + const blind = yield* baseline(["resolve"], env, { + ...answers, + failPullRequests: true, + failCompare: ["main"], + }); + expect(blind.exitCode, blind.log).toBe(0); + expect(blind.outputs).toMatchObject({ + branch: "main", + commit: "", + id: "main@unread", + label: "main", + }); + expect(blind.log).toContain("so no change is shown"); + // A refusal (not found) is an answer, not a failure: it is not tried again. + const gone = yield* baseline( + ["resolve"], + { ...env, BASE_REF: "effect-apps" }, + { + merge: { main: sha("6") }, + }, + ); + expect(gone.calls.filter((call) => call.includes("compare/effect-apps"))).toHaveLength(1); + // Fetching sizes is tried again too. + const sizes = yield* baseline( + ["fetch", ""], + { BASELINE_BRANCH: "main", BASELINE_COMMIT: sha("6") }, + { + runs: { [sha("6")]: "88" }, + downloads: { "worker-uploads": { api: { bytes: 1 } } }, + flaky: { "run list": 1 }, + }, + ); + expect(sizes.outputs).toMatchObject({ found: "true" }); + }), + ); + + /** A `skip` layer `lower` at `head`, its stack ending on `bottom`, and one layer above. */ + const lowerHead = sha("c"); + // A thousandth of the rule's time: a 240-second bound takes a quarter of a second. + const skipLayer = (bottom = "main") => ({ + HEAD_REF: "lower", + HEAD_SHA: lowerHead, + BASELINE_BRANCH: bottom, + GITHUB_RUN_ID: "500", + SKIP_ABOVE_TIME_SCALE: "0.001", + }); + const upperHead = sha("d"); + const record = (base: string, baseline: string, format = "2", tested?: string) => + `${format === "" ? "" : `format ${format}\n`}pull request 11\nmerge tree ${sha("e")}\nbase ${base}\n${tested === undefined ? "" : `tested ${tested}\n`}selection x\nworker-growth y\nbaseline ${baseline}\nabove none\n`; + const good = record(lowerHead, `main@${sha("6")}`); + const laundered = record(lowerHead, `effect-apps@${sha("5")}`); + const upper = (text: string, more: GitHubAnswers = {}): GitHubAnswers => ({ + uppers: [{ number: 11, headRefOid: upperHead }], + upperRuns: { [upperHead]: [{ id: 600 }] }, + records: { "600": { name: "ci-tested-pr11-key", text } }, + ...more, + }); + /** The decision line a run logged. */ + const decided = (log: string) => + JSON.parse(log.match(/^skip-check (\{.*\})$/m)?.[1] ?? "null") as { + mode: string; + decision: string; + oldFormat: boolean; + rule: string; + waited: number; + reason: string; + message: string; + } | null; + + it.effect( + "observing, the skip check logs what it would decide and passes, as a skip layer always did", + () => + Effect.gen(function* () { + const fits = yield* baseline(["above"], skipLayer(), upper(good)); + expect(fits.exitCode, fits.log).toBe(0); + expect(decided(fits.log)).toMatchObject({ + mode: "observe", + decision: "would pass", + oldFormat: false, + reason: + "#11's run https://github.com/owner/repository/actions/runs/600 was built on ccccccccc and compared its Worker sizes with main", + }); + expect(fits.outputs.decision).toBe("pass"); + expect(fits.log).toContain("the token's rate limit: core 120 used of 1000, 880 left"); + // The laundering case: X into effect-apps ran against effect-apps; then effect-apps was + // opened into main with skip. It would fail, and the job still passes. + const fails = yield* baseline(["above"], skipLayer(), upper(laundered)); + expect(fails.exitCode, fails.log).toBe(0); + const failure = decided(fails.log)!; + expect(failure).toMatchObject({ mode: "observe", decision: "would fail" }); + expect(failure.reason).toBe( + "#11's latest run on its head, https://github.com/owner/repository/actions/runs/600, compared its Worker sizes with effect-apps, not main, the base this stack now ends on", + ); + expect(failure.message).toContain( + "To fix it, run it again: gh run rerun 600 --repo owner/repository (a re-run reads the stack afresh); once that run has started, run this one again: gh run rerun 500 --repo owner/repository", + ); + // A stale record is waited on for its bound before it is called stale. + expect(failure.waited).toBeGreaterThanOrEqual(240); + expect(fails.outputs.decision).toBe("fail"); + // Looked at again while it waited, the same record is downloaded once. + expect(fails.calls.filter((call) => call.includes("/pulls?base="))).not.toHaveLength(1); + expect(fails.calls.filter((call) => call.startsWith("run download"))).toHaveLength(1); + // This layer moved and the layer above did not: a re-run keeps its old base, so push. + const moved = decided( + (yield* baseline(["above"], skipLayer(), upper(record(sha("b"), `main@${sha("6")}`)))) + .log, + )!; + expect(moved.reason).toContain("was built on bbbbbbbbb, not this layer's head ccccccccc"); + expect(moved.message).toContain("gh stack rebase, then gh stack push"); + // A record in this format without a baseline proves nothing; a later format is not read. + for (const [text, why] of [ + [good.replace(/baseline .*\n/, ""), "a baseline it did not record"], + [record(lowerHead, `main@${sha("6")}`, "4"), "wrote a record in format 4"], + [good.replace("above none", "above fail"), "its own check of the layer above it fails"], + [good.replace("above none", "above error"), "its own check of the layer above it fails"], + ] as const) + expect( + decided((yield* baseline(["above"], skipLayer(), upper(text))).log)?.reason, + ).toContain(why); + }), + ); + + it.effect( + "a record in the format from before the check is judged by the old rule and passes, with a notice", + () => + Effect.gen(function* () { + // The upper's run used the workflow of its merge into a lower branch from before this + // check, main's: no format and no baseline, and often no base. + for (const text of [ + record(sha("b"), "", "").replace(/baseline .*\n/, ""), + "pull request 11\nmerge tree x\nselection x\n", + ]) { + const old = yield* baseline(["above"], skipLayer(), upper(text)); + expect(old.exitCode, old.log).toBe(0); + expect(decided(old.log)).toMatchObject({ decision: "would pass", oldFormat: true }); + expect(old.log).toContain( + "::notice title=Skip check::The layer above runs a workflow from before the skip check. Rebasing the stack onto main turns the check on.", + ); + } + // A baseline without a format came only from this check's own pull requests before the + // format line (#4586's upper): it is judged by what it says. + const unformatted = yield* baseline( + ["above"], + skipLayer(), + upper(record(lowerHead, `scratch@${sha("5")}`, "")), + ); + expect(decided(unformatted.log)).toMatchObject({ + decision: "would fail", + oldFormat: false, + }); + }), + ); + + it.effect( + "the newest CI run of this repository that recorded decides, and its newest record", + () => + Effect.gen(function* () { + const verdict = (more: GitHubAnswers) => + Effect.map(baseline(["above"], skipLayer(), upper(good, more)), (run) => ({ + ...decided(run.log)!, + calls: run.calls, + })); + // Any run on the head whose record fits passes: an older fitting run tested this layer's + // head against the right bottom, whatever a newer one recorded. + expect( + (yield* verdict({ + upperRuns: { [upperHead]: [{ id: 600 }, { id: 599 }] }, + records: { + "600": { name: "ci-tested-pr11-new", text: laundered }, + "599": { name: "ci-tested-pr11-old", text: good }, + }, + })).decision, + ).toBe("would pass"); + // The live input (#4575, 16 Nov): a stack push started two runs on the top's head in the + // same second, and the newer recorded the bottom's previous head. + const raced = yield* verdict({ + upperRuns: { [upperHead]: [{ id: 43809112880 }, { id: 43809112745 }] }, + records: { + "43809112880": { + name: "ci-tested-pr11-newer", + text: record(sha("a"), `main@${sha("6")}`), + }, + "43809112745": { name: "ci-tested-pr11-older", text: good }, + }, + }); + expect(raced.decision).toBe("would pass"); + expect(raced.reason).toContain("runs/43809112745 was built on ccccccccc"); + // This stack's own push (16 Nov, 7ffb3f6f4): the duplicate that fits was still queued + // behind the other, whose record had the bottom's previous head. A run still choosing is + // waited for with its own bound, past the 4 minutes of a stale record. + const queued = decided( + (yield* baseline( + ["above"], + skipLayer(), + upper(good, { + upperRuns: { + [upperHead]: [ + { id: 43814604673, status: "in_progress", conclusion: null }, + { id: 43814603973 }, + ], + }, + records: { + "43814604673": { name: "ci-tested-pr11-queued", text: good, after: 7 }, + "43814603973": { + name: "ci-tested-pr11-first", + text: record(sha("a"), `main@${sha("6")}`), + }, + }, + selectJobs: { "43814604673": { status: "in_progress" } }, + }), + )).log, + )!; + expect(queued).toMatchObject({ decision: "would pass" }); + expect(queued.waited).toBeGreaterThan(240); + // With none fitting, the newest run that recorded says why. + expect( + (yield* verdict({ + upperRuns: { [upperHead]: [{ id: 600 }, { id: 599 }] }, + records: { + "600": { name: "ci-tested-pr11-new", text: record(sha("a"), `main@${sha("6")}`) }, + "599": { name: "ci-tested-pr11-old", text: laundered }, + }, + })).reason, + ).toContain("runs/600, was built on aaaaaaaaa"); + // Another workflow's run, a cancelled run, a fork's run on the same commit and a newer run + // still choosing are passed over for the newest run that recorded. + const passedOver = yield* verdict({ + upperRuns: { + [upperHead]: [ + { id: 605, path: ".github/workflows/pr-previews.yml" }, + { id: 604, conclusion: "cancelled" }, + { id: 603, repository: "someone/fork" }, + { id: 602, status: "in_progress", conclusion: null }, + { id: 600 }, + ], + }, + records: { + "605": { name: "ci-tested-pr11-other", text: laundered }, + "604": { name: "ci-tested-pr11-cancelled", text: laundered }, + "603": { name: "ci-tested-pr11-fork", text: laundered }, + "600": { name: "ci-tested-pr11-key", text: good }, + }, + selectJobs: { "602": { status: "in_progress" } }, + }); + expect(passedOver.decision).toBe("would pass"); + expect(passedOver.calls.filter((call) => call.startsWith("run download"))).toEqual([ + expect.stringMatching(/^run download 600 /), + ]); + // A newer run whose select finished without recording does not hide an older fitting + // record; with none, it says nothing was chosen. + expect( + (yield* verdict({ + upperRuns: { [upperHead]: [{ id: 601, conclusion: "failure" }, { id: 600 }] }, + })).decision, + ).toBe("would pass"); + expect( + (yield* verdict({ + upperRuns: { [upperHead]: [{ id: 601, conclusion: "failure" }, { id: 600 }] }, + records: { "600": { name: "ci-tested-pr11-old", text: laundered } }, + })).reason, + ).toContain("recorded nothing (its select failed)"); + // A re-run that chose differently leaves two records: the newest is what it chose. + const twice = yield* verdict({ + records: { + "600": [ + { id: 6002, name: "ci-tested-pr11-b", text: laundered }, + { id: 6003, name: "ci-tested-pr11-c", text: good }, + { id: 6001, name: "ci-tested-pr11-a", text: laundered }, + ], + }, + }); + expect(twice.decision).toBe("would pass"); + // Only this pull request's records: not another's, nor the size gate's. + const others = yield* verdict({ + otherArtifacts: { + "600": [ + { id: 9001, name: "ci-tested-pr12-other" }, + { id: 9002, name: "ci-sized-pr11-key" }, + ], + }, + }); + expect(others.decision).toBe("would pass"); + // A fork's pull request above runs the fork's code, which could write any record. + expect( + (yield* verdict({ + uppers: [{ number: 11, headRefOid: upperHead, isCrossRepository: true }], + })).reason, + ).toBe("no open pull request in this repository has lower as its base"); + // Only the runs on the current head count. + expect((yield* verdict({ upperRuns: { [sha("9")]: [{ id: 600 }] } })).reason).toBe( + "#11 has no CI run on its head ddddddddd", + ); + }), + ); + + it.effect( + "a run passes when the merge commit it tested contains this layer's head, whatever its event's base", + () => + Effect.gen(function* () { + const run = (text: string, more: GitHubAnswers = {}) => + Effect.map(baseline(["above"], skipLayer(), upper(text, more)), (result) => ({ + ...decided(result.log)!, + exitCode: result.exitCode, + calls: result.calls, + })); + const tested = sha("f"); + const pair = `${lowerHead}...${tested}`; + // A duplicate event of a stack push: its base is this layer's previous head, and the merge + // commit GitHub built already holds this layer's head. It passes at once, by what it tested. + const stale = yield* run(record(sha("b"), `main@${sha("6")}`, "3", tested), { + contains: { [pair]: "ahead" }, + }); + expect(stale).toMatchObject({ decision: "would pass", rule: "tested", waited: 0 }); + expect(stale.reason).toBe( + `#11's run https://github.com/owner/repository/actions/runs/600 tested a merge commit that contains ccccccccc, and compared its Worker sizes with main`, + ); + expect(stale.calls.filter((call) => call.includes("/compare/"))).toEqual([ + `api repos/owner/repository/compare/${pair}?per_page=1 --jq .status`, + ]); + expect( + (yield* run(record(sha("b"), `main@${sha("6")}`, "3", tested), { + contains: { [pair]: "identical" }, + })).rule, + ).toBe("tested"); + // A merge commit that does not contain this layer's head is not built on it. + for (const status of ["behind", "diverged"]) { + const not = yield* run(record(sha("b"), `main@${sha("6")}`, "3", tested), { + contains: { [pair]: status }, + }); + expect(not).toMatchObject({ decision: "would fail", rule: "none" }); + expect(not.waited).toBeGreaterThanOrEqual(240); + expect(not.reason).toBe( + "#11's latest run on its head, https://github.com/owner/repository/actions/runs/600, was built on bbbbbbbbb and tested fffffffff, which does not contain this layer's head ccccccccc", + ); + expect(not.exitCode).toBe(0); + // Asked once, and remembered while the check waits. + expect(not.calls.filter((call) => call.includes("/compare/"))).toHaveLength(1); + } + // A compare that cannot be read is no answer: the check waits, asks again each look, and + // at its bound says it could not tell. Observing, the step still passes. + const unread = yield* run(record(sha("b"), `main@${sha("6")}`, "3", tested), { + contains: { [pair]: "fail" }, + }); + expect(unread).toMatchObject({ decision: "would fail", exitCode: 0 }); + expect(unread.reason).toContain( + "tested fffffffff, and whether that contains this layer's head ccccccccc could not be read", + ); + expect(unread.calls.filter((call) => call.includes("/compare/")).length).toBeGreaterThan(1); + // Its event's base naming this layer's head is enough, as in format 2, with no compare. + const based = yield* run(record(lowerHead, `main@${sha("6")}`, "3", tested)); + expect(based).toMatchObject({ decision: "would pass", rule: "base" }); + expect(based.calls.filter((call) => call.includes("/compare/"))).toEqual([]); + // A format 2 record names no tested commit: judged by its base alone, as before. + expect(yield* run(record(lowerHead, `main@${sha("6")}`, "2"))).toMatchObject({ + decision: "would pass", + rule: "base", + }); + expect( + (yield* run(record(sha("b"), `main@${sha("6")}`, "2"), { + contains: { [`${lowerHead}...${sha("b")}`]: "ahead" }, + })).reason, + ).toContain("was built on bbbbbbbbb, not this layer's head ccccccccc"); + // Containing this layer's head is not enough: the Worker sizes must have been compared with + // the base this stack ends on. + expect( + (yield* run(record(sha("b"), `effect-apps@${sha("5")}`, "3", tested), { + contains: { [pair]: "ahead" }, + })).reason, + ).toContain("compared its Worker sizes with effect-apps, not main"); + // The upper's re-run: its first record compared sizes with the wrong base; the re-run, + // which keeps the event's stale base, records again later and passes by what it tested. + const rerun = yield* run(record(sha("b"), `main@${sha("6")}`, "3", tested), { + records: { + "600": [ + { + id: 6001, + name: "ci-tested-pr11-first", + text: record(sha("b"), `effect-apps@${sha("5")}`, "3", tested), + }, + { + id: 6002, + name: "ci-tested-pr11-rerun", + text: record(sha("b"), `main@${sha("6")}`, "3", tested), + after: 3, + }, + ], + }, + contains: { [pair]: "ahead" }, + }); + expect(rerun).toMatchObject({ decision: "would pass", rule: "tested" }); + expect(rerun.waited).toBeGreaterThan(0); + }), + ); + + it.effect( + "the five would-fail decisions of 16 Nov, replayed through the rule that reads what was tested", + () => + Effect.gen(function* () { + // The heads and bases of the live runs; the merge commits the uppers checked out were not + // recorded then, so each upper's head stands for it (a merge commit contains its head). + const layer = (head: string, bottom = "effect-apps") => ({ + ...skipLayer(bottom), + HEAD_SHA: head, + }); + const upperOf = (head: string, text: string, more: GitHubAnswers = {}): GitHubAnswers => ({ + uppers: [{ number: 11, headRefOid: head }], + upperRuns: { [head]: [{ id: 600 }] }, + records: { "600": { name: "ci-tested-pr11-key", text } }, + ...more, + }); + const at = (base: string, tested: string, bottom: string) => + `format 3\npull request 11\nmerge tree ${sha("e")}\nbase ${base}\ntested ${tested}\nselection x\nworker-growth y\nbaseline ${bottom}@${sha("6")}\nabove none\n`; + const verdict = (env: Record, answers: GitHubAnswers) => + Effect.map(baseline(["above"], env, answers), (result) => decided(result.log)!); + // #4717 under #4718: the upper's first run recorded the lower's previous head 1b8f78414 as + // its base; its head 66f85fc61 contains the lower's head 4ad0afbc3 (git, offline). + const gatewayLower = "4ad0afbc33d43392c0bc2f42b2b9ba94b5e8cc46"; + const gatewayUpper = "66f85fc61cf5b4954e43441a7951f782f249efc6"; + expect( + yield* verdict( + layer(gatewayLower), + upperOf( + gatewayUpper, + at("1b8f78414ff4a924ff47bf329e5f2bda654a215a", gatewayUpper, "effect-apps"), + { contains: { [`${gatewayLower}...${gatewayUpper}`]: "ahead" } }, + ), + ), + ).toMatchObject({ decision: "would pass", rule: "tested" }); + // #4692 under #4652: the upper's run recorded effect-apps' head 958abd22a as its base; its + // head 4565abc0d contains the lower's head d2a3db73d (git, offline). + const scaleLower = "d2a3db73d8d8f4d2eb4e9bc069afe0a5dc4ad51c"; + const searchUpper = "4565abc0d02794b172a53b6eda419e3a2be5c999"; + expect( + yield* verdict( + layer(scaleLower), + upperOf( + searchUpper, + at("958abd22a4b73542c8260c28d3f14c3e1ed5232f", searchUpper, "effect-apps"), + { contains: { [`${scaleLower}...${searchUpper}`]: "ahead" } }, + ), + ), + ).toMatchObject({ decision: "would pass", rule: "tested" }); + // #4691 under #4692: #4692 is itself a skip layer; with its check passing it records + // `above pass`, and its base is #4691's head 6fcd6f179. + const grantLower = "6fcd6f179a7f94ce4754075d8183a33af14ddca8"; + expect( + yield* verdict( + layer(grantLower), + upperOf( + scaleLower, + at(grantLower, scaleLower, "effect-apps").replace("above none", "above pass"), + ), + ), + ).toMatchObject({ decision: "would pass", rule: "base" }); + // ... and with #4692's check failing, as it did, #4691 still fails: the cascade is kept. + expect( + (yield* verdict( + layer(grantLower), + upperOf( + scaleLower, + at(grantLower, scaleLower, "effect-apps").replace("above none", "above fail"), + ), + )).decision, + ).toBe("would fail"); + // #4592 under #4593: both of the upper's runs on its head failed in select and recorded + // nothing; the re-runs came later. That was right, and stays right. + const sqlLower = "7d64b6538eab858e3b9cd58f2826dbf3599a1a46"; + const sqlUpper = sha("9"); + const sql = yield* verdict(layer(sqlLower), { + uppers: [{ number: 11, headRefOid: sqlUpper }], + upperRuns: { + [sqlUpper]: [ + { id: 43928367877, conclusion: "failure" }, + { id: 43928367036, conclusion: "failure" }, + ], + }, + }); + expect(sql).toMatchObject({ decision: "would fail", rule: "none" }); + expect(sql.reason).toContain("recorded nothing (its select failed)"); + }), + ); + + it.effect( + "the check waits by what it sees, reads again only what changed, and takes a failed read for no answer", + () => + Effect.gen(function* () { + // The run above records after a few looks: waited for, and each look after the first asks + // GitHub only for what changed; the record is downloaded once. + const late = yield* baseline( + ["above"], + skipLayer(), + upper(good, { + upperRuns: { [upperHead]: [{ id: 600, status: "in_progress", conclusion: null }] }, + records: { "600": { name: "ci-tested-pr11-key", text: good, after: 2 } }, + }), + ); + expect(decided(late.log)).toMatchObject({ decision: "would pass" }); + expect(late.calls.filter((call) => call.includes("/pulls?base="))).toHaveLength(3); + expect(late.calls.filter((call) => call.includes("If-None-Match"))).not.toHaveLength(0); + expect(late.calls.filter((call) => call.startsWith("run download"))).toHaveLength(1); + expect(late.log).toMatch(/this check made \d+ counted requests and [1-9]\d* unchanged/); + // While the run above is still choosing, the wait is `selecting` (15 minutes), not the + // 4 of a stale record: a record at about 5 minutes still passes. + const choosing = { + upperRuns: { [upperHead]: [{ id: 600, status: "in_progress", conclusion: null }] }, + records: { "600": { name: "ci-tested-pr11-key", text: good, after: 7 } }, + }; + const slow = decided((yield* baseline(["above"], skipLayer(), upper(good, choosing))).log)!; + expect(slow).toMatchObject({ decision: "would pass" }); + expect(slow.waited).toBeGreaterThan(240); + // A run above that is itself waiting on the layer above it gets `chained` (20 minutes): a + // record at about 16 minutes still passes. + const chained = decided( + (yield* baseline( + ["above"], + skipLayer(), + upper(good, { + ...choosing, + records: { "600": { name: "ci-tested-pr11-key", text: good, after: 18 } }, + selectJobs: { "600": { status: "in_progress", waitingAbove: true } }, + }), + )).log, + )!; + expect(chained).toMatchObject({ decision: "would pass" }); + expect(chained.waited).toBeGreaterThan(900); + // ...and without it, the same record is too late. + expect( + decided( + (yield* baseline( + ["above"], + skipLayer(), + upper(good, { + ...choosing, + records: { "600": { name: "ci-tested-pr11-key", text: good, after: 18 } }, + }), + )).log, + ), + ).toMatchObject({ decision: "would fail" }); + // A stale record is waited on: a push to the layer above within the bound passes. + const pushed = yield* baseline( + ["above"], + skipLayer(), + upper(good, { + upperSnapshots: [ + [{ number: 11, headRefOid: upperHead }], + [{ number: 11, headRefOid: upperHead }], + [{ number: 11, headRefOid: sha("8") }], + ], + upperRuns: { [upperHead]: [{ id: 600 }], [sha("8")]: [{ id: 610 }] }, + records: { + "600": { name: "ci-tested-pr11-old", text: laundered }, + "610": { name: "ci-tested-pr11-new", text: good }, + }, + }), + ); + expect(decided(pushed.log)).toMatchObject({ decision: "would pass" }); + // A read that fails after its tries, or a download that fails, is no answer: the check + // looks again, and only the deadline decides. + const flaky = yield* baseline( + ["above"], + skipLayer(), + upper(good, { flaky: { "run download": 4 } }), + ); + expect(decided(flaky.log)).toMatchObject({ decision: "would pass" }); + expect(flaky.log).toContain("Waiting: the pull requests above could not be read"); + const down = decided( + (yield* baseline(["above"], skipLayer(), upper(good, { failPullRequests: true }))).log, + )!; + expect(down).toMatchObject({ decision: "would fail" }); + expect(down.reason).toContain("the pull requests above could not be read"); + expect(down.waited).toBeGreaterThanOrEqual(240); + }), + ); + + it.effect( + "no read can hang the check: each has its own time limit, and one that passes is no answer", + () => + Effect.gen(function* () { + // Stalled twice, then answering: the check looks again and passes. + const stalled = yield* baseline( + ["above"], + skipLayer(), + upper(good, { hangs: { "/pulls?base=": 2 } }), + ); + expect(decided(stalled.log)).toMatchObject({ decision: "would pass" }); + expect(stalled.log).toContain("Waiting: the pull requests above could not be read ("); + // The read that stalled is named: each has its own limit (20 s; 1 s in this spec). + expect(stalled.log).toContain( + "repos/owner/repository/pulls?base=lower&state=open&per_page=100: no answer within 1 s", + ); + // A download that stalls is no answer either. + const download = yield* baseline( + ["above"], + skipLayer(), + upper(good, { hangs: { "run download": 1 } }), + ); + expect(decided(download.log)).toMatchObject({ decision: "would pass" }); + // Reads that each answer in time but together outlast the look: the look itself has a limit + // (a minute; 2 s in this spec), and ends as no answer. + const slow = yield* baseline( + ["above"], + skipLayer(), + upper(good, { slow: { "/actions/runs": 700, "run download": 700 } }), + ); + expect(decided(slow.log)).toMatchObject({ decision: "would fail" }); + expect(decided(slow.log)?.reason).toContain("the look got no answer within 2 s"); + // Never answering: the deadline decides, and the step ends. + const never = yield* baseline( + ["above"], + skipLayer(), + upper(good, { hangs: { "/pulls?base=": 1000 } }), + ); + expect(never.exitCode).toBe(0); + expect(decided(never.log)).toMatchObject({ decision: "would fail" }); + expect(decided(never.log)?.reason).toContain("no answer within"); + }), + ); + + it.effect( + "observing, nothing in the check can fail the step; enforcing, a defect still fails it", + () => + Effect.gen(function* () { + // A judge that throws: an error outcome while observing, a failure while enforcing. + const lower = { + ref: "lower", + head: lowerHead, + bottom: "main", + repository: "o/r", + runId: "5", + }; + const throwing = Effect.sync(() => + skipCheck.next( + skipCheck.judge( + { _tag: "Read", uppers: null } as unknown as skipCheck.Observation, + lower, + ), + lower, + 0, + 0, + ), + ).pipe( + Effect.map((step) => { + if (step._tag !== "Done") throw new Error("not done"); + return step.decision; + }), + ); + const observed = yield* skipCheck.guarded(throwing, false); + expect(observed._tag).toBe("Error"); + expect(skipCheck.fails(observed, false)).toBe(false); + expect(skipCheck.fails(observed, true)).toBe(true); + expect(skipCheck.decisionLine(observed, false)).toMatch( + /^skip-check \{"mode":"observe","decision":"error","reason":"TypeError: .*"\}$/, + ); + expect(skipCheck.summarySentence(observed, false)).toContain("It passes while observing."); + const enforced = yield* Effect.exit(skipCheck.guarded(throwing, true)); + expect(enforced._tag).toBe("Failure"); + // In the step: a missing variable, and a summary that cannot be written, leave it passing + // with the decision line. + const missing = yield* baseline(["above"], { ...skipLayer(), HEAD_SHA: "" }, upper(good)); + expect(missing.exitCode, missing.log).toBe(0); + expect(decided(missing.log)).toMatchObject({ mode: "observe", decision: "error" }); + expect(missing.outputs.decision).toBe("error"); + const unwritable = yield* baseline( + ["above"], + { ...skipLayer(), GITHUB_STEP_SUMMARY: "/" }, + upper(good), + ); + expect(unwritable.exitCode, unwritable.log).toBe(0); + expect(decided(unwritable.log)).toMatchObject({ decision: "would pass" }); + expect(unwritable.log).toContain("The skip check's report could not be written"); + }), + ); + + it.effect( + "select's baseline step cannot fail select: whatever goes wrong, it compares with main", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const checks = yield* fs.readFileString(".github/workflows/checks.yml"); + const select = checks.slice(checks.indexOf(" select:"), checks.indexOf("\n static:")); + const step = select.slice(select.indexOf(" id: baseline\n")); + const script = step + .slice(step.indexOf(" run: |\n") + " run: |\n".length) + .split("\n") + .filter((line, index, lines) => + lines.slice(0, index + 1).every((each) => each.startsWith(" ")), + ) + .map((line) => line.slice(10)) + .join("\n"); + expect(script).toContain("if ! node scripts/ci/worker-baseline.ts resolve; then"); + const run = (env: Readonly>) => + Effect.gen(function* () { + const directory = yield* fs.makeTempDirectoryScoped({ + prefix: "executor-baseline-step-", + }); + const output = path.join(directory, "output"); + yield* fs.writeFileString(output, ""); + const child = yield* processes.spawn( + ChildProcess.make("bash", ["-e", "-o", "pipefail", "-c", script], { + env: { DEFAULT_BRANCH: "main", GITHUB_OUTPUT: output, ...env }, + extendEnv: true, + }), + ); + const [log, exitCode] = yield* Effect.all( + [child.all.pipe(Stream.decodeText(), Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + return { log, exitCode: Number(exitCode), output: yield* fs.readFileString(output) }; + }).pipe(Effect.scoped); + // The script crashes (here: a variable it needs is missing). + const crashed = yield* run({ + GITHUB_REPOSITORY: "owner/repository", + BASE_REF: "effect-apps", + HEAD_SHA: head, + BASE_SHA: "", + PATH: process.env["PATH"] ?? "", + }); + expect(crashed.exitCode, crashed.log).toBe(0); + expect(crashed.log).toContain( + "::warning title=Worker sizes::The baseline for Worker sizes could not be chosen (see this step's log), so this run compares with main.", + ); + expect(crashed.output).toMatch(/branch=main\nid=main@unread\nlabel=main\n$/); + }), + ); + + it.effect( + "on a push to main, site's baseline steps cannot fail site; on a pull request they fail closed", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const checks = yield* fs.readFileString(".github/workflows/checks.yml"); + const site = checks.slice( + checks.indexOf("\n site:\n"), + checks.indexOf("\n apps-package:\n"), + ); + const body = (step: string) => { + const text = site.slice(site.indexOf(` - name: ${step}\n`)); + return text + .slice(text.indexOf(" run: |\n") + " run: |\n".length) + .split("\n") + .filter((line, index, lines) => + lines + .slice(0, index + 1) + .every((each) => each.startsWith(" ") || each === ""), + ) + .map((line) => line.slice(10)) + .join("\n"); + }; + for (const step of [ + "Choose the baseline for Worker sizes", + "Fetch the baseline's Worker sizes", + ]) { + const script = body(step); + expect(script, step).toContain("node scripts/ci/worker-baseline.ts"); + const run = (event: string) => + Effect.gen(function* () { + const directory = yield* fs.makeTempDirectoryScoped({ + prefix: "executor-site-step-", + }); + // A node that crashes, as any defect in the script would. + const bin = path.join(directory, "bin"); + yield* fs.makeDirectory(bin); + yield* fs.writeFileString( + path.join(bin, "node"), + "#!/bin/sh\necho crashed >&2\nexit 7\n", + ); + yield* fs.chmod(path.join(bin, "node"), 0o755); + // What a partial download left behind. + yield* fs.makeDirectory(path.join(directory, ".local/worker-uploads-base"), { + recursive: true, + }); + yield* fs.writeFileString( + path.join(directory, ".local/worker-uploads-base/api.json"), + "{", + ); + const child = yield* processes.spawn( + ChildProcess.make("bash", ["-e", "-o", "pipefail", "-c", script], { + cwd: directory, + env: { PATH: `${bin}:${process.env["PATH"] ?? ""}`, GITHUB_EVENT_NAME: event }, + extendEnv: true, + }), + ); + const [log, exitCode] = yield* Effect.all( + [child.all.pipe(Stream.decodeText(), Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + return { + log, + exitCode: Number(exitCode), + left: yield* fs.exists(path.join(directory, ".local/worker-uploads-base")), + }; + }).pipe(Effect.scoped); + const push = yield* run("push"); + expect(push.exitCode, `${step}: ${push.log}`).toBe(0); + expect(push.log, step).toContain("so this push's growth is reported and not checked."); + // A fetch that failed leaves no half-fetched sizes to compare with. + if (step === "Fetch the baseline's Worker sizes") expect(push.left).toBe(false); + const pull = yield* run("pull_request"); + expect(pull.exitCode, step).toBe(1); + } + }), + ); + + /** The run: body of a named step in a workflow's text, dedented. */ + const stepScript = (workflow: string, step: string) => { + const text = workflow.slice(workflow.indexOf(` - name: ${step}\n`)); + return text + .slice(text.indexOf(" run: |\n") + " run: |\n".length) + .split("\n") + .filter((line, index, lines) => + lines.slice(0, index + 1).every((each) => each.startsWith(" ") || each === ""), + ) + .map((line) => line.slice(10)) + .join("\n"); + }; + const timingStep = "Time the Workers' start-up against the previous push"; + + it.effect("the start-up record runs only on a push, and nothing in it can fail site", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const checks = yield* fs.readFileString(".github/workflows/checks.yml"); + const site = checks.slice( + checks.indexOf("\n site:\n"), + checks.indexOf("\n apps-package:\n"), + ); + const from = site.indexOf(" - name: Keep this push's built Workers for the next push\n"); + expect(from).toBeGreaterThan(-1); + const steps = site + .slice(from) + .split(/\n(?= {6}- )/) + .filter((step) => step.includes("worker-build") || step.includes("startup")); + expect(steps.length).toBe(5); + for (const step of steps) { + const condition = step.match(/^(?: {6}- | {8})if: (.*)$/m)?.[1] ?? ""; + // On a pull request: skipped, by the event or by an output only a push's step sets. + expect( + condition.includes("github.event_name == 'push'") || + /steps\.(build-kept|startup)\.outputs\.path != ''/.test(condition), + step.slice(0, 80), + ).toBe(true); + expect(condition, step.slice(0, 80)).toContain("!cancelled()"); + expect(step, step.slice(0, 80)).toContain(" continue-on-error: true\n"); + expect(step, step.slice(0, 80)).toMatch(/^ {8}timeout-minutes: \d+$/m); + } + // site's own limit leaves room for every step's: the build takes about a minute today. + const limit = Number(site.match(/^ {4}timeout-minutes: (\d+)$/m)?.[1]); + const steps_ = steps.map((step) => Number(step.match(/^ {8}timeout-minutes: (\d+)$/m)?.[1])); + expect(limit - steps_.reduce((sum, minutes) => sum + minutes, 0)).toBeGreaterThanOrEqual(0); + // The measurement is stopped, workerd with it, before its step's own limit. + expect(stepScript(site, timingStep)).toContain('timeout -s KILL "$LIMIT" node "$HARNESS"'); + expect(site).toContain(' LIMIT: "360"\n'); + }), + ); + + it.effect( + "a crash, a hang and a missing previous build each leave a notice; a record names what it compared", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const script = stepScript( + yield* fs.readFileString(".github/workflows/checks.yml"), + timingStep, + ); + const head = sha("c"); + const before = sha("b"); + const measured = { + workerd: "1.20260918.1", + runs: 21, + machine: { cpus: 4, loadBefore: [1, 1, 1], loadAfter: [1, 1, 1] }, + skipped: {}, + workers: { + api: { + staticModules: { base: 23, head: 24 }, + staticBytes: { base: 3_750_524, head: 3_800_524 }, + compile: { base: { median: 80, min: 78 }, head: { median: 82, min: 80 } }, + evaluate: { base: { median: 230, min: 226 }, head: { median: 236, min: 231 } }, + }, + }, + }; + const run = (harness: string, env: Readonly>) => + Effect.gen(function* () { + const directory = yield* fs.makeTempDirectoryScoped({ + prefix: "executor-startup-step-", + }); + const stand = path.join(directory, "harness.mjs"); + yield* fs.writeFileString(stand, harness); + yield* fs.makeDirectory(path.join(directory, "base", "bundles"), { recursive: true }); + const output = path.join(directory, "output"); + yield* fs.writeFileString(output, ""); + const started = Date.now(); + const child = yield* processes.spawn( + ChildProcess.make("bash", ["-e", "-o", "pipefail", "-c", script], { + env: { + GITHUB_EVENT_NAME: "push", + RUNNER_TEMP: directory, + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: path.join(directory, "summary"), + HEAD_SHA: head, + BEFORE_SHA: before, + BASE_COMMIT: before, + BASE_BUILD: path.join(directory, "base"), + HARNESS: stand, + LIMIT: "2", + ARGS_FILE: path.join(directory, "args"), + ...env, + }, + extendEnv: true, + }), + ); + const [log, exitCode] = yield* Effect.all( + [child.all.pipe(Stream.decodeText(), Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + const record = path.join(directory, "worker-startup", "record", "worker-startup.json"); + return { + log, + exitCode: Number(exitCode), + seconds: (Date.now() - started) / 1000, + outputs: yield* fs.readFileString(output), + record: (yield* fs.exists(record)) + ? (JSON.parse(yield* fs.readFileString(record)) as Record) + : undefined, + args: (yield* fs.exists(path.join(directory, "args"))) + ? yield* fs.readFileString(path.join(directory, "args")) + : "", + base: path.join(directory, "base"), + }; + }).pipe(Effect.scoped); + const printing = `import fs from "node:fs";\nfs.writeFileSync(process.env.ARGS_FILE, process.argv.slice(2).join(" "));\nconsole.log(${JSON.stringify(JSON.stringify(measured))});\n`; + // The previous build was found: the record names both commits and the difference. + const compared = yield* run(printing, {}); + expect(compared.exitCode, compared.log).toBe(0); + expect(compared.args).toBe(`--base ${compared.base} --head apps/hosted/cloud`); + expect(compared.record).toMatchObject({ + commit: head, + before, + base: before, + workers: { + api: { + staticBytes: { base: 3_750_524, head: 3_800_524 }, + evaluate: { + base: { median: 230, min: 226 }, + head: { median: 236, min: 231 }, + difference: 6, + }, + }, + }, + }); + expect(compared.outputs).toMatch(/^path=.*\/worker-startup\/record$/m); + // No previous build: the head is measured against itself and recorded alone. + const alone = yield* run(printing, { BASE_COMMIT: "" }); + expect(alone.exitCode, alone.log).toBe(0); + expect(alone.args).toBe("--base apps/hosted/cloud --head apps/hosted/cloud"); + expect(alone.record).toMatchObject({ + commit: head, + base: null, + workers: { api: { evaluate: { base: null, difference: null } } }, + }); + // A crash: a notice, no record, the step passes. + const crashed = yield* run("process.exit(3);\n", {}); + expect(crashed.exitCode, crashed.log).toBe(0); + expect(crashed.log).toContain( + "::notice title=Worker start-up::The start-up measurement did not finish (exit 3, 137 when its 2-second limit stopped it", + ); + expect(crashed.record).toBeUndefined(); + expect(crashed.outputs).not.toContain("path="); + // A hang: stopped at its limit, with what it started, and the step passes. + const hung = yield* run( + `import { spawn } from "node:child_process";\nspawn("sleep", ["637"], { stdio: "ignore" });\nsetInterval(() => {}, 1000);\n`, + {}, + ); + expect(hung.exitCode, hung.log).toBe(0); + expect(hung.log).toContain("did not finish (exit 137"); + expect(hung.seconds).toBeLessThan(15); + // What it started is stopped with it (timeout -s KILL signals the whole group). + const left = yield* processes.spawn(ChildProcess.make("pgrep", ["-f", "sleep 637"])); + const [found] = yield* Effect.all( + [left.stdout.pipe(Stream.decodeText(), Stream.mkString), left.exitCode], + { concurrency: "unbounded" }, + ); + expect(found.trim()).toBe(""); + expect(hung.record).toBeUndefined(); + }), + 20_000, + ); + + it.effect( + "the previous push's build is used only from main's push run of the commit it names", + () => + Effect.gen(function* () { + const env = { BASE_SHA: sha("a") }; + const kept = { "worker-build": { api: { bytes: 1 } } }; + const found = yield* baseline(["fetch-build", ""], env, { + runs: { [sha("a")]: "88" }, + downloads: kept, + builtFrom: { "worker-build": sha("a") }, + }); + expect(found.exitCode, found.log).toBe(0); + expect(found.outputs).toEqual({ found: "true", commit: sha("a") }); + expect(found.calls.find((call) => call.startsWith("run list"))).toContain("--event push"); + // An artifact that names another commit, or none, is not used. + for (const named of [sha("9"), null]) { + const other = yield* baseline(["fetch-build", ""], env, { + runs: { [sha("a")]: "88" }, + downloads: kept, + builtFrom: { "worker-build": named }, + }); + expect(other.outputs, String(named)).toEqual({ found: "false", commit: "" }); + expect(other.sizes, String(named)).toEqual([]); + expect(other.log).toContain("so it is not used."); + } + // A release commit has no run: the build of the commit before it. + const stepped = yield* baseline(["fetch-build", ""], env, { + commits: { + [sha("a")]: { + message: "Release 2.0.0-beta.56 [skip ci]", + author: "github-actions[bot]", + parents: [sha("b")], + files: releaseFiles("56"), + }, + }, + runs: { [sha("b")]: "87" }, + downloads: kept, + builtFrom: { "worker-build": sha("b") }, + }); + expect(stepped.outputs).toEqual({ found: "true", commit: sha("b") }); + // None: recorded alone, with a notice, and the step goes on. + const none = yield* baseline(["fetch-build", ""], env, {}); + expect(none.exitCode).toBe(0); + expect(none.outputs).toEqual({ found: "false", commit: "" }); + expect(none.log).toContain("so this push's start-up is recorded alone."); + }), + ); + + it.effect("one constant decides observe or enforce, and the bounds compose", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + // Observing until a day of decisions has been counted; flipping it is a one-line change. + expect(skipCheck.enforce).toBe(false); + const lower = { + ref: "lower", + head: lowerHead, + bottom: "main", + repository: "o/r", + runId: "5", + }; + const stale = skipCheck.judge( + { + _tag: "Read", + uppers: [ + { + pull: 11, + head: upperHead, + runs: [ + { + id: 1, + url: "u", + record: skipCheck.parseRecord(laundered), + selectDone: true, + waitingAbove: false, + }, + ], + }, + ], + }, + lower, + ); + // A stale record waits its bound for a push or a re-run, and only then fails. + expect(skipCheck.next(stale, lower, 239, 9)._tag).toBe("Poll"); + const done = skipCheck.next(stale, lower, 240, 9); + expect(done._tag).toBe("Done"); + if (done._tag !== "Done") return; + expect(done.decision.decision).toBe("fail"); + // Enforcing fails the step on a decision to fail; observing never does; a pass never does. + const decided = { _tag: "Decided" as const, decision: done.decision }; + expect(skipCheck.fails(decided, true)).toBe(true); + expect(skipCheck.fails(decided, false)).toBe(false); + expect( + skipCheck.fails( + { _tag: "Decided", decision: { ...done.decision, decision: "pass" } }, + true, + ), + ).toBe(false); + // A failed read keeps the bound of the last answer. + expect( + skipCheck.judge({ _tag: "Unread", why: "x" }, lower, { + _tag: "Wait", + bound: "selecting", + why: "y", + }), + ).toMatchObject({ _tag: "Wait", bound: "selecting" }); + // No look sooner than 20 seconds after the last, backing off to a minute. + const polls = Array.from({ length: 30 }, (_, index) => skipCheck.pollAfter(index)); + expect(polls.slice(0, 5)).toEqual([20, 30, 45, 60, 60]); + expect(Math.min(...polls)).toBeGreaterThanOrEqual(20); + // The bounds, and how they compose: a layer under a waiting layer waits five polls longer + // than that layer can wait for the top, and select has room for the longest wait after the + // description's 3 minutes. + expect(skipCheck.bounds).toEqual({ stale: 240, noRun: 240, selecting: 900, chained: 1200 }); + expect(skipCheck.bounds.chained - skipCheck.bounds.selecting).toBeGreaterThanOrEqual(5 * 60); + const checks = yield* fs.readFileString(".github/workflows/checks.yml"); + const select = checks.slice(checks.indexOf(" select:"), checks.indexOf("\n static:")); + const timeout = Number(select.match(/^ {4}timeout-minutes: (\d+)$/m)?.[1]); + expect(timeout * 60).toBeGreaterThanOrEqual(180 + skipCheck.bounds.chained + 5 * 60); + // The record format the workflow writes is the one the check reads. + expect(select).toContain(` RECORD_FORMAT: "${skipCheck.recordFormat}"\n`); + }), + ); + + it.effect( + "the skip layer's check runs before its record, and a failed check hands no skip on", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const checks = yield* fs.readFileString(".github/workflows/checks.yml"); + expect(checks).toContain( + " skip: ${{ steps.repeat.outputs.of == '' && steps.above.outcome == 'success' && steps.select.outputs.skip || '' }}\n", + ); + const select = checks.slice(checks.indexOf(" select:"), checks.indexOf("\n static:")); + const above = select.indexOf(" id: above\n"); + expect(above).toBeGreaterThan(select.indexOf(" id: baseline\n")); + expect(above).toBeLessThan(select.indexOf(" id: tested\n")); + expect(select).toContain( + " if: github.event_name == 'pull_request' && steps.select.outputs.skip == 'true'\n", + ); + expect(select).toContain( + " BASELINE_BRANCH: ${{ steps.baseline.outputs.branch }}\n", + ); + expect(select).toContain(" run: node scripts/ci/worker-baseline.ts above\n"); + // The record runs only when every step before it succeeded (no status function in its + // condition), so enforcing, a skip layer whose check failed writes none, and the layer + // below it finds nothing to pass on. It carries the format and the check's decision. + const tested = select.slice(select.indexOf(" id: tested\n")); + const condition = tested.match(/^ {8}if: (.*)$/m)?.[1] ?? ""; + expect(condition).toBe( + "github.event_name == 'pull_request' && steps.select.outputs.selection != '' && steps.select.outputs.worker-growth != ''", + ); + expect(tested).toContain( + " ABOVE: ${{ steps.above.outputs.decision || 'none' }}\n", + ); + // What the run tested, for the layer below: the merge commit this checkout holds. + expect(tested).toContain(" tested=$(git rev-parse HEAD)\n"); + expect(select).toContain( + `printf 'format %s\\npull request %s\\nmerge tree %s\\nbase %s\\ntested %s\\nselection %s\\nworker-growth %s\\nbaseline %s\\nabove %s\\n' "$RECORD_FORMAT" "$PULL" "$tree" "$BASE_SHA" "$tested" "$SELECTION" "$WORKER_GROWTH" "$BASELINE" "$ABOVE"`, + ); + // A recorded build names its commit. + expect(checks).toContain( + 'git rev-parse HEAD > "$GITHUB_WORKSPACE/.local/worker-uploads-base/baseline-commit.txt"', + ); + }), + ); + + /** A push to main that grows api by `added` bytes over the commit before it. */ + const pushed = (added: number, merged?: ReadonlyArray<{ number: number; body: string }>) => + check({ + uploads: { ...main, api: { bytes: main.api!.bytes + added, budget: 6_000_000 } }, + baseline: "main at 82eda7c59", + ...(merged === undefined ? {} : { merged }), + }); + const allows = (line: string) => `Adds a route.\n\n\`\`\`worker-growth\n${line}\n\`\`\`\n`; + + it.effect( + "a push to main is held to the rule, each Worker allowed what the pull requests it merges allow", + () => + Effect.gen(function* () { + // Growth within the rule passes; growth past it passes when a merged description allows it. + expect((yield* pushed(50_000, [{ number: 1, body: "No block." }])).exitCode).toBe(0); + const allowed = yield* pushed(60_000, [ + { number: 4344, body: allows("api +60,000: v1's MCP URLs") }, + ]); + expect(allowed.exitCode, allowed.log).toBe(0); + expect(allowed.summary).toContain("Allowed up to 60,000: #4344: v1's MCP URLs"); + // A stack merge pushes several: their allowances add up, by Worker. + const stack = yield* pushed(90_000, [ + { number: 2, body: allows("api +40,000: the first half") }, + { number: 3, body: allows("api +50,000: the second half") }, + ]); + expect(stack.exitCode, stack.log).toBe(0); + // Unallowed growth fails, naming the Worker, the bytes, the pull requests and what to do. + const grew = yield* pushed(60_001, [{ number: 4583, body: "No block." }]); + expect(grew.exitCode).toBe(1); + expect(grew.errors).toEqual([ + "::error title=api Worker grew on main without an allowance::This push adds +60,001 bytes to api over main at 82eda7c59, about 2 ms on each cold start; the pull requests it merges (#4583) allow none. Put the line `api +60,001: ` in a worker-growth block of the merged pull request's description, with your reason, and run this failed job again, which reads the descriptions afresh; or shrink api in a new pull request. See the job summary for what grew.", + ]); + // Too small an allowance, and a direct push that merges no pull request. + expect( + (yield* pushed(70_000, [{ number: 5, body: allows("api +60,000: most of it") }])) + .errors[0], + ).toContain("the pull requests it merges (#5) allow 60,000."); + expect((yield* pushed(60_001, [])).errors[0]).toContain( + "it merges no pull request, so no description allows it.", + ); + }), + ); + + it.effect("a push to main never fails for a reason that is not growth", () => + Effect.gen(function* () { + // The commit before has no sizes: no change is shown, nothing fails. + const missing = yield* check({ + uploads: { ...main, api: { bytes: main.api!.bytes + 200_000, budget: 6_000_000 } }, + base: "missing", + merged: [], + }); + expect(missing.exitCode, missing.log).toBe(0); + // A revert shrinks. + expect((yield* pushed(-120_000, [])).exitCode).toBe(0); + // A merged description's allowance this cannot read is a notice on main. + const unread = yield* pushed(10_000, [{ number: 6, body: allows("api: no size") }]); + expect(unread.exitCode, unread.log).toBe(0); + expect(unread.log).toContain( + "::notice title=Worker sizes::A merged pull request's allowance: `#6: api: no size` is not a Worker, a size and a reason", + ); + // The pull requests could not be read: no --merged, so growth is reported only, as before. + expect((yield* pushed(200_000)).exitCode).toBe(0); + }), + ); + + it.effect( + "the merged pull requests are read from the push's commits, and nothing unread fails the job", + () => + Effect.gen(function* () { + const env = { BEFORE_SHA: sha("1"), HEAD_SHA: sha("2") }; + const answers: GitHubAnswers = { + pushCommits: [sha("3"), sha("4"), sha("5")], + commitPulls: { [sha("3")]: [4574], [sha("4")]: [4575] }, + pullBodies: { "4574": "skip layer", "4575": allows("api +60,000: the reason") }, + }; + const read = yield* baseline(["merged", ""], env, answers); + expect(read.exitCode, read.log).toBe(0); + expect(read.outputs).toMatchObject({ read: "true", pulls: "#4574, #4575" }); + expect(JSON.parse(read.file!)).toEqual([ + { number: 4574, body: "skip layer" }, + { number: 4575, body: allows("api +60,000: the reason").trim() }, + ]); + expect(read.log).toContain( + "::notice title=Worker sizes::555555555 is a commit no pull request merged, so it allows no growth.", + ); + // GitHub unreadable after its retries: no file, a notice, and the job goes on. + const down = yield* baseline(["merged", ""], env, { ...answers, failPush: true }); + expect(down.exitCode, down.log).toBe(0); + expect(down.outputs).toEqual({ read: "false" }); + expect(down.file).toBeUndefined(); + expect(down.log).toContain("could not be read, so its growth is reported and not checked"); + // A body that cannot be read is the same. + const body = yield* baseline(["merged", ""], env, { + ...answers, + pullBodies: { "4574": "skip layer" }, + }); + expect(body.outputs).toEqual({ read: "false" }); + // The workflow: on a push only, its failure never fails site, and the gate reads its file. + const fs = yield* FileSystem.FileSystem; + const checks = yield* fs.readFileString(".github/workflows/checks.yml"); + expect(checks).toContain( + " if: github.event_name == 'push'\n env:\n GH_TOKEN: ${{ github.token }}\n BEFORE_SHA: ${{ github.event.before }}\n HEAD_SHA: ${{ github.sha }}\n run: |\n if ! node scripts/ci/worker-baseline.ts merged \"$RUNNER_TEMP/merged-pull-requests.json\"; then\n", + ); + expect(checks).toContain(" MERGED: ${{ steps.merged.outputs.file }}\n"); + expect(checks).toContain('${MERGED:+--merged "$MERGED"}'); + // And CI takes no `edited` events: a description edit starts nothing. + const ci = yield* fs.readFileString(".github/workflows/ci.yml"); + expect(ci).not.toContain("edited"); + expect(ci).not.toContain("run-name"); + }), + ); + + /** + * The 40 lower-layer pushes of 15 and 16 Nov (e2e/skip-check-replay.json): what each layer + * above showed over time, from GitHub's runs, jobs and records, in seconds from the moment the + * lower layer's check would have started. Each is replayed through the rule as the step runs it. + */ + it.effect("the last 40 lower-layer pushes, replayed", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const pushes = JSON.parse( + yield* fs.readFileString("e2e/skip-check-replay.json"), + ) as ReadonlyArray; + const results = Object.fromEntries( + pushes.map((push) => [`#${push.lower} ${push.run}/${push.attempt}`, replay(push)]), + ); + expect(results).toEqual(replayed); + }), + ); +}); + +interface Push { + readonly lower: number; + readonly run: number; + readonly attempt: number; + readonly ref: string; + readonly head: string; + readonly bottom: string; + readonly uppers: ReadonlyArray<{ readonly pull: number; readonly head: string }>; + readonly runs: ReadonlyArray<{ + readonly id: number; + readonly pull: number; + readonly head: string; + readonly fork?: boolean; + readonly cancelledAt?: number; + readonly phases: ReadonlyArray<{ + readonly at: number; + readonly selectDone: boolean; + readonly record?: string; + }>; + }>; +} + +/** What the rule decides for a push, looking when the step would and seeing what was there then. */ +const replay = (push: Push) => { + const lower = { + ref: push.ref, + head: push.head, + bottom: push.bottom, + repository: "UsefulSoftwareCo/executor-next", + runId: String(push.run), + }; + const look = (at: number): skipCheck.Observation => ({ + _tag: "Read", + uppers: push.uppers.map((upper) => ({ + pull: upper.pull, + head: upper.head, + runs: push.runs + .filter( + (run) => + run.pull === upper.pull && + run.head === upper.head && + run.fork !== true && + run.phases[0]!.at <= at && + (run.cancelledAt === undefined || run.cancelledAt > at), + ) + .toSorted((a, b) => b.id - a.id) + .map((run) => { + const phase = run.phases.findLast((each) => each.at <= at)!; + return { + id: run.id, + url: String(run.id), + record: phase.record === undefined ? undefined : skipCheck.parseRecord(phase.record), + selectDone: phase.selectDone, + waitingAbove: false, + }; + }), + })), + }); + let previous: skipCheck.Verdict | undefined; + for (let at = 0, polls = 0; ; polls += 1) { + const verdict = skipCheck.judge(look(at), lower, previous); + previous = verdict; + const step = skipCheck.next(verdict, lower, at, polls); + if (step._tag === "Done") + return `${step.decision.decision === "pass" ? (step.decision.oldFormat ? "old format" : "pass") : "fail"} at ${Math.round(at)} s`; + at += step.after; + } +}; + +/** + * What the replay decides, push by push (pull request, run and attempt). 37 uppers wrote main's + * old format and pass with the notice. #4574's upper at 03:16 wrote this check's record and passes. + * #4586 is the laundering case (its upper compared with the scratch branch it was based on) and + * fails. #4565 fails: its upper #4566's first select failed (a deployed-stage line was missing from + * its description), and only a re-run nine minutes later recorded; nothing above had chosen + * anything when the lower would have been checked. + */ +const replayed: Record = { + "#4364 43793324814/1": "old format at 20 s", + "#4364 43799666868/2": "old format at 20 s", + "#4436 43768234280/1": "old format at 20 s", + "#4436 43790114596/1": "old format at 20 s", + "#4437 43768232548/1": "old format at 20 s", + "#4437 43768233846/1": "old format at 0 s", + "#4437 43790115295/1": "old format at 20 s", + "#4437 43790115574/1": "old format at 0 s", + "#4438 43768232720/1": "old format at 20 s", + "#4438 43768233035/1": "old format at 0 s", + "#4526 43764086735/1": "old format at 20 s", + "#4526 43796363272/1": "old format at 20 s", + "#4535 43764086047/1": "old format at 20 s", + "#4535 43764086626/1": "old format at 0 s", + "#4535 43793228808/1": "old format at 20 s", + "#4535 43793228984/1": "old format at 0 s", + "#4535 43796363943/1": "old format at 20 s", + "#4535 43796364777/1": "old format at 0 s", + "#4543 43771845869/1": "old format at 20 s", + "#4544 43771846274/2": "old format at 0 s", + "#4544 43771846802/3": "old format at 0 s", + "#4548 43771846978/1": "old format at 20 s", + "#4548 43771847447/1": "old format at 0 s", + "#4551 43771845232/1": "old format at 20 s", + "#4551 43771846053/1": "old format at 0 s", + "#4557 43764101615/1": "old format at 20 s", + "#4557 43793229529/1": "old format at 20 s", + "#4557 43793230202/1": "old format at 0 s", + "#4557 43796364217/1": "old format at 20 s", + "#4559 43770386335/1": "old format at 50 s", + "#4562 43771920578/1": "old format at 20 s", + "#4562 43783348587/1": "old format at 20 s", + "#4565 43779277043/1": "fail at 240 s", + "#4568 43781460811/1": "old format at 20 s", + "#4568 43794794693/1": "old format at 20 s", + "#4574 43786864732/1": "old format at 20 s", + "#4574 43788608204/1": "old format at 20 s", + "#4574 43797130175/1": "pass at 20 s", + "#4582 43793923058/1": "old format at 20 s", + "#4586 43795625424/1": "fail at 240 s", +}; diff --git a/e2e/tests/workflow-caches.spec.ts b/e2e/tests/workflow-caches.spec.ts index 42966d0727..1034630a5e 100644 --- a/e2e/tests/workflow-caches.spec.ts +++ b/e2e/tests/workflow-caches.spec.ts @@ -414,19 +414,18 @@ const untrustedRestores = ( .map((cache) => `${file}: ${cache}`); }); -/** Commands that bring a ref's code, or a file from another run or release, onto the runner. */ +/** + * Commands that bring a ref's code, or a file from another run or release, onto the runner: in a + * shell, and in a TypeScript script as the arguments it passes to `gh` or `git`. + */ const fetching = - /\bgit(?: -[cC] \S+)* (fetch|checkout|clone|worktree)\b|\bgh (pr checkout|repo clone|run download|release download)\b/g; + /\bgit(?: -[cC] \S+)* (fetch|checkout|clone|worktree)\b|\bgh (pr checkout|repo clone|run download|release download)\b|\bgh\(\[\s*"(?:pr",\s*"checkout|repo",\s*"clone|run",\s*"download|release",\s*"download)"|"git",\s*\[\s*(?:"-[cC]",\s*[^,\]]+,\s*)*"(?:fetch|checkout|clone|worktree)"/g; /** * Each fetch in a `run:` step, or in a script under `scripts/` that one runs, that can run for an * event other than a pull request's own, with why what it fetches is not code from another ref. */ const fetchesAllowed: ReadonlyArray<{ readonly at: string; readonly why: string }> = [ - { - at: ".github/workflows/checks.yml gh run download", - why: "main's Worker sizes, a JSON file read as numbers to show a change", - }, { at: ".github/workflows/release-artifacts.yml git clone", why: "the Homebrew tap, whose formula file is rewritten and pushed; nothing in it runs", @@ -443,6 +442,10 @@ const fetchesAllowed: ReadonlyArray<{ readonly at: string; readonly why: string at: 'scripts/export-public.sh git -C "$target" fetch', why: "a ported contributor head, as a parent commit only: its files are never written or run", }, + { + at: 'scripts/ci/worker-baseline.ts gh(["run", "download"', + why: "Worker sizes, JSON the size gate reads and never runs: main's only from main's own push runs, a branch's only from CI runs of this repository's own pull requests whose sizes name the commit asked for; and, for a skip layer, the record of the run above, text read for two fields", + }, { at: "scripts/releases/bun-runtime/fetch.sh gh release download", why: "the Bun runtime, which runs, but only once it matches the digest pinned on main", @@ -454,7 +457,10 @@ const fetchesAllowed: ReadonlyArray<{ readonly at: string; readonly why: string * `scripts/releases/background.sh start ` starts in the background. */ const invoked = - /^\s*(?:- run: |run: )?(?:(?:\.\/)?scripts\/releases\/background\.sh start \S+ )?(?:bash )?(?:\.\/)?(scripts\/[\w./-]+\.sh)\b/gm; + /^\s*(?:- run: |run: )?(?:(?:\.\/)?scripts\/releases\/background\.sh start \S+ )?(?:bash )?(?:\.\/)?(scripts\/[\w./-]+\.sh)\b|\bnode (?:\.\/)?(scripts\/[\w./-]+\.ts)\b/gm; + +/** The script a match of `invoked` names, shell or TypeScript. */ +const scriptOf = (match: RegExpMatchArray) => (match[1] ?? match[2])!; /** A shell text with its continued lines joined, so `git \` and `fetch` on the next line match. */ const joined = (text: string) => text.replace(/[ \t]*\\\n\s*/g, " "); @@ -472,10 +478,12 @@ const codeFetches = (files: ReadonlyArray, scripts: ReadonlyArray) = if (start === -1) return []; const run = step.text.slice(start); const called = [...run.matchAll(invoked)].flatMap((match) => - scripts.filter((script) => script.file === match[1]), + scripts.filter((script) => script.file === scriptOf(match)), ); return [{ file: step.file, text: run }, ...called].flatMap(({ file, text }) => - [...joined(text).matchAll(fetching)].map((match) => `${file} ${match[0]}`), + [...joined(text).matchAll(fetching)].map( + (match) => `${file} ${match[0].replace(/\[\s+/g, "[").replace(/\s+/g, " ")}`, + ), ); }); @@ -505,15 +513,18 @@ const calledScripts = (files: ReadonlyArray) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const names = [ - ...new Set( - files.flatMap(({ text }) => [...text.matchAll(invoked)].map((match) => match[1]!)), - ), + ...new Set(files.flatMap(({ text }) => [...text.matchAll(invoked)].map(scriptOf))), ]; return yield* Effect.forEach(names, (file) => - fs.readFileString(file).pipe(Effect.map((text): File => ({ file, text }))), + fs.readFileString(runFrom[file] ?? file).pipe(Effect.map((text): File => ({ file, text }))), ); }); +/** Scripts a step runs from another working directory, by the path the step names. */ +const runFrom: Readonly> = { + "scripts/pr-preview.ts": "apps/hosted/cloud/scripts/pr-preview.ts", +}; + /** A one-job workflow for a case, its steps given as written under `steps:`. */ const workflow = (steps: string, job = "") => [ { @@ -868,6 +879,18 @@ layer(NodeServices.layer)("Workflow caches", (it) => { expect((yield* calledScripts(files)).map(({ file }) => file)).toContain( "scripts/releases/background.sh", ); + // A TypeScript script a step runs with node, and the commands it passes to gh or git. + expect( + codeFetches(workflow(" - run: node scripts/x.ts fetch\n"), [ + { + file: "scripts/x.ts", + text: 'yield* gh([\n "run",\n "download", run]);\nChildProcess.make("git", ["-C", dir, "fetch", "origin"]);\n', + }, + ]), + ).toEqual(['scripts/x.ts gh(["run", "download"', 'scripts/x.ts "git", ["-C", dir, "fetch"']); + expect((yield* calledScripts(files)).map(({ file }) => file)).toContain( + "scripts/ci/worker-baseline.ts", + ); }), ); diff --git a/e2e/tests/workflow-engine-calls.spec.ts b/e2e/tests/workflow-engine-calls.spec.ts new file mode 100644 index 0000000000..7fd0f5e054 --- /dev/null +++ b/e2e/tests/workflow-engine-calls.spec.ts @@ -0,0 +1,412 @@ +/** + * A run's body meets an engine whose calls fail. The server's E2E fault list + * (`EXECUTOR_TEST_WORKFLOW_ENGINE_FAULTS`, honoured only with `EXECUTOR_ENVIRONMENT=e2e`) fails + * chosen calls as the platform's engine would, so no case depends on timing or on the platform. + */ +import { expect, layer } from "@effect/vitest"; +import { Clock, Effect, Schedule, Schema } from "effect"; +import { randomUUID } from "node:crypto"; +import { Actors } from "../support/actors.ts"; +import { Api, body } from "../support/api.ts"; +import { type DeliveredSpan, Telemetry } from "../support/evidence.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { withApps } from "../support/apps-release.ts"; +import { WorkflowRun as Run } from "../support/workflow-app.ts"; +import { Target } from "../support/platform.ts"; +import { scenarios } from "../test-plan.ts"; + +/** How long `napOnce`'s first sleep waits; the case reads it to judge how that sleep ended. */ +const napOnceMillis = 50; + +/** Each workflow meets the fault the plan configures for its step names. */ +const files = [ + { path: "package.json", content: JSON.stringify({ dependencies: withApps() }) }, + { + path: "operations.ts", + content: `import { query, mutation, object, string } from "apps"; +export const count = mutation({ input: object({ label: string() }) }, async (ctx, input) => { + ctx.sql.transaction((tx) => tx.exec("INSERT INTO calls (label) VALUES (?)", input.label)); + return null; +}); +export const calls = query({ input: object({ label: string() }) }, async (ctx, input) => + ctx.sql.exec("SELECT COUNT(*) AS n FROM calls WHERE label = ?", input.label).one().n); +`, + }, + { + path: "workflows.ts", + content: `import { workflow, object } from "apps"; +import { count } from "./operations.ts"; +/** Each sleep's engine call fails once, before the engine sees it. */ +export const napOnce = workflow({ input: object({}) }, async (ctx) => { + await ctx.step.sleep("nap-once", ${napOnceMillis}); + await ctx.step.sleepUntil("wake-once", Date.now() + 2000); + return "awake"; +}); +/** The time this sleep waits for has passed before the call reaches the engine. */ +export const lateWake = workflow({ input: object({}) }, async (ctx) => { + await ctx.step.sleepUntil("late-wake", Date.now() - 1000); + return "awake"; +}); +/** Every engine call for this sleep fails. */ +export const napNever = workflow({ input: object({}) }, async (ctx) => { + await ctx.step.sleepUntil("nap-never", Date.now() + 60000); + return "unreachable"; +}); +/** The engine runs this step's write, then its answer fails once. */ +export const stepOnce = workflow({ input: object({}) }, async (ctx) => { + try { + await ctx.step.runMutation("count-once", count, { label: "step-once" }, { retries: { limit: 0, delay: 0 } }); + return { failed: false }; + } catch (error) { + return { failed: true, reason: error?.reason ?? null, errorName: error?.errorName ?? null, message: error?.message ?? null }; + } +}); +/** The engine waits this sleep and only then fails to answer, four times over. */ +export const waitedOut = workflow({ input: object({}) }, async (ctx) => { + await ctx.step.sleep("waited-out", "1500 milliseconds"); + return "awake"; +}); +/** + * Each of these sleeps' first call gets the engine's own answer, which must reach the body. A + * repeat would find the fault spent, and the sleep would succeed. + */ +const answered = (name) => async (ctx) => { + try { + await ctx.step.sleep(name, "10 milliseconds"); + return { slept: true }; + } catch (error) { + return { slept: false, reason: error?.reason ?? null, errorName: error?.errorName ?? null, message: error?.message ?? null }; + } +}; +/** As \`answered\`, for a sleepUntil whose engine waited until its time and then answered. */ +const answeredUntil = (name) => async (ctx) => { + try { + await ctx.step.sleepUntil(name, Date.now() + 50); + return { slept: true }; + } catch (error) { + return { slept: false, reason: error?.reason ?? null, errorName: error?.errorName ?? null, message: error?.message ?? null }; + } +}; +export const pausedOnce = workflow({ input: object({}) }, answered("paused-once")); +export const pausedUntil = workflow({ input: object({}) }, answeredUntil("paused-until")); +export const terminatedUntil = workflow({ input: object({}) }, answeredUntil("terminated-until")); +export const transientUntil = workflow({ input: object({}) }, answeredUntil("transient-until")); +export const terminatedOnce = workflow({ input: object({}) }, answered("terminated-once")); +export const relayedOnce = workflow({ input: object({}) }, answered("relayed-once")); +`, + }, + { + path: "index.ts", + content: `import { defineApp, router } from "apps"; +import { count, calls } from "./operations.ts"; +import * as workflows from "./workflows.ts"; +export default defineApp({}, { tools: router({ count, calls }), workflows });`, + }, + { + path: "migrations/0001_calls.sql", + content: "CREATE TABLE calls (label TEXT NOT NULL);\n", + }, +]; +const Deployed = Schema.Struct({ id: Schema.String }); +/** The four calls a sleep gets in all: the first and three repeats. */ +const sleepAttempts = 4; + +layer(HostedLive, { excludeTestServices: true })("Workflow engine calls", (it) => { + it.effect(scenarios.workflowEngineCalls.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors, + telemetry = yield* Telemetry; + // Only Cloud delivers spans from inside a run; self-host's runner hands the host the + // run's result as data (`app-failure-telemetry.spec.ts`). + const cloud = (yield* Target).metadata.target === "cloud"; + const prefix = `/api/organizations/${actors.organization.id}/apps`; + const runs: string[] = []; + const deployed = yield* api.request(actors.owner, "POST", `${prefix}/deploy`, { + name: `Engine calls ${randomUUID().slice(0, 8)}`, + files, + }); + expect(deployed.status, JSON.stringify(deployed.body)).toBe(200); + const app = yield* body(Deployed, deployed); + const path = `${prefix}/${app.id}`; + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + for (const run of runs) + yield* api.request(actors.owner, "POST", `${path}/workflow-runs/${run}/terminate`); + expect((yield* api.request(actors.owner, "DELETE", path)).status).toBe(200); + }).pipe(Effect.orDie), + ); + const start = (workflow: string) => + Effect.gen(function* () { + const response = yield* api.request(actors.owner, "POST", `${path}/workflow-runs`, { + workflow, + input: {}, + key: randomUUID(), + }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + const run = yield* body(Run, response); + runs.push(run.id); + return run.id; + }); + /** The run once it has ended, whichever way. */ + const ended = (id: string) => + Effect.gen(function* () { + const deadline = (yield* Clock.currentTimeMillis) + 40_000; + while (true) { + const response = yield* api.request( + actors.owner, + "GET", + `${path}/workflow-runs/${id}`, + ); + expect(response.status).toBe(200); + const run = yield* body(Run, response); + if (["complete", "errored", "terminated"].includes(run.status)) return run; + expect(yield* Clock.currentTimeMillis).toBeLessThan(deadline); + yield* Effect.sleep("100 millis"); + } + }); + const calls = (label: string) => + Effect.gen(function* () { + const response = yield* api.request(actors.owner, "POST", `${path}/tools/call`, { + tool: "calls", + kind: "query", + input: { label }, + }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + return response.body; + }); + /** The run's sleep spans, `count` of them once they have all been delivered. */ + const sleeps = (run: string, count: number) => + Effect.gen(function* () { + const read = Effect.gen(function* () { + const found = [ + ...(yield* telemetry.search("workflow.sleep", { run })).data, + ...(yield* telemetry.search("workflow.sleepUntil", { run })).data, + ]; + return found.length >= count + ? Effect.succeed(found.map(({ span }) => span)) + : Effect.fail(new Error(`${found.length} of ${count} sleep spans delivered`)); + }).pipe(Effect.flatten); + return yield* read.pipe( + Effect.retry({ schedule: Schedule.spaced("250 millis"), times: 80 }), + ); + }); + + // A sleep whose engine call fails once is called again and the run completes. + const once = yield* start("napOnce"); + const onceRun = yield* ended(once); + expect(onceRun, JSON.stringify(onceRun)).toMatchObject({ + status: "complete", + output: "awake", + }); + if (cloud) { + // A sleep's spans: one per execution of the body, and a span can be delivered twice. + const named = (spans: ReadonlyArray, name: string) => + spans.filter((span) => + (span.tags["executor.workflow.step"] ?? "").includes(`:${name}:`), + ); + const onceSpans = yield* sleeps(once, 1).pipe( + Effect.flatMap((spans) => + ["nap-once", "wake-once"].every((name) => named(spans, name).length > 0) + ? Effect.succeed(spans) + : Effect.fail(new Error("a span of nap-once or wake-once is not delivered yet")), + ), + Effect.retry({ schedule: Schedule.spaced("250 millis"), times: 80 }), + ); + const facts = (spans: ReadonlyArray) => + JSON.stringify( + spans.map((span) => ({ + status: span.status, + attempts: span.tags["executor.workflow.call.attempts"], + passed: span.tags["executor.workflow.call.passed"], + repeated: span.tags["executor.workflow.call.repeat_1.code"], + waited: span.tags["executor.workflow.call.waited"], + waitedMs: span.tags["executor.workflow.call.waited_ms"], + })), + ); + // nap-once is a sleep. Its fault fails the first call before the engine sees it, and the + // product then repeats it, unless that failed call itself lasted the sleep's duration: + // then the sleep's time has passed and the product ends it (a slow failure can outlast + // a 50 ms sleep on a busy host). Either is a correct end; any other is not. + const nap = named(onceSpans, "nap-once"); + expect( + nap.map((span) => span.status), + `nap-once: ${facts(nap)}`, + ).not.toContain("error"); + expect( + nap.some( + (span) => + (span.tags["executor.workflow.call.attempts"] === "2" && + span.tags["executor.workflow.call.repeat_1.code"] === "internal") || + // The product's own record of how long the failed call lasted, not this + // scenario's clock: a product that skipped the repeat would record less. + (span.tags["executor.workflow.call.attempts"] === "1" && + span.tags["executor.workflow.call.waited"] === "true" && + Number(span.tags["executor.workflow.call.waited_ms"]) >= napOnceMillis), + ), + `nap-once was repeated after its fault, or its failed call outlasted it: ${facts(nap)}`, + ).toBe(true); + // wake-once is a sleepUntil, which ends without the engine once its time has passed. + // The body computes that time with the app's clock, which advances only with its I/O, + // so on a slow host the time can have passed by the first call, before the engine is + // asked and before the fault is used. Either way the run is right: its fault failed the + // first call and the repeat ended it, or its first call found the time passed. + const wake = named(onceSpans, "wake-once"); + expect( + wake.map((span) => span.status), + `wake-once: ${facts(wake)}`, + ).not.toContain("error"); + expect( + wake.some( + (span) => + (span.tags["executor.workflow.call.attempts"] === "2" && + span.tags["executor.workflow.call.repeat_1.code"] === "internal") || + (span.tags["executor.workflow.call.attempts"] === "1" && + span.tags["executor.workflow.call.passed"] === "true"), + ), + `wake-once was repeated after its fault, or found its time passed: ${facts(wake)}`, + ).toBe(true); + } + + // A sleepUntil a time that has passed is over without the engine, which refuses it. + const late = yield* start("lateWake"); + const lateRun = yield* ended(late); + expect(lateRun, JSON.stringify(lateRun)).toMatchObject({ + status: "complete", + output: "awake", + }); + if (cloud) { + const [lateSpan] = yield* sleeps(late, 1); + expect(lateSpan?.tags).toMatchObject({ + "executor.workflow.call.attempts": "1", + "executor.workflow.call.passed": "true", + }); + } + + // A sleep whose every call fails ends the run, its cause recorded for the owner and on + // the span, after the bounded calls. + const never = yield* start("napNever"); + const neverRun = yield* ended(never); + expect(neverRun).toMatchObject({ + status: "errored", + error: "engine", + failure: { + step: "nap-never", + errorName: "WorkflowEngineCallFailed", + message: `Executor called the workflow engine ${sleepAttempts} times for this sleepUntil and every call failed. The last failure: hop=platform; class=Error; code=internal.`, + }, + }); + if (cloud) { + const [neverSpan] = yield* sleeps(never, 1); + expect(neverSpan?.tags).toMatchObject({ + "executor.workflow.call.attempts": String(sleepAttempts), + "executor.workflow.call.outcome": "failed", + "executor.workflow.call.hop": "platform", + "executor.workflow.call.error_class": "Error", + "executor.workflow.call.code": "internal", + }); + // The waits the product scheduled before each repeat: 200, 400 and 800 ms, each 80-120%. + for (const [repeat, base] of [ + [1, 200], + [2, 400], + [3, 800], + ] as const) { + const scheduled = Number( + neverSpan?.tags[`executor.workflow.call.repeat_${repeat}.wait_ms`], + ); + expect(scheduled).toBeGreaterThanOrEqual(base * 0.8); + expect(scheduled).toBeLessThanOrEqual(base * 1.2); + } + } + + // A step whose answer failed after its write ran is not called again: the write ran once. + const step = yield* start("stepOnce"); + const stepRun = yield* ended(step); + expect(stepRun, JSON.stringify(stepRun)).toMatchObject({ + status: "complete", + output: { + failed: true, + reason: "engine", + errorName: "WorkflowEngineCallFailed", + message: "hop=platform; class=Error; code=internal", + }, + }); + expect(yield* calls("step-once")).toBe(1); + + // A sleep the engine waited, whose answer was then lost, is over: it is not waited again. + // Its four faults would end the run if each call were repeated. + const waited = yield* start("waitedOut"); + const waitedRun = yield* ended(waited); + expect(waitedRun, JSON.stringify(waitedRun)).toMatchObject({ + status: "complete", + output: "awake", + }); + if (cloud) { + const waitedSpans = yield* sleeps(waited, 1); + expect( + waitedSpans.some( + (span) => + span.tags["executor.workflow.call.waited"] === "true" && + span.tags["executor.workflow.call.attempts"] === "1", + ), + ).toBe(true); + } + + // The engine's own answers reach the body at once, unrepeated: a paused or terminated + // instance, and a failure the engine relays from the app. Each fault fails one call. + for (const [workflow, expected] of [ + [ + "pausedOnce", + { + reason: "engine", + errorName: "WorkflowEngineCallFailed", + message: "hop=platform; class=Error; code=aborted", + }, + ], + [ + "terminatedOnce", + { + reason: "engine", + errorName: "WorkflowEngineCallFailed", + message: "hop=platform; class=Error; code=aborted", + }, + ], + ["relayedOnce", { reason: "execution" }], + // A pause or a termination that arrives after a sleepUntil's time is still the + // engine's answer: it reaches the body, which does not go on as if it had slept. + [ + "pausedUntil", + { + reason: "engine", + errorName: "WorkflowEngineCallFailed", + message: "hop=platform; class=Error; code=aborted", + }, + ], + [ + "terminatedUntil", + { + reason: "engine", + errorName: "WorkflowEngineCallFailed", + message: "hop=platform; class=Error; code=aborted", + }, + ], + ] as const) { + const run = yield* ended(yield* start(workflow)); + expect(run, `${workflow}: ${JSON.stringify(run)}`).toMatchObject({ + status: "complete", + output: { slept: false, ...expected }, + }); + } + + // A transient failure after a sleepUntil's time is over: the time has passed. + const transient = yield* ended(yield* start("transientUntil")); + expect(transient, JSON.stringify(transient)).toMatchObject({ + status: "complete", + output: { slept: true }, + }); + }), + ), + ); +}); diff --git a/e2e/timeouts.ts b/e2e/timeouts.ts new file mode 100644 index 0000000000..fc2cd4062f --- /dev/null +++ b/e2e/timeouts.ts @@ -0,0 +1,13 @@ +/** + * How long a scenario, and each of its hooks, may run before Vitest stops it. + * + * The fork retains its 60-second scenario and hook limits. Upstream raised its hang detector + * to 120 seconds for slow CI runners; this reconciliation does not extend deadlines to make + * scenarios pass. A failure still records the machine's speed for diagnosis + * (support/machine-speed.ts, lifecycle.ts). + * + * Speed has its own assertions, calibrated to the machine where needed (the catalog-scale search + * budget). A product that got slower everywhere is theirs to notice, not this. + */ +export const scenarioTimeoutMs = 60_000; +export const hookTimeoutMs = 60_000; diff --git a/e2e/vitest.config.ts b/e2e/vitest.config.ts index 2eee9a1d0d..a9064efb11 100644 --- a/e2e/vitest.config.ts +++ b/e2e/vitest.config.ts @@ -1,7 +1,8 @@ import { defineConfig } from "vitest/config"; import { Schema } from "effect"; import { Target } from "./report-model.ts"; -import { filesForTarget } from "./test-plan.ts"; +import { filesForTarget, patternForTarget } from "./test-plan.ts"; +import { hookTimeoutMs, scenarioTimeoutMs } from "./timeouts.ts"; // Vitest's process boundary receives an explicit target and output directory from run.ts. const directory = process.env.EXECUTOR_E2E_RUN; @@ -16,12 +17,16 @@ export default defineConfig({ name: target, setupFiles: ["e2e/setup.ts"], include: filesForTarget(target, suite, cloudMode, process.env.E2E_TEST_NAME ?? ""), + // Every scheduled title the run's filter selects, worked out here rather than passed as an + // argument: with no filter it names the whole suite, past the 128 KiB one argument may hold. + testNamePattern: patternForTarget(target, suite, process.env.E2E_TEST_NAME ?? "", cloudMode), fileParallelism: true, maxWorkers: Schema.decodeUnknownSync( Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 32 })), )(Number(process.env.E2E_WORKERS)), - testTimeout: process.env.E2E_INTERACTIVE === "1" ? 0 : 60000, - hookTimeout: 60000, + // A hang detector sized for slow runners, not a performance bound (timeouts.ts). + testTimeout: process.env.E2E_INTERACTIVE === "1" ? 0 : scenarioTimeoutMs, + hookTimeout: hookTimeoutMs, teardownTimeout: 30000, retry: 0, reporters: [ diff --git a/e2e/webhook-sweep-waits.config.ts b/e2e/webhook-sweep-waits.config.ts new file mode 100644 index 0000000000..210f681faa --- /dev/null +++ b/e2e/webhook-sweep-waits.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from "vitest/config"; + +/** What the webhook waits conclude from the sweep's passes, from synthetic spans. */ +export default defineConfig({ + test: { include: ["e2e/tests/webhook-sweep-waits.spec.ts"], testTimeout: 30_000 }, +}); diff --git a/notes/upstream-sync-20261117.md b/notes/upstream-sync-20261117.md new file mode 100644 index 0000000000..79dc1040c1 --- /dev/null +++ b/notes/upstream-sync-20261117.md @@ -0,0 +1,139 @@ +# Frozen upstream reconciliation — November 17, 2026 (PST) + +Implementation/PR only. The lead owns review, merge, private SDK distribution, exact-image +rollback rehearsal and deployment. No production, registry, account or network configuration +was accessed or changed by this work. + +## Identities and boundaries + +- Fork base: `4f9fce5001e57532a5632af286af9a9d9f3ebf8a` (#92 plus #95). +- Incorporated upstream: `9de0a387a564f625eb454e3ac0281c5822eb0da2`. +- Frozen target: `56d893c24d60e76bc532400c8cfaefc8f70fa41d`, **69 reachable commits**. + This includes a contributed HuJSON branch merged by an export; “exports” is the task's + shorthand, not a claim that every commit has an Export subject. +- The first 41 are inventoried in the approved plan. The remaining 28 below are derived + from each commit's first-parent diff, including the merge commit, not export labels. +- Never upstream `main`. The reconciliation uses a real two-parent merge. +- CLI beta.59; fork SDK `apps@0.0.1-beta.141.shuv.1`. Existing app pins are not rewritten. + #95's private publication tool and fail-closed guard survive; nothing was published. + +## Additional 28-commit inventory + +| # | Commit | Actual change and gate impact | +| --- | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 42 | `1d5cd9fac88a106b9704c82fcc1584ac89cad024` | Classify HuJSON as text rather than stringifying/parsing it as JSON in OpenAPI requests, responses and generated schemas. SDK content gate changes. | +| 43 | `aa07b772670dd26616d31d8492e3387baac1f734` | Classify workflow engine-call failures by bounded hop/class/code, retry only transient sleep/sleepUntil engine calls, finish elapsed waits, record exhausted engine failures, and add E2E-only fault injection. **Extends the retry gate beyond never-connected exchanges to durable keyed sleeps, not app steps or writes.** | +| 44 | `e9a908c50758496da91d244c76894e1b1fda1da0` | CLI beta.57; strengthens workflow proof that a repeat actually completes a failed sleep and elapsed waits are not repeated. | +| 45 | `2e8c8b7ed736b6c95f8f87c6a90f78160752919a` | Repeat read-only R2 get/list/head on classified transient 10001/10043 failures, at most three attempts; no replay of an opened failing body stream. Cloud adapter gate expands. | +| 46 | `502b7c415293a93ac00d6f4323f0d52cceba64d1` | Give each stage test request its own non-keepalive connection; record new/reused connections, written state and response-body loss through lifecycle/setup/teardown. Harness only, not product replay. | +| 47 | `f66f9cf2f0d40629b1cbf32186eeb97e96bb91c1` | Temporarily mark workflow-engine-calls not-run on Cloud after upstream failures; self-host remains scheduled. Reversed by #55; no new final skip is introduced. | +| 48 | `ece78469888ebffb3a51bf17afb2114fa2cdf023` | Measure catalog-scale phases and retain timing evidence before checking latency. No product behavior. | +| 49 | `734e0b76d97757d9d960ed886cbc51bcbc7194ee` | Add edge probe tooling, plus redaction/classification of nested certificate/connection errors. Probe is removed in #67; redaction tests remain. | +| 50 | `658397d4634775526256d777251c8593687c392a` | Share machine-speed calibration/timeout evidence and raise upstream scenario/hook hang detectors from 60 to 120 seconds. **Changes the plan's timeout gate; the fork retains 60 seconds under this task's no-longer-deadline rule.** The 7,000-tool scale budget is machine-calibrated upstream; the fork 8,549-tool cold gate is unchanged. | +| 51 | `15cca94353951efffd9761188921267e9091078c` | Repeat harness requests only when diagnostics prove nothing was written, with bounded backoff, cancellation/deadline/body checks and saved attempt receipts. **Harness gate now includes unwritten methods, never written POST/GET/DELETE.** | +| 52 | `9ec62d7c788ed0480d534597c972e9fbc05defd4` | Bound webhook sweep observations by scenario deadlines; preserve every external task/trace observation, queue progress and waits; add runner-owned webhook queue fixtures and standalone sweep-wait scenarios. Product source only corrects sweep comments; not a provider-write retry feature. | +| 53 | `5307d9fc1f8fc4a6b52d6c41056c9e29822b23c0` | Require a supported test-stage branch state before deletion, handle unreadable/deleted branches explicitly and cover command/orphan cleanup. Test-stage administration only; no branch operation was performed. | +| 54 | `e40e5ea0a29a6175b97e16855000fcb3d7987643` | Distinguish retryable framework/build storage failure from app compilation failure, observe concurrent content-addressed framework writes, and classify/retry Cloud R2 writes. **Storage gate expands: keyed content writes/confirmation are distinct from app-write replay.** | +| 55 | `12ee40be1d7a0ec66f7f0a6b229fc9688956e2bd` | Record failed sleep-call elapsed time, distinguish repeat versus elapsed sleep completion and restore managed Cloud coverage removed in #47. Retains the bounded sleep-only repeat gate. | +| 56 | `ec9dd1254be126e608c597e83208e12cc0821a77` | Make webhook FIFO fixture expose the actual lease/queue transitions and assert public observations instead of racing removed rows. No new product mutation. | +| 57 | `03b23ea1ccd35d06686bec144fddb076608129f6` | Persist privacy-safe stage request history with request timing, trace correlation and unfinished requests, including lifecycle evidence. Harness privacy/evidence gate expands. | +| 58 | `64362230c887aa4fac26ae688125ec5f300bb26f` | Wait for observed webhook-source work rather than assuming an immediate sweep; update app-account webhook assertions. No product data step. | +| 59 | `da00d81a4e3980b8c5b05110211292d8df5ef6ed` | Share one in-flight framework retention write between concurrent builds; avoid retaining a failed build; add R2 failure/contention and SDK real-store fixtures. Content-addressed storage gate, no arbitrary app-call replay. | +| 60 | `52c4834a074ea87c511a630c076d42b4da0433ad` | CLI beta.58 and explicit CI-selection coverage for app source/framework changes. No fork CI layout replacement. | +| 61 | `e160181f047f316e459358ec6161a7560a962e63` | Publish security reporting guidance and security.txt routes through site/Cloud edge contract. No infrastructure applied. | +| 62 | `fbb5ac1df109e54e5bbb55fa9d05f45f1ba7640a` | Reject worker-size/skip evidence from unsuitable workflow versions/attempts; strengthen missing/mismatched baseline cases. Upstream-layout tooling only. | +| 63 | `41120c3805ec9b5dcfd3aae52f04b09c19a11c22` | Make catalog-scale and connect-repeat fixtures account for process outcomes and protect opened/sent-call refusal cases. No classifier widening. | +| 64 | `dcd272aff3100cca9584962cefa3ca455a23c775` | Merge HuJSON contribution, broaden structured +json recognition and AWS JSON types, and check JSON before the text gate; apps beta.139. SDK package changes; no protocol bump. | +| 65 | `a8d3a2c555f6d14a019ef2ee9fdb8e0b9209ecfc` | Real-server HuJSON/JSON/AWS response and request coverage, with shared OpenAPI import helpers. SDK media gate now executable. | +| 66 | `f4416e915d7c4479a29c01f9d5fc6ec4343251fb` | Memoize shared schema identities/closures across account-router listings, validate merged schema entries once per catalog, preserve per-account branches, and record large-catalog/failure evidence; apps beta.140. **SDK content and #89 cyclic/shared-schema gates expand.** | +| 67 | `ab2a3469bcb22cacfd5e8c085b5ebe79d1da1d88` | Remove the edge-probe experiment and its tooling/test selection. Not a skip of a retained product invariant; the probe never reaches the final tree. | +| 68 | `d2daa20f473726d1a8553d28d9e35b749c6f5dfc` | CLI beta.59; operator OAuth held scopes can renew existing grants but cannot be requested by new/reconnect sign-ins, with bounded explicit refusal and own-client recovery. Cloud's Google registration holds gmail.modify; self-host operator-owned Google retains it. **OAuth contract/config gate changes, but #68 DCR registered scopes and #80 callback projection remain untouched.** | +| 69 | `56d893c24d60e76bc532400c8cfaefc8f70fa41d` | apps beta.141; linear srcset scanning; enforce private-fetch policy on off-registry tarballs/redirects, sanitize install failures and extend worker-bundler patch; add network/srcset scenarios. **Builder egress/private-registry compatibility gate expands.** | + +### Plan gates changed, plainly + +The 28 commits are not covered by the original plan's “no SDK source change” or “OAuth +usability refactor only” observations. They require a new immutable SDK version, held-scope +OAuth refusal/renewal evidence, multi-account schema closure checks, dependency-egress +checks and expanded storage/engine-call classification. Repeated sleeps and content-addressed +storage operations are not never-connected app RPC exchanges. No app write or workflow step +whose answer was lost is replayed. #93 is not diagnosed or fixed. The fork retains its 60-second scenario/hook limits rather than upstream's 120 seconds. No test retry, +skip or assertion allowance is added here to make these paths green. + +## Conflict ledger (all 23 unmerged paths) + +| Path | Choice and reason | +| ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `apps/hosted/self-host/README.md` | Reconcile Rust/store split with fork's persistent telemetry volume and #95 private SDK procedure. Do not copy upstream's deletion command or claim default fork telemetry is ephemeral. | +| `apps/hosted/self-host/compose.yaml` | Keep flat/default-organization host settings and telemetry volume; update only collector terminology. | +| `apps/hosted/self-host/native/main.go` | Keep fork separate-app-process/watchdog/proxy/cancellation paths; use upstream collector executable/socket/supervision and COLLECTOR binding, including shutdown. | +| `apps/hosted/self-host/scripts/package-runtime.ts` | Keep fork shared/product/apps configurations and APPS callback bindings; replace Motel with collector external binding, remove Motel config/payload; stage the collector separately in native fixtures. | +| `apps/hosted/server/src/contracts/mcp-connections.ts` | Keep member/API confined mutations, connection-key schemas and 5-second–90-day bounds. Keep fork manage-scoped metadata listing rather than silently broadening read-key metadata; selected-tools grants still cannot list siblings. | +| `apps/hosted/server/src/implementation/access.ts` | Keep request-owned access-bearing CheckedApp and pinned active-version authorization. Accept upstream AppAccountReads/selectedAppFrom and verified slug lookup; remove superseded second batch-selection read path. | +| `apps/hosted/server/src/implementation/mcp-connections.ts` | Keep requireCallingScope/member-owner confinement and API management contract, not upstream browser-only mutation comment. | +| `apps/hosted/server/src/implementation/mcp.ts` | Keep verified CurrentCaller, discoveryChecked `{app,access}`, account/connection checks and listAccountsEach. Reconcile one upstream searchReads/PrefetchHost snapshot with the fork's batched kept-row facade; misses/refusals use that same snapshot, not another query cache. | +| `apps/hosted/server/src/implementation/resource-lifecycle.ts` | Keep creator/root-owner group audience semantics; accept upstream owner-batched rechecks and per-app Result failures, removing the old duplicate appsResolving implementation. | +| `apps/hosted/server/src/implementation/tools.ts` | Accept upstream prefetched selection for live search; retain fork batched kept-row API and policy/router/account filters, using those same upstream selections. | +| `bun.lock` | Accept upstream resolutions/patch changes, then regenerate with bun install for the exact .141.shuv.1 workspace identity; keep Playwright 1.64.1 patch. | +| `e2e/tests/mcp-connections.spec.ts` | Keep fork API/full-grant/anti-widening assertions rather than upstream's assertion that mutation tools do not exist. | +| `packages/app-templates/src/implementation/executor-skill-digests.gen.ts` | Regenerate with bun run executor:intro after source reconciliation; never combine hash strings. | +| `packages/apps/package.json` | New immutable fork SDK 0.0.1-beta.141.shuv.1, not upstream .141 or reused .138.shuv.1. | +| `packages/mcp/src/implementation/execute.ts` | Keep dotted-path resolver and batched kept-row/account projection. Accept upstream per-search batch ownership for misses; never batch unrelated locked apps. | +| `packages/sdk/src/contracts/executor.ts` | Accept upstream owner-batched appsResolving Result contract; retain fork host-only searchableKept and caller/key extensions. | +| `packages/sdk/src/implementation/declarations.ts` | Accept upstream batched account usability/recheck semantics and authorizeAll; retain unchanged fork caller-bearing snapshot/key behavior. | +| `packages/sdk/src/implementation/listings.ts` | Accept upstream Schema decoding/prefetch. Reconcile kept-row projection with shared prefetched rows/access so misses remain explainable without repeating reads; preserve per-catalog shared projections. | +| `packages/sdk/src/implementation/tool-search-store.ts` | Accept upstream ordered batch rows rather than fork map; adapt kept-row consumer to ordered indexing. | +| `packages/sdk/src/implementation/tool-search.ts` | Accept batch-row documentation/type; retain automatic merge of fork namespace-relative dotted search. | +| `packages/sdk/src/implementation/tools.ts` | Keep verified requester/approver and per-app sharedRuntime/callerConnection. Accept upstream app-entry counting and safe failure classification, including resumed calls with one prior app entry. Kept-row fast path consumes only served entries of the shared owner/actor/version/account prefetch. | +| `packages/sdk/src/implementation/worker-dependencies.ts` | Keep fork trailing-slash registry normalization and upstream tarballFetch/egress/redaction. | +| `packages/sdk/src/implementation/workerd-entry.ts` | Keep fork durable progress/stalled-body and attempt-limit watchdogs. Integrate upstream engine fault classification/sleep semantics; do not drop crash-loop bounds. | + +### Non-conflicting preservation and fixture reconciliation + +#88 resolver, #89 OpenAPI shared closures, #90 awaited feedback receipt/privacy, #91 shared +notifications/cancellation, root-owner multi-org admission, organization-only publishing, +short names, apps call, Playwright patch/Bun 1.4.4, migration mapping and #95 were reviewed +beyond conflict markers. The pending personal-account scenario remains retained and skipped; +neither hosted target is unskipped. No #68/#80/#93 feature work is included. + +Native fixtures now require the staged collector executable instead of motel.capnp and find +the process by `--data=` instead of Motel's workerd directory argument. Native Go +shutdown calls the collector supervisor, not the removed Motel supervisor. + +The upstream search-statements assertion is **tightened from exactly 12 to exactly 10 SQL +statements**: the fork already carries access with its app list and does not reread those two +access folds. The original fork warm search still requires four backend operations; its +scenario is retained. Both small/large app counts, forbidden private apps and reconnect +refusals remain asserted. No assertion is relaxed to allow more work or less security. + +## Migration/data impact and remaining release gates + +The full 69-commit range adds **no product schema migration and no hosted/Cloud data step**. +The sole changed migration file changes a comment from Motel to telemetry collector. +Fork Cloud 14 remains immutable; upstream 14/15/16 stay fork 15/16/17. SDK/product/registry +journals and the pending personal-account decision remain unchanged. Existing installed app +versions, credentials, source pins and app data are not migrated for telemetry. + +The Rust collector owns a separate SQLite store, never reads/resets/deletes nested `motel/`, +and does not import old history. Exact old image + independently preserved old telemetry bytes +are still required to read old traces. Live mounted data, old-reader restoration, existing +personal-account impact, current private builder resolution, exact-image rollback, deployed +Cloud and non-Linux-x64 platforms are not established by fresh fixture tests. These remain +lead-owned release gates, not inferred from PR CI. + +## Guidance followed + +Read charter/lead handoff, AGENTS, CONTEXT, RELEASING, executor-e2e skill, migration numbering, +personal-account decision, approved plan/summary/resolutions/approval/amendment, #92 and fork CI. +Missing public-export engineering notes remain known papercut `pc_dcf8902b5987`. +Rat-stack pages read: `ratstack://repo/AGENTS.md`, `ratstack://skills/ship`, +`ratstack://skills/investigate-a-failure`, `ratstack://skills/add-a-store`. +Only language-neutral fence/evidence/migration/rollback rules apply; Executor conventions win. + +## Evidence + +Command logs and receipts are copied from `.local/upstream-sync/` to +`/home/shuv/.cache/agent-ws/executor-upstream-sync-20261117/`. The final PR and handoff give +executed command/scenario receipts, including failures and fixes; an unmatched filter is never +a pass. Real-server diagnostics/results and privacy-safe named evidence are retained outside +the prunable `.local/e2e/`; no sessions or product databases are published. diff --git a/package.json b/package.json index 0f5b72a038..5d7c16aad3 100644 --- a/package.json +++ b/package.json @@ -67,13 +67,14 @@ "site:links": "node apps/hosted/cloud/scripts/check-site-links.ts", "workspace:check": "bash scripts/workspace-check.sh", "reference:effect": "bun scripts/reference-effect.ts", - "telemetry:build": "bun run --cwd packages/telemetry build:motel", + "telemetry:build": "bun run collector:build && bun run viewer:build", "collector:build": "node packages/telemetry/scripts/build-collector.ts build", + "viewer:build": "bun run --cwd packages/telemetry/viewer build", "format": "oxfmt .", "format:check": "oxfmt --check .", "lint": "oxlint -c .oxlintrc.jsonc . --deny-warnings", "lint:fix": "oxlint -c .oxlintrc.jsonc --fix .", - "check": "bun run patches:check && bun run versions:check && bun run format:check && bun run lint && bun run typecheck && bun run apps:protocols && bun run apps:telemetry && bun run api:errors && bun run auth:rate-limit && node packages/apps/scripts/browser-scripts.mjs --check && node packages/app-templates/scripts/executor-intro.ts --check && node scripts/check-no-unit-tests.ts && node packages/sdk/scripts/check-storage-versions.ts && node scripts/check-local-data-steps.ts && node apps/hosted/cloud/scripts/edge-contract.ts --check && node e2e/check-boundary.ts", + "check": "node scripts/check-conflict-markers.ts && bun run patches:check && bun run versions:check && bun run format:check && bun run lint && bun run typecheck && bun run apps:protocols && bun run apps:telemetry && bun run api:errors && bun run auth:rate-limit && node packages/apps/scripts/browser-scripts.mjs --check && node packages/app-templates/scripts/executor-intro.ts --check && node scripts/check-no-unit-tests.ts && node packages/sdk/scripts/check-storage-versions.ts && node scripts/check-local-data-steps.ts && node apps/hosted/cloud/scripts/edge-contract.ts --check && node e2e/check-boundary.ts", "edge:contract": "node apps/hosted/cloud/scripts/edge-contract.ts", "e2e:prepare": "node scripts/e2e-prepare.ts", "e2e:runtime": "node scripts/build-test-runtime.ts", @@ -110,14 +111,18 @@ "e2e:apps-job": "vitest run --config e2e/apps-job.config.ts", "e2e:collector-job": "vitest run --config e2e/collector-job.config.ts", "e2e:macos-staple": "vitest run --config e2e/macos-staple.config.ts", + "e2e:npm-channel": "vitest run --config e2e/npm-channel.config.ts", + "e2e:publish-guard": "vitest run --config e2e/publish-guard.config.ts", "e2e:apps-published": "vitest run --config e2e/apps-published.config.ts", "e2e:ci-selection": "vitest run --config e2e/ci-selection.config.ts", "e2e:deploy-decision": "vitest run --config e2e/deploy-decision.config.ts", "e2e:ci-supersede": "vitest run --config e2e/ci-supersede.config.ts", "e2e:ci-repeat": "vitest run --config e2e/ci-repeat.config.ts", + "e2e:check-description": "node e2e/check-description.ts", "e2e:workflow-caches": "vitest run --config e2e/workflow-caches.config.ts", "e2e:telemetry-event-lifetime": "vitest run --config e2e/telemetry-event-lifetime.config.ts", "e2e:process-identity": "vitest run --config e2e/process-identity.config.ts", + "e2e:cloud-postgres-volumes": "vitest run --config e2e/cloud-postgres-volumes.config.ts", "e2e:part-results": "vitest run --config e2e/part-results.config.ts", "e2e:release-evidence": "vitest run --config e2e/release-evidence.config.ts", "e2e:desktop-next-server": "vitest run --config e2e/desktop-next-server.config.ts", @@ -127,10 +132,15 @@ "e2e:cli-startup-imports": "vitest run --config e2e/cli-startup-imports.config.ts", "e2e:process-output-exit": "vitest run --config e2e/process-output-exit.config.ts", "e2e:release-archives": "vitest run --config e2e/release-archives.config.ts", + "e2e:conflict-markers": "vitest run --config e2e/conflict-markers.config.ts", "e2e:emulator-failures": "vitest run --config e2e/emulator-failures.config.ts", "e2e:database-connections": "vitest run --config e2e/database-connections.config.ts", "e2e:removal-evidence": "vitest run --config e2e/removal-evidence.config.ts", + "e2e:webhook-sweep-waits": "vitest run --config e2e/webhook-sweep-waits.config.ts", "e2e:request-failures": "vitest run --config e2e/request-failures.config.ts", + "e2e:lost-connections": "vitest run --config e2e/lost-connections.config.ts", + "e2e:lost-connections-lifecycle": "vitest run --config e2e/lost-connections-lifecycle.config.ts", + "e2e:test-stage-branch": "vitest run --config e2e/test-stage-branch.config.ts", "e2e:test-stage-orphans": "vitest run --config e2e/test-stage-orphans.config.ts", "e2e:typecheck-runner": "vitest run --config e2e/typecheck-runner.config.ts", "e2e:prepare-cache": "vitest run --config e2e/prepare-cache.config.ts", diff --git a/packages/app-management/src/cli.ts b/packages/app-management/src/cli.ts index 779523dc4c..27d28328b1 100644 --- a/packages/app-management/src/cli.ts +++ b/packages/app-management/src/cli.ts @@ -36,6 +36,7 @@ import { inlineBytes, PackageName, hostedExecutorOrigin, + compiledFileLimit, publicFileLimits, sourceLimits, Json, @@ -124,7 +125,7 @@ const commit = (purpose: string) => ); const files = Flag.Path("files", { pathType: "either", mustExist: true }).pipe( Flag.withDescription( - `Complete app source: a directory read recursively (skipping ${ignoredSource.join(", ")}), or a JSON file containing [{"path": "index.ts", "content": "..."}]. It must include a root index.ts. Code outside ui/public/ is UTF-8 text, at most ${formatCount(sourceLimits.files)} files and ${formatBytes(sourceLimits.bytes)}. Files under ui/public/ may be binary: at most ${formatCount(publicFileLimits.files)} files, each at most ${formatCount(publicFileLimits.fileBytes)} bytes. A source larger than ${formatBytes(sourceLimits.bytes)} in total is pushed with Git. The CLI checks this before uploading`, + `Complete app source: a directory read recursively (skipping ${ignoredSource.join(", ")}), or a JSON file containing [{"path": "index.ts", "content": "..."}]. It must include a root index.ts. Code outside ui/public/ is UTF-8 text, at most ${formatCount(sourceLimits.files)} files and ${formatBytes(sourceLimits.bytes)}; files under ui/public/ may be binary: at most ${formatCount(publicFileLimits.files)} files, each at most ${formatCount(publicFileLimits.fileBytes)} bytes. A source larger than ${formatBytes(sourceLimits.bytes)} in total is pushed with Git. The CLI checks these limits before uploading. The host's build also refuses any of the app's own JavaScript or TypeScript files over ${formatCount(compiledFileLimit)} bytes; that limit is checked by the host, not before uploading, and does not cover files inside imported packages`, ), ); const publication = { diff --git a/packages/app-management/src/implementation/tool-files.ts b/packages/app-management/src/implementation/tool-files.ts new file mode 100644 index 0000000000..912e60658c --- /dev/null +++ b/packages/app-management/src/implementation/tool-files.ts @@ -0,0 +1,98 @@ +/** + * The images, audio and files a tool result carries as MCP content blocks, written to disk for the + * tool commands. The host moves each block's bytes out of the result's JSON and leaves the block + * without them, plus its `contentIndex`, in its place. The CLI prints only that JSON, so it writes + * each block's bytes to a file and adds the file's `path` beside the `contentIndex`. + */ +import { Effect, FileSystem, Option, Path, Schema } from "effect"; + +const Media = Schema.Struct({ + type: Schema.Literals(["image", "audio"]), + mimeType: Schema.String, + data: Schema.Uint8ArrayFromBase64, +}); +const BlobResource = Schema.Struct({ + type: Schema.Literal("resource"), + resource: Schema.Struct({ + uri: Schema.String, + mimeType: Schema.optional(Schema.String), + blob: Schema.Uint8ArrayFromBase64, + }), +}); +const decodeBlock = Schema.decodeUnknownOption(Schema.Union([Media, BlobResource])); + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const decodeComponent = Option.liftThrowable(decodeURIComponent); + +/** `.pdf` for `application/pdf`, `.svg` for `image/svg+xml`; nothing for an unusual type. */ +const extension = (mimeType: string | undefined) => { + const subtype = mimeType?.split(";")[0]?.split("/")[1]?.split("+")[0]?.trim().toLowerCase(); + if (subtype === undefined || !/^[a-z0-9-]{1,16}$/u.test(subtype)) return ""; + return subtype === "jpeg" ? ".jpg" : `.${subtype}`; +}; + +/** + * A safe file name for the block at `index`: the index, then the resource URI's last path segment + * with anything but letters, digits, `.`, `_` and `-` replaced, or `content` and the extension its + * MIME type implies. + */ +const fileName = (index: number, uri: string | undefined, mimeType: string | undefined) => { + const segment = uri === undefined ? "" : (URL.parse(uri)?.pathname ?? uri).split("/").at(-1); + const decoded = Option.getOrElse(decodeComponent(segment ?? ""), () => segment ?? ""); + const safe = decoded + .replaceAll(/[^A-Za-z0-9._-]+/gu, "-") + .replace(/^[.-]+/u, "") + .slice(0, 100); + const name = safe === "" ? "content" : safe; + return `${index}-${name}${name.includes(".") ? "" : extension(mimeType)}`; +}; + +/** + * Write each image, audio and blob resource block in `content` to its own file in `directory`, or + * in a new temporary directory when none is given, and return `value` with each placeholder that + * names one of those blocks by `contentIndex` given the file's `path`. A result without such + * blocks is returned unchanged and writes nothing. + */ +export const saveContentFiles = ( + value: unknown, + content: ReadonlyArray, + directory: Option.Option, +) => + Effect.gen(function* () { + const blocks = content.flatMap((block, index) => + Option.toArray(decodeBlock(block)).map((decoded) => ({ index, decoded })), + ); + if (blocks.length === 0) return value; + const fs = yield* FileSystem.FileSystem, + path = yield* Path.Path; + const into = yield* Option.match(directory, { + onNone: () => fs.makeTempDirectory({ prefix: "executor-files-" }), + onSome: (directory) => + fs.makeDirectory(directory, { recursive: true }).pipe(Effect.as(path.resolve(directory))), + }); + const paths = new Map(); + for (const { index, decoded } of blocks) { + const [uri, mimeType, bytes] = + decoded.type === "resource" + ? [decoded.resource.uri, decoded.resource.mimeType, decoded.resource.blob] + : [undefined, decoded.mimeType, decoded.data]; + const file = path.join(into, fileName(index, uri, mimeType)); + yield* fs.writeFile(file, bytes); + paths.set(index, file); + } + const annotate = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(annotate); + if (!isRecord(value)) return value; + const placeholder = + (value.type === "image" || value.type === "audio" || value.type === "resource") && + typeof value.contentIndex === "number"; + const saved = placeholder ? paths.get(Number(value.contentIndex)) : undefined; + if (saved !== undefined) return { ...value, path: saved }; + return Object.fromEntries( + Object.entries(value).map(([key, field]) => [key, annotate(field)]), + ); + }; + return annotate(value); + }); diff --git a/packages/app-management/src/implementation/tools-client.ts b/packages/app-management/src/implementation/tools-client.ts index 39b32f09b7..cb8061b039 100644 --- a/packages/app-management/src/implementation/tools-client.ts +++ b/packages/app-management/src/implementation/tools-client.ts @@ -99,7 +99,9 @@ export type McpSession = typeof McpSession.Type; /** * Call one of the host's MCP tools and return its result as JSON: its structured content, or the - * JSON its first text block holds. A result the tool marks as an error fails with its text. With + * JSON its first text block holds, with every content block the result carries, such as the + * images and files the JSON refers to by `contentIndex`. A result the tool marks as an error fails + * with its text. With * `session`, the call continues that MCP session instead of opening one; either way the result * names the session it used. */ @@ -147,8 +149,9 @@ export const callTool = ( ); if (result.isError === true) return yield* new ToolCallRefused({ tool, reason: "tool", message: texts.join("\n") }); + const content: ReadonlyArray = Array.isArray(result.content) ? result.content : []; if (result.structuredContent !== undefined) - return { value: result.structuredContent, session: used }; + return { value: result.structuredContent, content, session: used }; const first = texts[0]; const json = first === undefined ? Option.none() : decodeJson(first); if (Option.isNone(json)) @@ -157,7 +160,7 @@ export const callTool = ( method: "POST", path: URL.parse(target.url)?.pathname ?? "/mcp", }); - return { value: json.value, session: used }; + return { value: json.value, content, session: used }; }).pipe(Effect.scoped); /** Print a tool's result as tool commands print every success: indented JSON on stdout. */ diff --git a/packages/app-management/src/tools-cli.ts b/packages/app-management/src/tools-cli.ts index 9f5816dc29..267f612d78 100644 --- a/packages/app-management/src/tools-cli.ts +++ b/packages/app-management/src/tools-cli.ts @@ -20,6 +20,7 @@ import { hostedExecutorOrigin } from "@executor-js/sdk/core"; import { ToolInputInvalid, ToolsSignInRequired } from "./client-error.ts"; import { hostFlag, namedHost, toolsTarget, type LocalServer } from "./implementation/hosts.ts"; import { hasRegistrySession, logout, toolsLogin } from "./implementation/node-auth.ts"; +import { saveContentFiles } from "./implementation/tool-files.ts"; /** * The MCP client and the paused-call records, loaded when a command uses them, so other commands @@ -77,7 +78,13 @@ const fileFlag = (name: string, description: string) => * recovery on stderr, and `{"error": ...}` on stdout when stdout is not a terminal. */ const prints = "Prints the result as JSON"; -const pauses = `${prints}. A paused call prints its pending request, with an approvalUrl when the host gives one; continue it with executor resume`; +const pauses = `${prints}, with the path of each image, audio or file it holds, written to a new temporary directory unless --files names one. A paused call prints its pending request, with an approvalUrl when the host gives one; continue it with executor resume`; + +/** Where `call`, `execute` and `resume` write the images, audio and files a result holds. */ +const filesFlag = fileFlag( + "files", + "Write images, audio and files the result holds into this directory instead of a new temporary one", +); /** * The tool commands, `login`, `logout`, `whoami` and `mcp`. `localServer` resolves the local @@ -88,24 +95,26 @@ export const toolCommands = ( localServer?: Effect.Effect, ) => { /** - * Call one MCP tool on the resolved host and print its result. A call paused for browser - * approval is answered in the session that made it, so `resume` continues that session. + * Call one MCP tool on the resolved host and print its result, with the images, audio and files + * it holds written into `files` or a new temporary directory. A call paused for browser approval + * is answered in the session that made it, so `resume` continues that session. */ const run = ( host: Option.Option, tool: string, input: Record, - resumed?: string, + options: { readonly resumed?: string; readonly files?: Option.Option } = {}, ) => Effect.gen(function* () { const target = yield* toolsTarget(host, localServer, { fresh: false }); const { callTool, printJson } = yield* toolsClient; const { pausedSession, recordPause } = yield* pausedCalls; + const { resumed, files = Option.none() } = options; const session = resumed === undefined ? Option.none() : yield* pausedSession(target.host, resumed); const result = yield* callTool(target, tool, input, Option.getOrUndefined(session)); yield* recordPause(target.host, result.value, result.session, resumed); - yield* printJson(result.value); + yield* printJson(yield* saveContentFiles(result.value, result.content, files)); }); const loginHost = Flag.String("host").pipe( Flag.withDescription( @@ -255,6 +264,7 @@ export const toolCommands = ( "input-file", "Read the input JSON from this file, or from standard input with -", ), + files: filesFlag, }).pipe( Command.withDescription( `Call app tools directly. Use executor execute when one result feeds another, or to keep only part of a large result. ${pauses}`, @@ -275,19 +285,22 @@ export const toolCommands = ( description: "Read the input from standard input", }, ]), - Command.withHandler(({ host, path, input, inputFile }) => + Command.withHandler(({ host, path, input, inputFile, files }) => Effect.gen(function* () { const json = yield* jsonFrom(input, inputFile, "--input and --input-file", "The input"); if (Option.isSome(path)) - return yield* run(host, "call", { - calls: { path: path.value, input: Option.getOrElse(json, () => ({})) }, - }); + return yield* run( + host, + "call", + { calls: { path: path.value, input: Option.getOrElse(json, () => ({})) } }, + { files }, + ); if (Option.isNone(json) || !Array.isArray(json.value)) return yield* new ToolInputInvalid({ message: "Pass a tool path, or a JSON array of {path, input} calls with --input or --input-file.", }); - return yield* run(host, "call", { calls: json.value }); + return yield* run(host, "call", { calls: json.value }, { files }); }), ), ), @@ -298,6 +311,7 @@ export const toolCommands = ( Flag.optional, ), file: fileFlag("file", "Read the program from this file, or from standard input with -"), + files: filesFlag, }).pipe( Command.withDescription( `Run a JavaScript program over your apps, where app tools are async functions such as tools.github.issues.list(input); return only what you need. ${pauses}`, @@ -312,14 +326,14 @@ export const toolCommands = ( description: "Run a program from a file and keep its result in a file", }, ]), - Command.withHandler(({ host, code, file }) => + Command.withHandler(({ host, code, file, files }) => Effect.gen(function* () { const program = yield* textFrom(code, file, "--code and --file"); if (Option.isNone(program)) return yield* new ToolInputInvalid({ message: "Pass the program with --code, or with --file (- reads standard input).", }); - return yield* run(host, "execute", { code: program.value }); + return yield* run(host, "execute", { code: program.value }, { files }); }), ), ), @@ -342,6 +356,7 @@ export const toolCommands = ( "content-file", "Read the form fields JSON from this file, or from standard input with -", ), + files: filesFlag, }).pipe( Command.withDescription( `Continue a paused call or program with the person's answer; never run it again instead. ${pauses}`, @@ -353,7 +368,7 @@ export const toolCommands = ( description: "Collect an answer given on the approvalUrl page", }, ]), - Command.withHandler(({ host, requestId, action, content, contentFile }) => + Command.withHandler(({ host, requestId, action, content, contentFile, files }) => Effect.gen(function* () { const fields = yield* jsonFrom( content, @@ -364,7 +379,7 @@ export const toolCommands = ( if (Option.isNone(action)) { if (Option.isSome(fields)) return yield* new ToolInputInvalid({ message: "Pass --action with --content." }); - return yield* run(host, "resume", { requestId }, requestId); + return yield* run(host, "resume", { requestId }, { resumed: requestId, files }); } return yield* run( host, @@ -376,7 +391,7 @@ export const toolCommands = ( ...Option.match(fields, { onNone: () => ({}), onSome: (content) => ({ content }) }), }, }, - requestId, + { resumed: requestId, files }, ); }), ), diff --git a/packages/app-templates/executor/skills/app-authoring/deploy.md b/packages/app-templates/executor/skills/app-authoring/deploy.md index 8768c1bdfb..1b9fc77540 100644 --- a/packages/app-templates/executor/skills/app-authoring/deploy.md +++ b/packages/app-templates/executor/skills/app-authoring/deploy.md @@ -22,11 +22,22 @@ from the app. `--files` also accepts a JSON file holding an array of `{ "path", "content" }` objects. Code, every file outside `ui/public/`, is UTF-8 text: at most 4,096 files and -16 MiB of file contents, on every host. Files under `ui/public/` are served as +16 MiB of file contents, on every host. Each of the app's own JavaScript or +TypeScript files is at most 2,000,000 bytes (2 MB). JSON files are outside this +limit, so keep a large document such as an OpenAPI description in a `.json` +file. The host checks this limit while it builds, not the CLI before uploading: +a build that reaches a larger file of the app's own fails with +`VersionBuildFailed` at the `source` stage, naming the file, its size and the +limit; no app or version is created. Files inside npm packages the app imports +are not limited and are still compiled. On self-host and the local server a +build compiles on the thread that runs apps, so a large package file delays +every app call, and any other request served from that thread, while it +compiles: a 3,000,000-byte module of dense code took about 1.2 seconds in our +tests. Files under `ui/public/` are served as they are and may be binary, such as images, fonts, audio, WebAssembly or a prebuilt bundle: at most 10,000 files, each at most 32 MB and 1 GB in total. Builds never load them; each deploy copies them to the host's own storage. -The CLI checks these limits before it uploads anything. A source over them +The CLI checks the other limits before it uploads anything. A source over them fails with `SourceRejected`, which states the count or size and names the largest files; a binary file outside `ui/public/` fails the same way and is named. The host rejects a larger source with `SourceError` and HTTP 413. diff --git a/packages/app-templates/executor/skills/code-mode/SKILL.md b/packages/app-templates/executor/skills/code-mode/SKILL.md index 21b7cf6c7e..5a57a9f25b 100644 --- a/packages/app-templates/executor/skills/code-mode/SKILL.md +++ b/packages/app-templates/executor/skills/code-mode/SKILL.md @@ -223,26 +223,39 @@ Inside a program, a caught tool error's `message` is the same JSON as `status: "capacity-exceeded"` means the server is running too many programs. Wait briefly, then retry. -## See images and hear audio +## See images, hear audio and read files -You can look at any image and listen to any audio a tool gives you. Wherever -the value `call` or `execute` returns holds `{ type: "image", mimeType, data }` -or `{ type: "audio", mimeType, data }`, at any depth, with `data` in standard or -URL-safe base64, you receive that block as media, and the JSON shows -`{ type, mimeType, contentIndex }` in its place; the same block repeated is sent -once. Screenshots from MCP servers already arrive in that shape. Inside the -program the block is whole, so its data can be passed to another tool. +You can look at any image, listen to any audio and read any file a tool gives +you. Wherever the value `call` or `execute` returns holds one of these MCP +blocks, at any depth, with its bytes in standard or URL-safe base64, you receive +it as its own content block instead of base64 text: -When a tool returns the bytes some other way, such as an email attachment as -`{ base64 }`, return them in that shape from `execute`: +- `{ type: "image", mimeType, data }` and `{ type: "audio", mimeType, data }` + arrive as media. The JSON shows `{ type, mimeType, contentIndex }` in place. +- `{ type: "resource", resource: { uri, mimeType, blob } }` arrives as a file, + such as a PDF, which your client opens or saves for you to read. The JSON + shows `{ type, resource: { uri, mimeType }, contentIndex }` in place. + +The same block repeated is sent once. Screenshots from MCP servers already +arrive as images, and some tools return files as resources, such as email +attachments. Inside the program the block is whole, so its bytes can be passed +to another tool. + +When a tool returns bytes some other way, such as `{ base64 }`, return them in +one of these shapes from `execute`: an image block for an image, and otherwise +a resource whose `uri` ends in the file's name: ```js -const file = await tools.gmail.attachment({ messageId, attachmentId }); -return { type: "image", mimeType: "image/png", data: file.base64 }; +const file = await tools.example.download({ id }); +return { + type: "resource", + resource: { uri: "file:///report.pdf", mimeType: "application/pdf", blob: file.base64 }, +}; ``` -Never return, read, decode or describe base64 as text; return it as media and -look. +Never return, read, decode or describe base64 as text; return it as a block and +look. The `executor` CLI writes each block to a file and prints its `path` +beside its `contentIndex`. Send feedback with the Executor app's `feedback.submit` tool when Executor gets in your way or something works especially well; the `executor` skill says how. diff --git a/packages/app-templates/executor/skills/executor/SKILL.md b/packages/app-templates/executor/skills/executor/SKILL.md index 0db3c71048..0e6afa1229 100644 --- a/packages/app-templates/executor/skills/executor/SKILL.md +++ b/packages/app-templates/executor/skills/executor/SKILL.md @@ -28,7 +28,7 @@ Write a JavaScript program for `execute` when one call's result feeds another, or a large result should be filtered before it reaches you. Inside it, the same paths are async functions, and `tools.search` and `tools.search.describe` work as above. Return only the data you need. For depth, read the `code-mode` skill; -it says how to see images and hear audio a tool returns, without reading base64. +it says how to see images, hear audio and read files tools return, not base64. When `call` or `execute` pauses for approval or input, show the request to the user and resume it; never repeat the call or run the program again. diff --git a/packages/app-templates/src/implementation/executor-intro.gen.ts b/packages/app-templates/src/implementation/executor-intro.gen.ts index 7c898b4d80..7216b29f8b 100644 --- a/packages/app-templates/src/implementation/executor-intro.gen.ts +++ b/packages/app-templates/src/implementation/executor-intro.gen.ts @@ -1,4 +1,4 @@ // Generated by packages/app-templates/scripts/executor-intro.ts. Do not edit. /** The body of skills/executor/SKILL.md, without its frontmatter. MCP servers send it as their instructions. */ -export const executorIntro = "# Executor\n\n## What Executor is\n\nExecutor is a host that runs apps: TypeScript packages that expose tools,\nstorage, UI, accounts and background work. Agents reach every app through the\nMCP tools `skills`, `search`, `describe`, `call` and `execute`, with each app's\nslug as its namespace. Credentials go only through Executor's secure connection\nlinks, never through chat or source code. Some calls pause for user approval.\nThis is the Executor app's `executor` skill. Read its other skills with\n`skills({app: \"executor\", name: \"code-mode\"})`; `skills({})` lists them all.\n\n## Find and call tools\n\n`search({ query: \"...\" })` returns each tool's exact `path`, such as\n`tools..`, and input type; `describe({ paths })` returns full\nsignatures. Never guess argument names.\n\nFor a few independent calls, use `call({ calls: { path, input } })`; an array\nof calls runs concurrently. Each result has `ok` and its `value` or `error`.\n\nWrite a JavaScript program for `execute` when one call's result feeds another,\nor a large result should be filtered before it reaches you. Inside it, the same\npaths are async functions, and `tools.search` and `tools.search.describe` work\nas above. Return only the data you need. For depth, read the `code-mode` skill;\nit says how to see images and hear audio a tool returns, without reading base64.\nWhen `call` or `execute` pauses for approval or input, show the request to the\nuser and resume it; never repeat the call or run the program again.\n\n## Build an app\n\nBuild an app when no existing app provides the tools, data, UI or automation\nthe user needs. For a known service, search the catalog first:\n`organization.catalog({ query: { search: \"gmail\", limit: \"20\" } })`, then\n`apps.install` a match or `apps.importCustom` a remote MCP URL. With a shell,\nuse the `executor apps` CLI, else `execute`; read the `app-authoring` skill first.\n\n## Send feedback\n\nSend feedback with the Executor app's `feedback.submit` tool when Executor gets\nin your way, can't do something you need, works especially well, or the user\nhas feedback. Before your final reply, check whether anything is worth\nreporting. Be specific; never include credentials or user data (`feedback.md`)."; +export const executorIntro = "# Executor\n\n## What Executor is\n\nExecutor is a host that runs apps: TypeScript packages that expose tools,\nstorage, UI, accounts and background work. Agents reach every app through the\nMCP tools `skills`, `search`, `describe`, `call` and `execute`, with each app's\nslug as its namespace. Credentials go only through Executor's secure connection\nlinks, never through chat or source code. Some calls pause for user approval.\nThis is the Executor app's `executor` skill. Read its other skills with\n`skills({app: \"executor\", name: \"code-mode\"})`; `skills({})` lists them all.\n\n## Find and call tools\n\n`search({ query: \"...\" })` returns each tool's exact `path`, such as\n`tools..`, and input type; `describe({ paths })` returns full\nsignatures. Never guess argument names.\n\nFor a few independent calls, use `call({ calls: { path, input } })`; an array\nof calls runs concurrently. Each result has `ok` and its `value` or `error`.\n\nWrite a JavaScript program for `execute` when one call's result feeds another,\nor a large result should be filtered before it reaches you. Inside it, the same\npaths are async functions, and `tools.search` and `tools.search.describe` work\nas above. Return only the data you need. For depth, read the `code-mode` skill;\nit says how to see images, hear audio and read files tools return, not base64.\nWhen `call` or `execute` pauses for approval or input, show the request to the\nuser and resume it; never repeat the call or run the program again.\n\n## Build an app\n\nBuild an app when no existing app provides the tools, data, UI or automation\nthe user needs. For a known service, search the catalog first:\n`organization.catalog({ query: { search: \"gmail\", limit: \"20\" } })`, then\n`apps.install` a match or `apps.importCustom` a remote MCP URL. With a shell,\nuse the `executor apps` CLI, else `execute`; read the `app-authoring` skill first.\n\n## Send feedback\n\nSend feedback with the Executor app's `feedback.submit` tool when Executor gets\nin your way, can't do something you need, works especially well, or the user\nhas feedback. Before your final reply, check whether anything is worth\nreporting. Be specific; never include credentials or user data (`feedback.md`)."; diff --git a/packages/app-templates/src/implementation/executor-skill-digests.gen.ts b/packages/app-templates/src/implementation/executor-skill-digests.gen.ts index 7e503e0eec..5440f96075 100644 --- a/packages/app-templates/src/implementation/executor-skill-digests.gen.ts +++ b/packages/app-templates/src/implementation/executor-skill-digests.gen.ts @@ -4,7 +4,7 @@ export const executorSkillDigests: Readonly> = { "app-authoring/SKILL.md": "460e14bb77686cde90c21558281739df5d52fb8f5f683a218c481ce714ca0942", "app-authoring/accounts.md": "573fe7e7645d2c281598122972c068104215c0bafba5e6d04c96020ab1c3ca19", - "app-authoring/deploy.md": "823259311d78f579ddd6d10c05e87b926fd82ea9d8ea1713d366c1495b42eb1f", + "app-authoring/deploy.md": "1eca49c4cb62c27e8eb7cb798d326fe553fc5f66e434ae51898bbc26eec7d2dd", "app-authoring/events.md": "421c322f147a6da7a0bdc5513367e25b96667cf34f74b71918c84fdbe91727d1", "app-authoring/integrations.md": "1788ddee892c69a9d895a2d8c7bec63f1d41963ed5b00ed3e6f0b5c4a671d867", "app-authoring/starter.md": "382bd769d7b10e9c09eaa2367755f23153a31a484aaca9c449ab4aac7d61e224", @@ -15,7 +15,7 @@ export const executorSkillDigests: Readonly> = { "app-authoring/upgrades/0.0.1-beta.90.md": "38c727888b2773efa1d83795e8704f7c38ad9ec27c62fcdd89b5177ec5e338c3", "app-authoring/webhooks.md": "8b6e4fa3fbc1641bfa741bc449d8ff1b1db7c7923c2ddec0a134be7a2f7de5d8", "app-authoring/workflows.md": "1496edbdb85e138715efa138c63068a11674e6188179f6ee0a92b81e6d9d63e2", - "code-mode/SKILL.md": "ff91dd2c510765840d0ab03ef5ccfc1819b5d13cbcdcacdfc8335aa59a47a406", - "executor/SKILL.md": "afd7eeb8e66effd46173a22da3bde6aa820110a4122b32f6715b76245c6de227", + "code-mode/SKILL.md": "475e60fbf70654c439ceb13aa8ff73c22d79ee26dcf2ceb7898025d8b5740a02", + "executor/SKILL.md": "3e3d4b7b620f67f88c7f8d4ead095af8f0f56f29dbdcd9beab6fb53d27866f95", "executor/feedback.md": "29e684fc80cef9f9fae46e8fe183c310a64f12d2737dbe0bcb855a4450dfd95b" }; diff --git a/packages/apps/package.json b/packages/apps/package.json index 0375645b53..a6b2a209fb 100644 --- a/packages/apps/package.json +++ b/packages/apps/package.json @@ -1,6 +1,6 @@ { "name": "apps", - "version": "0.0.1-beta.138.shuv.1", + "version": "0.0.1-beta.141.shuv.1", "private": true, "description": "TypeScript apps with typed tools, accounts, storage, and UI.", "type": "module", diff --git a/packages/apps/src/contracts/openapi.ts b/packages/apps/src/contracts/openapi.ts index 966f7f002d..f06f901999 100644 --- a/packages/apps/src/contracts/openapi.ts +++ b/packages/apps/src/contracts/openapi.ts @@ -185,10 +185,23 @@ export const defaultOpenapiResponseLimits = { */ export const isOpenapiJsonSequence = (type: string): boolean => /^application\/(?:x-)?(?:ndjson|jsonl|jsonlines|json-seq)\s*(?:;|$)/i.test(type); +/** A type or subtype name, as RFC 6838 section 4.2 allows it. */ +const mediaName = "[a-z0-9][a-z0-9!#$&^_.+-]*"; +const jsonMedia = new RegExp( + `^(?:(?:application|text)/(?:x-)?json|application/x-amz-json-1\\.[01]|${mediaName}/${mediaName}\\+json)\\s*(?:;|$)`, + "i", +); +/** + * One JSON value: `application/json`, `text/json`, `application/x-json`, AWS's + * `application/x-amz-json-1.0` and `1.1`, or a structured `+json` type under any top-level type + * (RFC 6839). Not every type that contains "json": Tailscale's `application/hujson`, JSONC and + * JSON5 are text with comments, which encoding or parsing as JSON would break. + */ +export const isOpenapiJsonMedia = (type: string): boolean => jsonMedia.test(type); /** Media returned as text rather than a base64 file. JSON sequences remain unparsed text. */ export const isOpenapiTextMedia = (type: string): boolean => isOpenapiJsonSequence(type) || - /^(?:text\/|application\/(?:[\w.-]+\+)?(?:json|xml)|application\/(?:javascript|x-www-form-urlencoded))/i.test( + /^(?:text\/|application\/(?:[\w.-]+\+)?(?:json|xml)|application\/(?:hujson|javascript|x-www-form-urlencoded))/i.test( type, ); /** A form field that carries raw file bytes: OpenAPI 3.1 `contentMediaType` without an @@ -209,7 +222,7 @@ export const openapiBinaryResultSchema: JsonObject = { /** Classify request media for JSON-safe tool arguments; Swagger owns wire serialization. */ export const openapiMediaKind = (type: string) => - type.includes("json") && !isOpenapiJsonSequence(type) + isOpenapiJsonMedia(type) ? "json" : type === "application/x-www-form-urlencoded" ? "form" diff --git a/packages/apps/src/implementation/account-router.ts b/packages/apps/src/implementation/account-router.ts index 767da90f41..ffc045eb09 100644 --- a/packages/apps/src/implementation/account-router.ts +++ b/packages/apps/src/implementation/account-router.ts @@ -11,7 +11,10 @@ import { combineJsonSchemas, importedJsonSchema, once, + schemaIdentity, + schemaMemo, withLazyJsonSchemaDocument, + type SchemaMemo, } from "./schema.ts"; type Selection = { readonly accountId: string; readonly input: unknown }; @@ -39,10 +42,14 @@ const decoderDocument = (decoder: Schema.Decoder) => { * through many accounts would otherwise be described once per account. Branch definitions stay in * the root `$defs`, where a listing shares them among its tools. */ -const accountBranches = (variants: readonly (readonly [string, JsonObject])[]) => { +const accountBranches = ( + variants: readonly (readonly [string, JsonObject])[], + memo: SchemaMemo = schemaMemo(), +): JsonObject => { const groups = new Map(); for (const [accountId, schema] of variants) { - const key = JSON.stringify(schema); + // One account has one branch; only several need a key, which names definitions by number. + const key = variants.length === 1 ? "" : schemaIdentity(schema, memo); const group = groups.get(key); if (group === undefined) groups.set(key, { schema, first: accountId, rest: [] }); else group.rest.push(accountId); @@ -51,6 +58,7 @@ const accountBranches = (variants: readonly (readonly [string, JsonObject])[]) = const { $defs, schemas } = combineJsonSchemas( branches.map(({ schema }) => schema), (index) => `#/anyOf/${index}/properties/input`, + memo, ); return { type: "object", @@ -73,11 +81,15 @@ const accountBranches = (variants: readonly (readonly [string, JsonObject])[]) = }; /** Each distinct output schema once: a value matches the union whichever account returned it. */ -const outputBranches = (outputs: readonly JsonObject[]) => { - const { $defs, schemas } = combineJsonSchemas( - [...new Map(outputs.map((output) => [JSON.stringify(output), output])).values()], - (index) => `#/anyOf/${index}`, - ); +const outputBranches = ( + outputs: readonly JsonObject[], + memo: SchemaMemo = schemaMemo(), +): JsonObject => { + const distinct = + outputs.length === 1 + ? outputs + : [...new Map(outputs.map((output) => [schemaIdentity(output, memo), output])).values()]; + const { $defs, schemas } = combineJsonSchemas(distinct, (index) => `#/anyOf/${index}`, memo); return { ...($defs === undefined ? {} : { $defs }), anyOf: schemas }; }; @@ -207,24 +219,43 @@ const mergeSummary = ( }); }; +/** + * A merged tool with its schemas read as objects of entries. Their values are checked by a catalog + * read, which checks every tool's schemas in one pass (`DescribedTools`) and so walks a definition + * many tools share once. Checking them for each merged tool walked those definitions once per tool, + * as did writing each tool's schemas to compare its accounts': for Stripe's description that cost + * seconds of CPU for every account a listing merges. + */ +const Entries = Schema.Record(Schema.String, Schema.Unknown); +const MergedTool = HostedTool.mapFields((fields) => ({ + ...fields, + inputSchema: Entries, + outputSchema: Schema.optionalKey(Entries), + _meta: Schema.optionalKey(Entries), +})); + /** Calls take { accountId, input }, so each distinct account schema is one branch of the union. */ -const merge = (name: string, variants: readonly (readonly [string, HostedTool])[]) => - Schema.decodeUnknownSync(HostedTool)({ +const merge = ( + name: string, + variants: readonly (readonly [string, HostedTool])[], + memo: SchemaMemo, +): HostedTool => { + const outputs = variants.flatMap(([, tool]) => + tool.outputSchema === undefined ? [] : [tool.outputSchema], + ); + const merged: HostedTool = { ...mergeSummary(name, variants), inputSchema: accountBranches( variants.map(([accountId, tool]) => [accountId, tool.inputSchema]), + memo, ), - ...(variants.every(([, tool]) => tool.outputSchema !== undefined) - ? { - outputSchema: outputBranches( - variants.flatMap(([, tool]) => - tool.outputSchema === undefined ? [] : [tool.outputSchema], - ), - ), - } - : {}), + ...(outputs.length === variants.length ? { outputSchema: outputBranches(outputs, memo) } : {}), ...sameAcross("_meta", variants), - }); + }; + // A tool that does not fit fails its source here, as before, so its router reports it. + Schema.decodeUnknownSync(MergedTool)(merged); + return merged; +}; /** Accounts whose sources are read at once. Each read may reach its account's upstream service. */ const accountConcurrency = 8; @@ -305,7 +336,15 @@ const combineDynamic = (sources: ReadonlyMap): DynamicRou }), }), ...(first?.overrides === undefined ? {} : { overrides: first.overrides }), - list: () => grouped((source) => source.list(), merge), + // One memo per listing: its tools share definition objects, which it reads once. + list: () => + Effect.suspend(() => { + const memo = schemaMemo(); + return grouped( + (source) => source.list(), + (name, variants) => merge(name, variants, memo), + ); + }), summaries: () => grouped( (source) => (source.summaries === undefined ? source.list() : source.summaries()), @@ -324,7 +363,7 @@ const combineDynamic = (sources: ReadonlyMap): DynamicRou // No account that answered has it; it may be a failed account's. if (variants.length === 0) return failed.size > 0 ? Effect.fail(firstFailure(failed)) : Effect.succeed(undefined); - return Effect.succeed(merge(name, variants)); + return Effect.succeed(merge(name, variants, schemaMemo())); }), ), resolve: (name: string) => diff --git a/packages/apps/src/implementation/openapi-compile.ts b/packages/apps/src/implementation/openapi-compile.ts index 1f6a8c6eb4..a628b5b0f6 100644 --- a/packages/apps/src/implementation/openapi-compile.ts +++ b/packages/apps/src/implementation/openapi-compile.ts @@ -7,7 +7,7 @@ import { OpenapiOperation, openapiMediaKind, isOpenapiFileSchema, - isOpenapiJsonSequence, + isOpenapiJsonMedia, isOpenapiTextMedia, openapiBinaryResultSchema, } from "../contracts/openapi.ts"; @@ -175,7 +175,7 @@ function responseSchema( const media = Object.entries(record(content)); if (media.length === 0) return undefined; for (const [type, body] of media) { - if (!type.includes("json") || isOpenapiJsonSequence(type)) { + if (!isOpenapiJsonMedia(type)) { shapes.push(() => isOpenapiTextMedia(type) ? { type: "string" } : openapiBinaryResultSchema, ); diff --git a/packages/apps/src/implementation/openapi-request.ts b/packages/apps/src/implementation/openapi-request.ts index c17f722637..1c9f488f70 100644 --- a/packages/apps/src/implementation/openapi-request.ts +++ b/packages/apps/src/implementation/openapi-request.ts @@ -26,7 +26,7 @@ import { OpenapiMediaType, defaultOpenapiResponseLimits, isOpenapiFileSchema, - isOpenapiJsonSequence, + isOpenapiJsonMedia, isOpenapiTextMedia, openapiMediaKind, type CredentialBinding, @@ -468,13 +468,14 @@ export function createRequest(config: { data.set(chunk, offset); offset += chunk.length; } + // JSON first: a JSON type the text gate does not name, such as AWS's, still has a JSON + // result schema. + if (isOpenapiJsonMedia(contentType)) + return yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json))( + new TextDecoder().decode(data), + ).pipe(Effect.mapError(() => unreadable)); if (!isOpenapiTextMedia(contentType)) return { base64: Base64.encode(data), contentType }; - const text = new TextDecoder().decode(data); - return contentType.includes("json") && !isOpenapiJsonSequence(contentType) - ? yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json))(text).pipe( - Effect.mapError(() => unreadable), - ) - : text; + return new TextDecoder().decode(data); }), ).pipe( owned("upstream", "provider.openapi.call", { diff --git a/packages/apps/src/implementation/schema.ts b/packages/apps/src/implementation/schema.ts index f029b67172..80967388ab 100644 --- a/packages/apps/src/implementation/schema.ts +++ b/packages/apps/src/implementation/schema.ts @@ -414,13 +414,11 @@ const ownDefinition = (reference: unknown) => { return segments.length === 2 ? segments[1] : undefined; }; /** - * The root `$defs` of a schema whose every reference names one of them, so the schema reads the - * same beside them at any root. A schema referring to its root, a nested pointer or an identifier - * has none. Only schema positions are read, as the validator evaluates them. + * The `$defs` names a schema's references name, read at schema positions only, as the validator + * evaluates them; undefined when one refers to anything else or the schema uses an identifier. */ -const rootDefinitions = (schema: JsonObject): JsonObject | undefined => { - const definitions = schema.$defs ?? {}; - if (!isJsonRecord(definitions)) return undefined; +const definitionReferences = (schema: unknown): ReadonlySet | undefined => { + const names = new Set(); const pending: unknown[] = [schema]; for (let value = pending.pop(); value !== undefined; value = pending.pop()) { if (!isSchemaObject(value)) continue; @@ -428,7 +426,8 @@ const rootDefinitions = (schema: JsonObject): JsonObject | undefined => { if (identifiers.has(key)) return undefined; if (key === "$ref") { const name = ownDefinition(item); - if (name === undefined || !Object.hasOwn(definitions, name)) return undefined; + if (name === undefined) return undefined; + names.add(name); } else if (Array.isArray(item)) { if (schemaArrays.has(key) || key === "items") pending.push(...item); } else if (schemaMaps.has(key) || key === "dependencies") { @@ -436,34 +435,113 @@ const rootDefinitions = (schema: JsonObject): JsonObject | undefined => { } else if (schemaNodes.has(key)) pending.push(item); } } + return names; +}; + +/** + * What one listing learns about the definitions its tools' schemas hold. Routers such as the + * OpenAPI importer give every tool that reaches a definition the same object, and a densely + * connected API reaches most of its definitions from every tool: Stripe's 610 tools name about + * 330,000 `$defs` entries but only a few thousand distinct definitions. Combining schemas reads + * each definition object once per listing, rather than once for every tool that reaches it, so a + * listing's cost follows its distinct definitions instead of its tools times its definitions. + * Values are only read, never changed: the memo is valid while the listing's objects are, so + * make one for each listing or description and drop it with them. + */ +export interface SchemaMemo { + /** Equal numbers for definitions whose JSON text is equal. */ + readonly textId: (definition: JsonValue) => number; + /** The `$defs` names the definition refers to, or undefined, as `definitionReferences`. */ + readonly references: (definition: JsonValue) => ReadonlySet | undefined; +} +export const schemaMemo = (): SchemaMemo => { + const ids = new Map(); + const objectIds = new WeakMap(); + const textId = (definition: JsonValue) => { + const known = + typeof definition === "object" && definition !== null ? objectIds.get(definition) : undefined; + if (known !== undefined) return known; + const text = JSON.stringify(definition); + let id = ids.get(text); + if (id === undefined) ids.set(text, (id = ids.size)); + if (typeof definition === "object" && definition !== null) objectIds.set(definition, id); + return id; + }; + const referenced = new WeakMap | undefined>(); + const references = (definition: JsonValue) => { + if (typeof definition !== "object" || definition === null) + return definitionReferences(definition); + if (referenced.has(definition)) return referenced.get(definition); + const names = definitionReferences(definition); + referenced.set(definition, names); + return names; + }; + return { textId, references }; +}; + +/** + * The root `$defs` of a schema whose every reference names one of them, so the schema reads the + * same beside them at any root. A schema referring to its root, a nested pointer or an identifier + * has none. Each definition's references come from the memo, so a definition many tools share is + * walked once. + */ +const rootDefinitions = (schema: JsonObject, memo: SchemaMemo): JsonObject | undefined => { + const definitions = schema.$defs ?? {}; + if (!isJsonRecord(definitions)) return undefined; + const { $defs: _, ...root } = schema; + for (const names of [ + definitionReferences(root), + ...Object.values(definitions).map(memo.references), + ]) { + if (names === undefined) return undefined; + for (const name of names) if (!Object.hasOwn(definitions, name)) return undefined; + } return definitions; }; +/** + * A key equal for schemas whose JSON text is equal, without writing the definitions in their + * `$defs`: each definition stands for its memo number. + */ +export const schemaIdentity = (schema: JsonObject, memo: SchemaMemo) => + JSON.stringify( + Object.entries(schema).map(([key, value]) => + key === "$defs" && isJsonRecord(value) + ? [key, Object.entries(value).map(([name, definition]) => [name, memo.textId(definition)])] + : [key, value], + ), + ); + /** * Self-contained schemas placed side by side in one document, such as the branches of a union. * A schema whose references all name its root `$defs` gives those entries to the document's root * `$defs` when no entry of the same name differs, so definitions stay where listings share them * among tools. Any other schema is relocated with `nestJsonSchema` under `pointer(index)`. + * Definitions are compared by object first and by text only when two objects share a name. */ export const combineJsonSchemas = ( schemas: readonly JsonObject[], pointer: (index: number) => string, + memo: SchemaMemo = schemaMemo(), ): { readonly $defs?: JsonObject; readonly schemas: readonly JsonObject[] } => { const shared: Record = {}; - const texts = new Map(); + const kept = new Map(); + const differs = (name: string, definition: JsonValue) => { + if (!kept.has(name)) return false; + const first = kept.get(name) ?? null; + return first !== definition && memo.textId(first) !== memo.textId(definition); + }; const combined = schemas.map((schema, index) => { - const definitions = rootDefinitions(schema); - const entries = Object.entries(definitions ?? {}).map( - ([name, definition]) => [name, definition, JSON.stringify(definition)] as const, - ); + const definitions = rootDefinitions(schema, memo); + const entries = Object.entries(definitions ?? {}); if ( definitions === undefined || - entries.some(([name, , text]) => texts.has(name) && texts.get(name) !== text) + entries.some(([name, definition]) => differs(name, definition)) ) return nestJsonSchema(schema, pointer(index)); - for (const [name, definition, text] of entries) { - if (texts.has(name)) continue; - texts.set(name, text); + for (const [name, definition] of entries) { + if (kept.has(name)) continue; + kept.set(name, definition); // Defined as an own entry: assignment would set the prototype of an entry named `__proto__`. Object.defineProperty(shared, name, { value: definition, @@ -475,7 +553,7 @@ export const combineJsonSchemas = ( const { $defs: _, ...rest } = schema; return rest; }); - return texts.size === 0 ? { schemas: combined } : { $defs: shared, schemas: combined }; + return kept.size === 0 ? { schemas: combined } : { $defs: shared, schemas: combined }; }; // Walk schema positions only. A property named "format" or "default" is still ordinary user data. diff --git a/packages/apps/src/implementation/ui-document.ts b/packages/apps/src/implementation/ui-document.ts index 8c3659727e..72786ccc47 100644 --- a/packages/apps/src/implementation/ui-document.ts +++ b/packages/apps/src/implementation/ui-document.ts @@ -57,19 +57,28 @@ const retained = (url: string) => url !== "" && !/^(?:[a-z][a-z\d+.-]*:|[/#?])/i * Rewrite each candidate URL as the HTML `srcset` parser splits them: a URL is a run of * non-whitespace whose trailing commas end the candidate, and descriptors end at a comma outside * parentheses. Commas inside a URL, as in `data:` URLs, are part of it. + * + * Every step reads forward from `index`, and trailing commas are counted back from the token's + * end, so the time taken is linear in the attribute's length whatever it holds. */ const rewriteSrcset = (value: string, resolve: (url: string) => string) => { + const separators = /[\s,]*/y; + const tokens = /\S+/y; let output = ""; let index = 0; while (index < value.length) { - const separator = /^[\s,]*/.exec(value.slice(index))?.[0] ?? ""; + separators.lastIndex = index; + const separator = separators.exec(value)?.[0] ?? ""; output += separator; index += separator.length; - const token = /^\S+/.exec(value.slice(index))?.[0]; + tokens.lastIndex = index; + const token = tokens.exec(value)?.[0]; if (token === undefined) break; index += token.length; - const commas = /,*$/.exec(token)?.[0] ?? ""; - output += resolve(token.slice(0, token.length - commas.length)) + commas; + let url = token.length; + while (url > 0 && token[url - 1] === ",") url--; + const commas = token.slice(url); + output += resolve(token.slice(0, url)) + commas; if (commas !== "") continue; let depth = 0; let end = index; diff --git a/packages/mcp/src/contracts/media.ts b/packages/mcp/src/contracts/media.ts index 2e8cce1130..f527e1b6c0 100644 --- a/packages/mcp/src/contracts/media.ts +++ b/packages/mcp/src/contracts/media.ts @@ -1,11 +1,20 @@ -/** How images and audio in an execution result reach the MCP client. */ +/** How images, audio and files in an execution result reach the MCP client. */ import { Option, Schema } from "effect"; import { McpSchema, type McpServer } from "effect/ai"; -type MediaBlock = typeof McpSchema.ImageContent.Encoded | typeof McpSchema.AudioContent.Encoded; +type BlobResource = typeof McpSchema.EmbeddedResource.Encoded & { + readonly resource: typeof McpSchema.BlobResourceContents.Encoded; +}; +type MediaBlock = + | typeof McpSchema.ImageContent.Encoded + | typeof McpSchema.AudioContent.Encoded + | BlobResource; const isImage = Schema.decodeUnknownOption(McpSchema.ImageContent); const isAudio = Schema.decodeUnknownOption(McpSchema.AudioContent); +const isBlobResource = Schema.decodeUnknownOption( + Schema.Struct({ type: Schema.Literal("resource"), resource: McpSchema.BlobResourceContents }), +); /** * Standard padded base64 for data in any common base64 form. Services and programs return base64url @@ -18,39 +27,69 @@ const standardBase64 = (data: string) => { : compact.padEnd(compact.length + 4 - (compact.length % 4), "="); }; -/** One MCP image or audio block whose data decodes as base64, with its data in the standard form. */ -const mediaBlock = (value: unknown): MediaBlock | undefined => { - if (typeof value !== "object" || value === null || !("type" in value)) return undefined; - if (value.type !== "image" && value.type !== "audio") return undefined; - if (!("data" in value) || typeof value.data !== "string") return undefined; - const block = { ...value, data: standardBase64(value.data) }; - const valid = - value.type === "image" ? Option.isSome(isImage(block)) : Option.isSome(isAudio(block)); - return valid ? (block as MediaBlock) : undefined; +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +/** One media block, the key that identifies its bytes, and what the JSON keeps in its place. */ +interface Lifted { + readonly block: MediaBlock; + readonly key: string; + readonly placeholder: (contentIndex: number) => Record; +} + +/** + * An MCP image or audio block, or an embedded resource holding a blob, whose data decodes as + * base64, with its data in the standard form. A text resource is already readable and stays in the + * JSON. + */ +const mediaBlock = (value: Record): Lifted | undefined => { + if ((value.type === "image" || value.type === "audio") && typeof value.data === "string") { + const block = { ...value, data: standardBase64(value.data) }; + const valid = + value.type === "image" ? Option.isSome(isImage(block)) : Option.isSome(isAudio(block)); + if (!valid) return undefined; + const media = block as typeof McpSchema.ImageContent.Encoded; + return { + block: media, + key: `${media.type}\0${media.mimeType}\0${media.data}`, + placeholder: (contentIndex) => ({ type: media.type, mimeType: media.mimeType, contentIndex }), + }; + } + if (value.type !== "resource" || !isRecord(value.resource)) return undefined; + const { blob } = value.resource; + if (typeof blob !== "string") return undefined; + const block = { ...value, resource: { ...value.resource, blob: standardBase64(blob) } }; + if (Option.isNone(isBlobResource(block))) return undefined; + const { uri, mimeType } = (block as BlobResource).resource; + const described = mimeType === undefined ? { uri } : { uri, mimeType }; + return { + block: block as BlobResource, + key: `resource\0${uri}\0${mimeType ?? ""}\0${block.resource.blob}`, + placeholder: (contentIndex) => ({ type: "resource", resource: described, contentIndex }), + }; }; /** - * An execution result as JSON text and `structuredContent`, with each MCP image or audio block it - * holds moved into its own content block. The JSON keeps `{ type, mimeType, contentIndex }` in its - * place, the block's index in `content`, so the bytes reach the model as media instead of base64 - * text, once even when the result repeats them. + * An execution result as JSON text and `structuredContent`, with each MCP image, audio or blob + * resource block it holds moved into its own content block. The JSON keeps the block without its + * bytes, plus `contentIndex`, the block's index in `content`, so the bytes reach the client as + * media or a file instead of base64 text, once even when the result repeats them. */ export const executionResultContent: McpServer.ResultContent["Service"] = (encoded) => { const media: Array = []; const indexes = new Map(); const lift = (value: unknown): unknown => { if (Array.isArray(value)) return value.map(lift); - if (typeof value !== "object" || value === null) return value; - const block = mediaBlock(value); - if (block !== undefined) { - const key = `${block.type}\0${block.mimeType}\0${block.data}`; - let contentIndex = indexes.get(key); + if (!isRecord(value)) return value; + const lifted = mediaBlock(value); + if (lifted !== undefined) { + let contentIndex = indexes.get(lifted.key); if (contentIndex === undefined) { - media.push(block); + media.push(lifted.block); contentIndex = media.length; - indexes.set(key, contentIndex); + indexes.set(lifted.key, contentIndex); } - return { type: block.type, mimeType: block.mimeType, contentIndex }; + return lifted.placeholder(contentIndex); } return Object.fromEntries(Object.entries(value).map(([key, field]) => [key, lift(field)])); }; diff --git a/packages/mcp/src/implementation/execute.ts b/packages/mcp/src/implementation/execute.ts index f2f76229e8..4ab198cc59 100644 --- a/packages/mcp/src/implementation/execute.ts +++ b/packages/mcp/src/implementation/execute.ts @@ -1287,6 +1287,17 @@ function catalog( }); yield* settled; }).pipe(Effect.withSpan("mcp.search.discovery.batch")); + /** + * The batch each app is made searchable in: the apps the first search that reached it reached + * and no earlier search had, so the backend reads a search's apps together and never with an + * app only another search reaches. + */ + const searchBatches = new Map>(); + const claimSearchBatch = (selected: ReadonlyArray<{ readonly app: ListedApp }>) => + Effect.sync(() => { + const batch = selected.map(({ app }) => app.id).filter((app) => !searchBatches.has(app)); + for (const app of batch) searchBatches.set(app, batch); + }); const searchedApps = yield* Effect.forEach(apps, (app) => Effect.cached( Effect.suspend(() => { @@ -1297,7 +1308,11 @@ function catalog( backend.searchableTools( { app: app.id, accounts }, // Search reads every app's rows; refreshing aged ones would load each app's Worker. - { reportRunningAfterMillis: discoveryWaitMs, refreshStale: false }, + { + reportRunningAfterMillis: discoveryWaitMs, + refreshStale: false, + searchBatch: searchBatches.get(app.id) ?? [app.id], + }, ), ); }).pipe( @@ -1424,6 +1439,7 @@ function catalog( Effect.gen(function* () { const selected = named(searchedApps, names); yield* searchAtOnce(selected.map(({ app }) => app)); + yield* claimSearchBatch(selected); yield* load(selected); if (names !== "all") yield* revealNamed(selected); const targets = selected diff --git a/packages/sdk/scripts/build-retain-fixture.ts b/packages/sdk/scripts/build-retain-fixture.ts new file mode 100644 index 0000000000..c5be382378 --- /dev/null +++ b/packages/sdk/scripts/build-retain-fixture.ts @@ -0,0 +1,253 @@ +/** + * Local-only probe of what a deploy says when its build's retain stage meets a store that refuses + * writes to one key faster than a limit, through the SDK's own deploy path: `executor.apps.deploy`, + * the real `appRuntime` and the real `retainWorkerBuild`, PGlite in memory, and a stub in place of + * the compiler only. Every build links the same framework, so concurrent deploys write the same + * framework object, as Cloud's builds did on 17 Nov (17 writes in 2 s, one refused with R2's + * `r2.10058`, its deploy answered 422). + * + * The stand-in store refuses a put to a key while another put to that key is in flight or finished + * under a second ago, as R2 does (one write a second per key), and otherwise stores it after + * 300 ms. Prints, as JSON, each deploy's outcome and what the store saw, for + * `build-retain-storage.spec.ts` to judge. + * + * - `concurrent`: 17 deploys at once. + * - `refused`: the store refuses every write to the framework's key: the deploy fails. + * - `source`: the app's own code does not compile: the deploy fails once, unrepeated. + * - `ui-name`: a UI file's name holds a control character, which source paths allow and no + * storage key does: the deploy fails once as the app's, unrepeated. + * - `invalid-key`: the store refuses the framework's key as invalid (HTTP 400, as R2's 10020 for + * a key over 1,024 bytes): the deploy fails once as the app's, unrepeated. + * - `spans`: one deploy under a tracer that keeps every span: each write the deploy makes through + * the executor's own store (its version source and initial source) has a `storage.blob.put` + * span with its kind, size and outcome. + */ +import { Effect, Layer, Redacted, Schema, Stream, Tracer } from "effect"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import { pgliteLayer } from "fumadb-effect/pglite"; +import { SqlClient } from "effect/sql"; +import { BlobKey, BlobStore, BlobStoreError, type BlobStorage } from "../src/contracts/blobs.ts"; +import { BuildId } from "../src/contracts/shared.ts"; +import { BuiltApp, RuntimeBuildFailed } from "../src/contracts/runtime.ts"; +import { CurrentActor, createExecutor, makeExecutorStorage, runtimeAdapter } from "../src/core.ts"; +import { appRuntime } from "../src/implementation/app-runtime.ts"; +import { memoryBlobStore } from "../src/implementation/memory-blobs.ts"; +import { retainWorkerBuild } from "../src/implementation/worker-build-storage.ts"; +import { markdownRefusals } from "../src/markdown.ts"; +import { memoryRegistry, memoryRepositories } from "../src/testing.ts"; + +const cases = ["concurrent", "refused", "source", "ui-name", "invalid-key", "spans"] as const; +type Case = (typeof cases)[number]; +const name = process.argv[process.argv.indexOf("--case") + 1] as Case; +if (!cases.includes(name)) throw new Error(`--case must be one of ${cases.join(", ")}`); + +const writeMs = 300; +const spacingMs = 1_000; +/** What the stand-in saw of writes to the framework's key. */ +const framework = { stored: 0, refused: 0, existsAnswers: [] as boolean[] }; +let builds = 0; + +/** A store over memory that refuses a key's writes faster than one a second, as R2 does. */ +const limitedStore = (inner: BlobStorage): BlobStorage => { + const lastWrite = new Map(); + const isFramework = (key: string) => key.startsWith("frameworks/"); + return { + ...inner, + exists: (key) => + inner + .exists(key) + .pipe( + Effect.tap((found) => + Effect.sync(() => (isFramework(key) ? framework.existsAnswers.push(found) : undefined)), + ), + ), + put: (key, body) => + Effect.gen(function* () { + const now = Date.now(); + const last = lastWrite.get(key); + if (name === "invalid-key" && isFramework(key)) { + framework.refused += 1; + return yield* new BlobStoreError({ operation: "put", status: 400 }); + } + if ( + (name === "refused" && isFramework(key)) || + (last !== undefined && now - last < spacingMs) + ) { + if (isFramework(key)) framework.refused += 1; + return yield* new BlobStoreError({ operation: "put", status: 429 }); + } + // The write holds the key from its start to a second after it lands. + lastWrite.set(key, now + writeMs); + yield* Effect.sleep(`${writeMs} millis`); + yield* inner.put(key, body); + lastWrite.set(key, Date.now()); + if (isFramework(key)) framework.stored += 1; + }), + }; +}; + +const blobs = limitedStore(memoryBlobStore()); +/** The same framework for every build, as one host release links one. */ +const sharedFramework = { + version: "0.0.1-fixture", + modules: { "apps.js": "export const apps = 1;" }, +}; + +/** The compiler stub: compiles instantly, then retains through the real retain stage. */ +const transport = { + name: "runtime.stub", + loadBuild: () => Effect.die(new Error("The stub runtime keeps no builds")), + build: () => + Effect.gen(function* () { + builds += 1; + if (name === "source") + return yield* new RuntimeBuildFailed({ + stage: "compile", + message: "index.ts:1:1: ERROR: Expected an expression", + }); + const build = yield* Schema.decodeUnknownEffect(BuildId)(`bld_${crypto.randomUUID()}`).pipe( + Effect.orDie, + ); + yield* retainWorkerBuild( + build, + { + mainModule: "index.js", + modules: { "index.js": "export default {};" }, + database: false, + protocol: 1, + }, + sharedFramework, + name === "ui-name" + ? [ + { + path: "assets/report\u0007.js", + contentType: "text/javascript", + body: new TextEncoder().encode("export {};"), + }, + ] + : undefined, + ).pipe(Effect.provideService(BlobStore, blobs)); + return yield* Schema.decodeUnknownEffect(BuiltApp)({ + build, + requirements: { accounts: { app: {}, user: {} } }, + }).pipe(Effect.orDie); + }), + asset: () => Effect.succeed(undefined), + changes: () => Stream.empty, + invoke: () => Effect.die(new Error("The stub runtime runs no tools")), +}; + +/** Every span the deploy opens, kept to read their attributes once they end. */ +const spans: Array = []; +const keepingTracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + spans.push(span); + return span; + }, +}); + +const program = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* makeExecutorStorage({ provider: "postgresql" }).pipe( + Effect.provideService(SqlClient.SqlClient, sql), + ); + yield* storage.migrate; + const runtime = yield* appRuntime(transport as never); + const executor = yield* createExecutor({ + database: storage, + secret: Redacted.make("0".repeat(64)), + git: memoryRepositories(), + markdown: markdownRefusals, + blobs, + runtime: runtimeAdapter(runtime as never), + registry: memoryRegistry(), + } as never); + const deploy = (index: number) => + executor.apps + .deploy({ + owner: "local", + name: `Retain ${index}`, + files: [ + { path: "index.ts", content: "export default {}" }, + { + path: "package.json", + content: JSON.stringify({ type: "module", dependencies: { apps: "0.0.1" } }), + }, + ], + } as never) + .pipe( + Effect.match({ + onSuccess: () => ({ outcome: "deployed" as const }), + onFailure: (error) => { + const failure = error as unknown as Record; + return { + outcome: "failed" as const, + tag: failure._tag, + stage: failure.stage, + message: typeof failure.message === "string" ? failure.message : undefined, + }; + }, + }), + ); + const count = name === "concurrent" ? 17 : 1; + const started = Date.now(); + const deploys = yield* Effect.forEach( + Array.from({ length: count }, (_, index) => index), + deploy, + { concurrency: "unbounded" }, + ); + // The bytes the deploy's two source writes carry: its files as JSON. + const sourceBytes = new TextEncoder().encode( + JSON.stringify([ + { path: "index.ts", content: "export default {}" }, + { + path: "package.json", + content: JSON.stringify({ type: "module", dependencies: { apps: "0.0.1" } }), + }, + ]), + ).byteLength; + const writes = spans + .filter((span) => span.name === "storage.blob.put") + .map((span) => ({ + kind: span.attributes.get("storage.blob.kind"), + size: span.attributes.get("storage.blob.size"), + outcome: span.attributes.get("storage.blob.outcome"), + ended: span.status._tag === "Ended", + keys: [...span.attributes.keys()].sort(), + })); + return { + deploys, + builds, + framework, + ...(name === "spans" && { writes, sourceBytes }), + elapsedMs: Date.now() - started, + frameworkObjects: (yield* blobs.list( + yield* Schema.decodeUnknownEffect(BlobKey)("frameworks").pipe(Effect.orDie), + { + limit: 10, + }, + )).names.length, + }; +}).pipe( + Effect.provideService(CurrentActor, "local" as never), + name === "spans" ? Effect.withTracer(keepingTracer) : (self) => self, +); + +Effect.runPromise( + Effect.scoped(program).pipe( + Effect.provide( + Layer.mergeAll(pgliteLayer({}), NodeCrypto.layerSecureRandom, NodeCrypto.layerDigest), + ), + Effect.timeout("120 seconds"), + ) as never, +).then( + (result) => { + console.log(JSON.stringify(result)); + process.exit(0); + }, + (error) => { + console.error(String(error).slice(0, 2000)); + process.exit(1); + }, +); diff --git a/packages/sdk/scripts/call-path-fixture.ts b/packages/sdk/scripts/call-path-fixture.ts new file mode 100644 index 0000000000..999933fb48 --- /dev/null +++ b/packages/sdk/scripts/call-path-fixture.ts @@ -0,0 +1,168 @@ +/** + * Local-only probe of what a tool call says when its exchange with the app runtime never formed, + * through the SDK's own call path: `executor.tools.call` and `executor.tools.resume`, the real + * `appRuntime` (which marks each hand-off to the app's code), PGlite in memory, and a stub in + * place of the runtime's transport only. The stub answers the catalog read for a call's kind, + * asks approval for a call when told to, and fails a call the way the transport does when its + * socket never opened twice (`RuntimeProtocolFailed`, `notSent`). Prints, as JSON, what the caller + * received, for `runtime-connect-repeat.spec.ts` to judge. + * + * - `call-once`: the call names its kind, so the call is its only hand-off. + * - `call-after-kind-read`: the call names no kind, so the catalog read for its kind comes first. + * - `call-resumed`: the call asks approval; the resumed call is its second hand-off. + */ +import { Effect, Layer, Redacted, Schema, Stream } from "effect"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import { pgliteLayer } from "fumadb-effect/pglite"; +import { SqlClient } from "effect/sql"; +import { approvalElicitation } from "apps/contracts"; +import { BuiltApp, RuntimeProtocolFailed } from "../src/contracts/runtime.ts"; +import { CurrentActor, createExecutor, makeExecutorStorage, runtimeAdapter } from "../src/core.ts"; +import { appRuntime } from "../src/implementation/app-runtime.ts"; +import { memoryBlobStore } from "../src/implementation/memory-blobs.ts"; +import { markdownRefusals } from "../src/markdown.ts"; +import { memoryRegistry, memoryRepositories } from "../src/testing.ts"; + +const cases = ["call-once", "call-after-kind-read", "call-resumed"] as const; +type Case = (typeof cases)[number]; +const name = process.argv[process.argv.indexOf("--case") + 1] as Case; +if (!cases.includes(name)) throw new Error(`--case must be one of ${cases.join(", ")}`); + +/** What the stub transport was asked for, in order. */ +const operations: string[] = []; +let calls = 0; + +/** The transport the real `appRuntime` sends each invocation through. */ +const transport = { + name: "runtime.stub", + loadBuild: () => Effect.die(new Error("The stub runtime keeps no builds")), + build: () => + Schema.decodeUnknownEffect(BuiltApp)({ + build: "bld_00000000-0000-4000-8000-000000000001", + requirements: { accounts: { app: {}, user: {} } }, + }), + asset: () => Effect.succeed(undefined), + changes: () => Stream.empty, + invoke: (invocation: { readonly command: { readonly operation: string } }) => { + operations.push(invocation.command.operation); + switch (invocation.command.operation) { + case "inspect": + // The tool declares no read-only kind: a mutation. + return Effect.succeed({ + ok: true, + value: { + tools: [{ name: "probe", description: "Probe", inputSchema: { type: "object" } }], + routers: [], + }, + }); + case "call": + calls += 1; + // Asked approval first when the case says so; after that, never connected. + return name === "call-resumed" && calls === 1 + ? Effect.succeed({ + ok: false, + error: { + _tag: "HostToolApprovalRequired", + input: {}, + elicitation: approvalElicitation("probe", {}), + }, + }) + : Effect.fail(new RuntimeProtocolFailed({ reason: "disconnected", notSent: true })); + default: + return Effect.die( + new Error(`The stub runtime does not run ${invocation.command.operation}`), + ); + } + }, +}; + +const failureOf = (exit: unknown) => { + const reasons = (exit as { cause?: { reasons?: Array<{ error?: Record }> } }) + .cause?.reasons; + const error = reasons?.[0]?.error; + return error === undefined + ? undefined + : { + tag: error._tag, + reason: error.reason, + notSent: error.notSent === true, + mayHaveWritten: error.mayHaveWritten === true, + }; +}; + +const program = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* makeExecutorStorage({ provider: "postgresql" }).pipe( + Effect.provideService(SqlClient.SqlClient, sql), + ); + yield* storage.migrate; + const runtime = yield* appRuntime(transport as never); + const executor = yield* createExecutor({ + database: storage, + secret: Redacted.make("0".repeat(64)), + git: memoryRepositories(), + markdown: markdownRefusals, + blobs: memoryBlobStore(), + runtime: runtimeAdapter(runtime as never), + registry: memoryRegistry(), + } as never); + const { app } = yield* executor.apps.deploy({ + owner: "local", + name: "Probe", + files: [ + { path: "index.ts", content: "export default {}" }, + { + path: "package.json", + content: JSON.stringify({ type: "module", dependencies: { apps: "0.0.1" } }), + }, + ], + } as never); + const call = (kind?: "mutation") => + executor.tools.call({ + app: app.id, + tool: "probe", + input: {}, + ...(kind === undefined ? {} : { kind }), + } as never); + if (name === "call-resumed") { + const asked = (yield* call("mutation")) as { status?: string; requestId?: string }; + const resumed = yield* Effect.exit( + executor.tools.resume({ + requestId: asked.requestId, + response: { action: "accept" }, + } as never), + ); + // A resume answers with its outcome; a failed one carries the call's error. + const answered = + resumed._tag === "Success" ? (resumed.value as { error?: unknown }) : undefined; + return { + asked: asked.status, + failure: + answered?.error === undefined + ? failureOf(resumed) + : failureOf({ cause: { reasons: [{ error: answered.error }] } }), + resumed: resumed._tag === "Success" ? JSON.stringify(resumed.value).slice(0, 400) : undefined, + operations, + }; + } + const exit = yield* Effect.exit(call(name === "call-once" ? "mutation" : undefined)); + return { failure: failureOf(exit), operations }; +}).pipe(Effect.provideService(CurrentActor, "local" as never)); + +Effect.runPromise( + Effect.scoped(program).pipe( + Effect.provide( + Layer.mergeAll(pgliteLayer({}), NodeCrypto.layerSecureRandom, NodeCrypto.layerDigest), + ), + Effect.timeout("60 seconds"), + ) as never, +).then( + (result) => { + console.log(JSON.stringify(result)); + process.exit(0); + }, + (error) => { + console.error(String(error).slice(0, 2000)); + process.exit(1); + }, +); diff --git a/packages/sdk/scripts/runtime-connect-fixture.ts b/packages/sdk/scripts/runtime-connect-fixture.ts new file mode 100644 index 0000000000..49633cef6f --- /dev/null +++ b/packages/sdk/scripts/runtime-connect-fixture.ts @@ -0,0 +1,717 @@ +/** + * Local-only probe of the one repeat of a runtime exchange whose connection never formed + * (`runtime-connect.ts`). Each case runs an exchange against a stub on 127.0.0.1 and prints, as + * JSON, how it ended, how many connections the stub saw and what the span recorded, for + * `runtime-connect-repeat.spec.ts` to judge. The RPC cases compose a session as `workerd-apps.ts` + * does: a `ws` socket, the hop's tracker, the open watch, nothing sent before it opens. The HTTP + * cases use the same `FetchHttpClient` the runtime's hops use, on this process's runtime (Node or + * Bun, whichever runs the script). + */ +import { Cause, Effect, Exit, Schema, type Scope } from "effect"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http"; +import * as Net from "node:net"; +import WebSocket, { WebSocketServer } from "ws"; +import { appEntries } from "../src/implementation/tools.ts"; +import { newWebSocketRpcSession, RpcTarget } from "capnweb"; +import { + AppEvaluationFailed, + CallFailure, + ToolCallFailed, + callFailurePresentation, + evaluationFailure, + mayHaveWritten, +} from "../src/contracts/tools.ts"; +import { RuntimeProtocolFailed } from "../src/contracts/runtime.ts"; +import { ScheduleErrors } from "../src/contracts/schedules.ts"; +import { runtimeHops, type SocketRecord } from "../src/implementation/runtime-hop.ts"; +import { + failedAtConnect, + repeatAtConnect, + repeatUnopened, + type Opening, +} from "../src/implementation/runtime-connect.ts"; + +const cases = [ + "rpc-rescued", + "rpc-capnweb-rescued", + "rpc-unreachable", + "rpc-never-opens", + "rpc-closed-after-open", + "rpc-sent-then-closed", + "http-rescued", + "http-unreachable", + "http-reset-after-write", + "http-closed-after-write", + "call-first-exchange", + "call-second-exchange", + "call-second-exchange-read", + "connect-decision-shapes", + "wire-previous-release", + "rpc-check-before-each-connect", + "rpc-never-opens-own-reason", +] as const; +type Case = (typeof cases)[number]; + +/** Milliseconds of delay forced at a case's ordering point, from `--delay`. */ +const delay = Number(process.argv[process.argv.indexOf("--delay") + 1] ?? 0) || 0; +const forced = Effect.sleep(`${delay} millis`); + +/** The `executor.runtime.*` attributes the current span carries. */ +const recorded = Effect.map(Effect.currentSpan, (span) => + Object.fromEntries([...span.attributes].filter(([key]) => key.startsWith("executor.runtime."))), +); + +/** A TCP server on 127.0.0.1 handling its `n`th connection (from 1) with `onConnection`. */ +const tcpServer = (onConnection: (socket: Net.Socket, n: number) => void) => + Effect.acquireRelease( + Effect.callback<{ server: Net.Server; connections: () => number }>((resume) => { + let n = 0; + const server = Net.createServer((socket) => onConnection(socket, ++n)); + server.listen(0, "127.0.0.1", () => resume(Effect.succeed({ server, connections: () => n }))); + }), + ({ server }) => Effect.sync(() => server.close()), + ).pipe( + Effect.map(({ server, connections }) => ({ + port: (server.address() as Net.AddressInfo).port, + connections, + })), + ); + +/** + * A loopback port that nothing listens on, and nothing can listen on, for the rest of the case: the + * local end of an open connection holds it. A socket bound without listening answers a connect with + * a reset, so the connect is refused (`ECONNREFUSED`) on Linux, macOS and Windows alike, and no other + * process can bind the port while the connection stays open. + */ +const closedPort = Effect.gen(function* () { + const { port: listener } = yield* tcpServer((socket) => socket.on("error", () => {})); + const holder = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const socket = Net.connect({ host: "127.0.0.1", port: listener }, () => + resume(Effect.succeed(socket)), + ); + socket.on("error", () => {}); + }), + (socket) => Effect.sync(() => socket.destroy()), + ); + return holder.localPort!; +}); + +/** Run `then` once the request's headers have arrived on `socket`. */ +const untilHeaders = (socket: Net.Socket, then: (head: Buffer) => void) => { + let buffered = Buffer.alloc(0); + const onData = (chunk: Buffer) => { + buffered = Buffer.concat([buffered, chunk]); + if (buffered.indexOf("\r\n\r\n") === -1) return; + socket.off("data", onData); + then(buffered); + }; + socket.on("data", onData); +}; + +/** + * One RPC exchange composed as `workerd-apps.ts` composes it: the socket is tracked and watched as + * it is made, and `work` runs only once it has opened. `work` here sends `messages` and then waits + * for a reply to each, or for the socket to close. + */ +const exchange = ( + port: number, + messages: number, + options: { + /** Run before each connect, as a host's own pre-connect check would be. */ + readonly beforeEachConnect?: Effect.Effect; + /** Replace the session's failure, as a session that chose its own reason would. */ + readonly ownReason?: RuntimeProtocolFailed; + } = {}, +) => { + const hops = runtimeHops(); + const session = (opening: Opening) => + Effect.gen(function* () { + let record: SocketRecord | undefined; + const socket = yield* Effect.acquireRelease( + Effect.sync(() => { + const socket = new WebSocket(`ws://127.0.0.1:${port}/rpc`, { perMessageDeflate: false }); + record = hops.track(socket); + opening.watch(socket); + socket.on("error", () => {}); + return socket; + }), + (socket) => Effect.sync(() => socket.terminate()), + ); + const work = Effect.callback((resume) => { + let replies = 0; + socket.on("message", () => { + if (++replies === messages) resume(Effect.succeed("answered")); + }); + socket.on("close", () => resume(Effect.fail(new Error("closed")))); + for (let i = 0; i < messages; i++) socket.send("call"); + if (messages === 0) return; + }); + return yield* hops.rpc( + () => record, + Effect.andThen(opening.opened, () => work), + ); + }).pipe( + Effect.scoped, + Effect.mapError((error) => options.ownReason ?? error), + ); + return repeatUnopened( + session, + options.beforeEachConnect === undefined ? {} : { beforeEachConnect: options.beforeEachConnect }, + ); +}; + +/** A WebSocket server for the RPC cases, which upgrades the connections `upgrade` accepts. */ +const wsStub = ( + upgrade: (n: number) => "upgrade" | "close-before-upgrade", + serve: (socket: WebSocket) => void, +) => + Effect.gen(function* () { + const wss: WebSocketServer = new WebSocketServer({ noServer: true }); + wss.on("connection", serve); + return yield* tcpServer((socket, n) => { + if (upgrade(n) === "close-before-upgrade") + return untilHeaders(socket, () => socket.destroy()); + // Hand the raw connection to `ws` as an HTTP server would, with the bytes it has read. + untilHeaders(socket, (head) => { + const text = head.toString("latin1"); + const [requestLine, ...lines] = text.slice(0, text.indexOf("\r\n\r\n")).split("\r\n"); + const headers = Object.fromEntries( + lines.map((line) => [ + line.slice(0, line.indexOf(":")).toLowerCase(), + line.slice(line.indexOf(":") + 1).trim(), + ]), + ); + const request = Object.assign(Object.create(null), { + method: requestLine!.split(" ")[0], + url: requestLine!.split(" ")[1], + headers, + }); + wss.handleUpgrade(request as never, socket, Buffer.alloc(0), (ws) => + wss.emit("connection", ws, request), + ); + }); + }); + }); + +/** An RPC case: how the exchange ended, the stub's connection count and what the span recorded. */ +const rpcCase = ( + stub: Effect.Effect<{ port: number; connections: () => number }, never, Scope.Scope>, + messages: number, +) => + Effect.gen(function* () { + const { port, connections } = yield* stub; + const exit = yield* Effect.exit(exchange(port, messages)); + return { + outcome: Exit.isSuccess(exit) ? "answered" : "failed", + failure: Exit.isFailure(exit) ? failureOf(exit.cause) : undefined, + connections: connections(), + attributes: yield* recorded, + }; + }).pipe(Effect.withSpan("probe")); + +const failureOf = (cause: Cause.Cause) => { + const reason = cause.reasons[0]; + if (reason === undefined || !Cause.isFailReason(reason)) return "other"; + const error = reason.error as { + _tag?: string; + reason?: string; + notSent?: boolean; + message?: string; + }; + return { + tag: error._tag ?? (error instanceof Error ? error.message : "other"), + ...(error.reason === undefined || typeof error.reason !== "string" + ? {} + : { reason: error.reason }), + ...(error.notSent === true ? { notSent: true } : {}), + }; +}; + +/** One HTTP exchange through `hops.http`, repeated as `workerd-apps.ts` repeats it. */ +const httpCase = ( + port: () => number, + connections: () => number, + body: string, + beforeConnect: (attempt: number) => Effect.Effect = () => Effect.void, +) => + Effect.gen(function* () { + const hops = runtimeHops(); + const http = yield* HttpClient.HttpClient; + let attempt = 0; + // The exchange's own first statement runs again on the repeat, as a check before connecting + // would in the product. + const once = Effect.gen(function* () { + yield* beforeConnect(++attempt); + const request = HttpClientRequest.post(`http://127.0.0.1:${port()}/workflow`, { + headers: { connection: "close" }, + }).pipe(HttpClientRequest.bodyText(body, "application/json")); + return yield* hops.http("/workflow", http.execute(request), (response) => response.text); + }); + const exit = yield* Effect.exit(repeatAtConnect(once)); + return { + outcome: Exit.isSuccess(exit) ? "answered" : "failed", + attempts: attempt, + connections: connections(), + attributes: yield* recorded, + }; + }).pipe(Effect.withSpan("probe"), Effect.provide(FetchHttpClient.layer)); + +/** + * A call's surfaced failure after `entries` hand-offs to the app's code, the last never sent, judged + * by the tracker both of the host's call paths use (`appEntries`), which `AppCodeEntered` reaches. + */ +const callCase = (entries: number, read: boolean) => { + let enteredApp = false; + const tracker = appEntries(() => { + enteredApp = true; + }); + for (let i = 0; i < entries; i++) tracker.entered(); + const failure = new ToolCallFailed({ + app: "app_00000000-0000-4000-8000-000000000000" as never, + version: "dpl_00000000-0000-4000-8000-000000000000" as never, + tool: "probe" as never, + reason: "Executor could not connect to the app runtime, so it did not send this tool call.", + notSent: true, + }); + const surfaced = tracker.callFailure(read, enteredApp)(failure) as ToolCallFailed; + const presented = callFailurePresentation(surfaced); + return Effect.succeed({ + notSent: surfaced.notSent === true, + mayHaveWritten: surfaced.mayHaveWritten === true, + reason: surfaced.reason, + retryable: presented?.retryable ?? null, + action: presented?.recovery.action ?? null, + attributes: {}, + }); +}; + +/** What `failedAtConnect` decides for each platform's error shape, without its text. */ +const decisions = () => { + const transport = (cause: unknown) => ({ + _tag: "HttpClientError", + reason: { _tag: "TransportError", cause }, + }); + const shaped = (fields: Record) => + Object.assign(new TypeError("synthetic"), fields); + return Effect.succeed({ + decisions: { + // Bun: a connect that failed, and its ETIMEDOUT, which is also its idle time-out. + bunRefused: failedAtConnect( + transport(shaped({ code: "ECONNREFUSED", errno: 0, path: "/x" })), + ), + bunTimedOut: failedAtConnect(transport(shaped({ code: "ETIMEDOUT", errno: 0, path: "/x" }))), + bunReset: failedAtConnect(transport(shaped({ code: "ECONNRESET", errno: 0, path: "/x" }))), + // Node: a connect error carries its syscall. + nodeConnectTimedOut: failedAtConnect( + transport( + Object.assign(new TypeError("fetch failed"), { + cause: shaped({ code: "ETIMEDOUT", syscall: "connect" }), + }), + ), + ), + nodeReadReset: failedAtConnect( + transport( + Object.assign(new TypeError("fetch failed"), { + cause: shaped({ code: "ECONNRESET", syscall: "read" }), + }), + ), + ), + undiciConnectTimeout: failedAtConnect( + transport( + Object.assign(new TypeError("fetch failed"), { + cause: shaped({ code: "UND_ERR_CONNECT_TIMEOUT" }), + }), + ), + ), + notTransport: failedAtConnect({ + _tag: "HttpClientError", + reason: { _tag: "StatusCodeError" }, + }), + }, + attributes: {}, + }); +}; + +/** + * The previous release's view of the wire (main at 9cbea0adc): its closed list of runtime failures, + * and what its server sends for an exchange that never connected. A reader from that release + * decodes `RuntimeProtocolFailed.reason` and `EvaluationRuntime.failure` against that list, so a + * failure this release sends must carry one of its values; the new fact rides in optional fields, + * which that reader drops. And this release reads that server's answers. + */ +const previousRuntimeFailures = [ + "cold-start", + "memory", + "cpu", + "timeout", + "overloaded", + "reset", + "disconnected", + "hung", + "internal", + "invalid-reply", + "too-large", + "data", + "build", + "unsupported", + "managed", + "unrecognized", +]; +const previousServerAnswers: ReadonlyArray = [ + [ + "runtime failure", + "RuntimeProtocolFailed", + '{"_tag":"RuntimeProtocolFailed","reason":"disconnected","message":""}', + ], + [ + "tool listing failure", + "AppEvaluationFailed", + '{"_tag":"AppEvaluationFailed","app":"app_00000000-0000-4000-8000-000000000000","version":"dpl_00000000-0000-4000-8000-000000000000","reason":"App evaluation failed","runtime":{"kind":"worker","failure":"disconnected"},"message":"Executor could not load this app\u2019s tool definitions. The app\u2019s Worker stopped without an answer from the app\u2019s code. Its error matches this cause, which Executor has not confirmed: The connection to the app\'s Worker was lost.","recovery":{"action":"Try again once. If it fails the same way, report the failure with its reference.","instructions":"This happened while Executor loaded the app\'s definition, before any tool ran. Do not change the app\'s accounts to work around it. Retry the unchanged request at most once. This does not show whether the app\'s code or Executor\'s runtime caused it. If it fails again, report the failure with the reference shown beside this error, if any, and check what the app\'s recent changes run while it lists its tools. Verify that the app\'s tools load after the repair."}}', + ], + [ + "tool listing failure in a call's failures", + "CallFailure", + '{"_tag":"AppEvaluationFailed","app":"app_00000000-0000-4000-8000-000000000000","version":"dpl_00000000-0000-4000-8000-000000000000","reason":"App evaluation failed","runtime":{"kind":"worker","failure":"disconnected"},"message":"Executor could not load this app\u2019s tool definitions. The app\u2019s Worker stopped without an answer from the app\u2019s code. Its error matches this cause, which Executor has not confirmed: The connection to the app\'s Worker was lost.","recovery":{"action":"Try again once. If it fails the same way, report the failure with its reference.","instructions":"This happened while Executor loaded the app\'s definition, before any tool ran. Do not change the app\'s accounts to work around it. Retry the unchanged request at most once. This does not show whether the app\'s code or Executor\'s runtime caused it. If it fails again, report the failure with the reference shown beside this error, if any, and check what the app\'s recent changes run while it lists its tools. Verify that the app\'s tools load after the repair."}}', + ], + [ + "tool listing failure in a schedule's failures", + "ScheduleErrors", + '{"_tag":"AppEvaluationFailed","app":"app_00000000-0000-4000-8000-000000000000","version":"dpl_00000000-0000-4000-8000-000000000000","reason":"App evaluation failed","runtime":{"kind":"worker","failure":"disconnected"},"message":"Executor could not load this app\u2019s tool definitions. The app\u2019s Worker stopped without an answer from the app\u2019s code. Its error matches this cause, which Executor has not confirmed: The connection to the app\'s Worker was lost.","recovery":{"action":"Try again once. If it fails the same way, report the failure with its reference.","instructions":"This happened while Executor loaded the app\'s definition, before any tool ran. Do not change the app\'s accounts to work around it. Retry the unchanged request at most once. This does not show whether the app\'s code or Executor\'s runtime caused it. If it fails again, report the failure with the reference shown beside this error, if any, and check what the app\'s recent changes run while it lists its tools. Verify that the app\'s tools load after the repair."}}', + ], + [ + "tool call not sent", + "ToolCallFailed", + '{"_tag":"ToolCallFailed","app":"app_00000000-0000-4000-8000-000000000000","version":"dpl_00000000-0000-4000-8000-000000000000","tool":"probe","reason":"Executor did not receive a usable result from this tool call. The runtime\u2019s error matches this cause, which Executor has not confirmed: The connection to the app\'s Worker was lost.","mayHaveWritten":true,"message":"Executor did not receive a usable result from this tool call. The runtime\u2019s error matches this cause, which Executor has not confirmed: The connection to the app\'s Worker was lost. No further failure detail is available.","recovery":{"action":"Do not automatically repeat this call or duplicate its change. Executor could not confirm its outcome.","instructions":"This call may have changed data or may still complete. A safe read can show effects but cannot rule out later completion. Tell the user what is known and get their agreement before repeating the change. For a call that only reads, the advice for this failure is: \u201cTry once more. If it fails again, inspect the available diagnostics or report the failure. No further failure detail is available, and this error does not establish whether the tool completed. For a read, one further attempt may help. If it fails again, inspect the available diagnostics or report the failure.\u201d Because this call may have changed data, do not repeat it automatically; steps in that advice other than repeating the call still apply. Repairing the cause does not establish whether the earlier call completed."}}', + ], + [ + "tool call not sent in a call's failures", + "CallFailure", + '{"_tag":"ToolCallFailed","app":"app_00000000-0000-4000-8000-000000000000","version":"dpl_00000000-0000-4000-8000-000000000000","tool":"probe","reason":"Executor did not receive a usable result from this tool call. The runtime\u2019s error matches this cause, which Executor has not confirmed: The connection to the app\'s Worker was lost.","mayHaveWritten":true,"message":"Executor did not receive a usable result from this tool call. The runtime\u2019s error matches this cause, which Executor has not confirmed: The connection to the app\'s Worker was lost. No further failure detail is available.","recovery":{"action":"Do not automatically repeat this call or duplicate its change. Executor could not confirm its outcome.","instructions":"This call may have changed data or may still complete. A safe read can show effects but cannot rule out later completion. Tell the user what is known and get their agreement before repeating the change. For a call that only reads, the advice for this failure is: \u201cTry once more. If it fails again, inspect the available diagnostics or report the failure. No further failure detail is available, and this error does not establish whether the tool completed. For a read, one further attempt may help. If it fails again, inspect the available diagnostics or report the failure.\u201d Because this call may have changed data, do not repeat it automatically; steps in that advice other than repeating the call still apply. Repairing the cause does not establish whether the earlier call completed."}}', + ], +]; +const wireSchemas = { + RuntimeProtocolFailed, + ToolCallFailed, + AppEvaluationFailed, + CallFailure, + ScheduleErrors: Schema.Union(ScheduleErrors), +} as const; +const wireJson = (name: keyof typeof wireSchemas) => + Schema.fromJsonString(wireSchemas[name] as never); + +/** Every runtime failure value a never-opened exchange sends, and whether the old list has it. */ +const wireCase = () => + Effect.gen(function* () { + const app = "app_00000000-0000-4000-8000-000000000000" as never; + const version = "dpl_00000000-0000-4000-8000-000000000000" as never; + // The failure the repeat itself ends with when nothing listens. + const port = yield* closedPort; + const exit = yield* Effect.exit(exchange(port, 1)); + const reason = Exit.isFailure(exit) ? exit.cause.reasons[0] : undefined; + const runtime = + reason !== undefined && + Cause.isFailReason(reason) && + reason.error instanceof RuntimeProtocolFailed + ? reason.error + : new RuntimeProtocolFailed({}); + const listing = evaluationFailure({ app, version }, runtime); + const sent: Array = [ + ["runtime failure", "RuntimeProtocolFailed", runtime], + ["tool listing failure", "AppEvaluationFailed", listing], + ["tool listing failure in a schedule's failures", "ScheduleErrors", listing], + ]; + const values = sent.map(([label, schema, value]) => { + const encoded = JSON.parse(Schema.encodeSync(wireJson(schema))(value as never)) as { + reason?: string; + notSent?: boolean; + runtime?: { failure?: string; notSent?: boolean }; + }; + const failure = + schema === "RuntimeProtocolFailed" ? encoded.reason : encoded.runtime?.failure; + return { + label, + failure, + known: failure !== undefined && previousRuntimeFailures.includes(failure), + notSent: encoded.notSent === true || encoded.runtime?.notSent === true, + }; + }); + const read = previousServerAnswers.map(([label, schema, encoded]) => { + try { + Schema.decodeUnknownSync(wireJson(schema))(encoded); + return { label, decoded: true }; + } catch { + return { label, decoded: false }; + } + }); + // The not-sent and partly-run tool call failures: optional fields only, and never both facts. + const call = mayHaveWritten( + ToolCallFailed, + new ToolCallFailed({ app, version, tool: "probe" as never, reason: "x", notSent: true }), + ); + return { + wire: { + values, + read, + partlyRunDropsNotSent: call.notSent === undefined && call.mayHaveWritten === true, + }, + attributes: {}, + }; + }); + +/** The host's side of a Cap'n Web session, which counts the calls it runs. */ +class CountingApi extends RpcTarget { + invocations = 0; + invoke() { + this.invocations += 1; + return "ran"; + } +} + +/** + * A real Cap'n Web session over `ws`, composed as `workerd-apps.ts` composes it: the stub closes the + * first connection before its upgrade and serves the API on the second. The call must run once. + */ +const capnwebRescue = Effect.gen(function* () { + const api = new CountingApi(); + const { port, connections } = yield* wsStub( + (n) => (n === 1 ? "close-before-upgrade" : "upgrade"), + (socket) => { + newWebSocketRpcSession(socket as unknown as globalThis.WebSocket, api); + }, + ); + const hops = runtimeHops(); + const exit = yield* Effect.exit( + repeatUnopened((opening: Opening) => + Effect.scoped( + Effect.gen(function* () { + let record: SocketRecord | undefined; + const peer = yield* Effect.acquireRelease( + Effect.sync(() => { + const socket = new WebSocket(`ws://127.0.0.1:${port}/rpc`, { + perMessageDeflate: false, + }); + record = hops.track(socket); + opening.watch(socket); + socket.on("error", () => {}); + return newWebSocketRpcSession(socket as unknown as globalThis.WebSocket); + }), + (peer) => Effect.sync(() => peer[Symbol.dispose]()), + ); + return yield* hops.rpc( + () => record, + Effect.andThen(opening.opened, () => + Effect.tryPromise(async () => String(await peer.invoke())), + ), + ); + }), + ), + ), + ); + return { + outcome: Exit.isSuccess(exit) ? "answered" : "failed", + value: Exit.isSuccess(exit) ? exit.value : undefined, + invocations: api.invocations, + connections: connections(), + attributes: yield* recorded, + }; +}).pipe(Effect.withSpan("probe")); + +const run = (name: Case): Effect.Effect => { + switch (name) { + case "rpc-rescued": + // The stub closes the first connection before answering its upgrade and upgrades the + // second: which connection is which is the stub's count, not a time. + return rpcCase( + wsStub( + (n) => (n === 1 ? "close-before-upgrade" : "upgrade"), + (socket) => + socket.on("message", () => + Effect.runFork( + Effect.andThen( + forced, + Effect.sync(() => socket.send("reply")), + ), + ), + ), + ), + 1, + ); + case "rpc-unreachable": + return rpcCase(closedPort.pipe(Effect.map((port) => ({ port, connections: () => 0 }))), 1); + case "rpc-never-opens": + // The stub closes every connection before answering its upgrade: two connections, no more. + return rpcCase( + wsStub( + () => "close-before-upgrade", + () => {}, + ), + 1, + ); + case "rpc-closed-after-open": + // Opens, then the stub closes it with a code before any message: never repeated. + return rpcCase( + wsStub( + () => "upgrade", + (socket) => + Effect.runFork( + Effect.andThen( + forced, + Effect.sync(() => socket.close(4001)), + ), + ), + ), + 0, + ); + case "rpc-sent-then-closed": + // Opens, the client sends, the stub closes on receiving it: never repeated. + return rpcCase( + wsStub( + () => "upgrade", + (socket) => + socket.once("message", () => + Effect.runFork( + Effect.andThen( + forced, + Effect.sync(() => socket.close(1011)), + ), + ), + ), + ), + 1, + ); + case "http-rescued": + // Nothing listens for the first attempt; the exchange's own start opens the listener before + // the second, so the repeat is what reaches it. + return Effect.gen(function* () { + const closed = yield* closedPort; + let open: { port: number; connections: () => number } | undefined; + return yield* httpCase( + () => (open === undefined ? closed : open.port), + () => open?.connections() ?? 0, + "{}", + (attempt) => + attempt === 2 + ? Effect.andThen( + forced, + tcpServer((socket) => + untilHeaders(socket, () => + socket.end( + "HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\nok", + ), + ), + ).pipe( + Effect.tap((server) => Effect.sync(() => (open = server))), + Effect.asVoid, + ), + ) + : Effect.void, + ); + }); + case "http-unreachable": + return closedPort.pipe( + Effect.flatMap((port) => + httpCase( + () => port, + () => 0, + "{}", + ), + ), + ); + case "http-reset-after-write": + // The stub resets once the request's headers arrived: bytes were written. Never repeated. + return tcpServer((socket) => + untilHeaders(socket, () => + Effect.runFork( + Effect.andThen( + forced, + Effect.sync(() => socket.resetAndDestroy()), + ), + ), + ), + ).pipe( + Effect.flatMap(({ port, connections }) => + httpCase(() => port, connections, "x".repeat(16 * 1024 * 1024)), + ), + ); + case "http-closed-after-write": + // The stub closes without answering once the request arrived. Never repeated. + return tcpServer((socket) => + untilHeaders(socket, () => + Effect.runFork( + Effect.andThen( + forced, + Effect.sync(() => socket.end()), + ), + ), + ), + ).pipe(Effect.flatMap(({ port, connections }) => httpCase(() => port, connections, "{}"))); + case "rpc-capnweb-rescued": + return capnwebRescue; + case "connect-decision-shapes": + return decisions(); + case "wire-previous-release": + return wireCase(); + case "rpc-check-before-each-connect": + // The check runs before each connect: the stub has seen 0 connections, then 1. + return Effect.gen(function* () { + const stub = yield* wsStub( + () => "close-before-upgrade", + () => {}, + ); + const seen: number[] = []; + const exit = yield* Effect.exit( + exchange(stub.port, 1, { + beforeEachConnect: Effect.sync(() => { + seen.push(stub.connections()); + }), + }), + ); + return { + outcome: Exit.isSuccess(exit) ? "answered" : "failed", + checks: seen, + connections: stub.connections(), + attributes: {}, + }; + }); + case "rpc-never-opens-own-reason": + // The session ends a never-opened attempt with a reason of its own: it is kept, with + // `notSent` added. + return Effect.gen(function* () { + const stub = yield* wsStub( + () => "close-before-upgrade", + () => {}, + ); + const exit = yield* Effect.exit( + exchange(stub.port, 1, { + ownReason: new RuntimeProtocolFailed({ reason: "overloaded" }), + }), + ); + return { + outcome: Exit.isSuccess(exit) ? "answered" : "failed", + failure: Exit.isFailure(exit) ? failureOf(exit.cause) : undefined, + connections: stub.connections(), + attributes: {}, + }; + }); + case "call-first-exchange": + return callCase(1, false); + case "call-second-exchange": + return callCase(2, false); + case "call-second-exchange-read": + return callCase(2, true); + } +}; + +const name = process.argv[process.argv.indexOf("--case") + 1] as Case; +if (!cases.includes(name)) throw new Error(`--case must be one of ${cases.join(", ")}`); +Effect.runPromise(Effect.scoped(run(name)).pipe(Effect.timeout("30 seconds"))).then( + (result) => { + console.log(JSON.stringify(result)); + process.exit(0); + }, + (error) => { + console.error(String(error)); + process.exit(1); + }, +); diff --git a/packages/sdk/scripts/runtime-hop-fixture.ts b/packages/sdk/scripts/runtime-hop-fixture.ts new file mode 100644 index 0000000000..f39c2433ca --- /dev/null +++ b/packages/sdk/scripts/runtime-hop-fixture.ts @@ -0,0 +1,413 @@ +/** + * Local-only probe of what a failed exchange with the app runtime records (`runtime-hop.ts`). Each + * case runs exchanges against a stub on 127.0.0.1, or a failure built to break the recording, and + * prints what the span recorded as JSON for `runtime-hop-shape.spec.ts` to judge. The HTTP cases + * use the same `FetchHttpClient` the runtime's hops use, here on the fixture's own runtime (Node in + * the scenario). The RPC cases use a `ws` socket, as the runtime's sessions do. + */ +import { Cause, Deferred, Effect, Exit, Fiber } from "effect"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http"; +import * as Net from "node:net"; +import WebSocket, { WebSocketServer } from "ws"; +import { classify } from "../src/implementation/app-runner.ts"; +import { + runtimeHops, + type RuntimeView, + type SocketRecord, +} from "../src/implementation/runtime-hop.ts"; + +const cases = [ + "http-unanswered", + "http-answer-reset", + "http-reset-half-written", + "http-unread-body", + "http-idle", + "http-restarted", + "guard-throwing-getter", + "guard-proxy", + "guard-throwing-context", + "guard-throwing-socket", + "context-checked", + "code-long", + "ws-refused", + "ws-closed-before-upgrade", + "ws-close-before-message", + "ws-close-after-message", + "rpc-app-failure", + "rpc-success", + "rpc-interrupt", +] as const; +type Case = (typeof cases)[number]; + +/** The `executor.runtime.*` attributes the current span carries. */ +const recorded = Effect.map(Effect.currentSpan, (span) => + Object.fromEntries([...span.attributes].filter(([key]) => key.startsWith("executor.runtime."))), +); + +/** A TCP server on 127.0.0.1 that handles each connection with `onConnection`. */ +const tcpServer = (onConnection: (socket: Net.Socket) => void) => + Effect.acquireRelease( + Effect.callback((resume) => { + const server = Net.createServer(onConnection); + server.listen(0, "127.0.0.1", () => resume(Effect.succeed(server))); + }), + (server) => Effect.sync(() => server.close()), + ).pipe(Effect.map((server) => (server.address() as Net.AddressInfo).port)); + +/** Run `then` once the request's headers have arrived on `socket`. */ +const untilHeaders = (socket: Net.Socket, then: () => void) => { + let buffered = Buffer.alloc(0); + const onData = (chunk: Buffer) => { + buffered = Buffer.concat([buffered, chunk]); + if (buffered.indexOf("\r\n\r\n") === -1) return; + socket.off("data", onData); + then(); + }; + socket.on("data", onData); +}; + +const answering = tcpServer((socket) => + untilHeaders(socket, () => + socket.end("HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\nok"), + ), +); + +/** One exchange through `hops.http` as the runtime's `/workflow` hop makes it. */ +const exchange = ( + hops: ReturnType, + port: number, + body: string, + onHeaders: Effect.Effect = Effect.void, +) => + Effect.gen(function* () { + const http = yield* HttpClient.HttpClient; + const request = HttpClientRequest.post(`http://127.0.0.1:${port}/workflow`, { + headers: { connection: "close" }, + }).pipe(HttpClientRequest.bodyText(body, "application/json")); + return yield* Effect.exit( + hops.http("/workflow", http.execute(request), (response) => + response.status !== 200 + ? Effect.fail(new Error("status")) + : Effect.andThen(onHeaders, response.text), + ), + ); + }); + +/** A span around `program`, reporting its outcome and what the span recorded. */ +const probe = (program: Effect.Effect, E, R>) => + Effect.gen(function* () { + const exit = yield* program; + return { outcome: Exit.isSuccess(exit) ? "answered" : "failed", attributes: yield* recorded }; + }).pipe(Effect.withSpan("probe"), Effect.provide(FetchHttpClient.layer)); + +/** + * A failure built to break the recording, sent through the HTTP hop and through the RPC hop: the + * same object must come out, classified as it would be without the recording. + */ +const guardCase = (error: unknown, view?: RuntimeView, socket?: SocketRecord) => + Effect.gen(function* () { + const hops = runtimeHops(view); + const viaHttp = yield* Effect.exit( + hops.http("/workflow", Effect.fail(error), () => Effect.succeed("unreachable")), + ); + const httpAttributes = yield* recorded; + const rpc = yield* Effect.gen(function* () { + const exit = yield* Effect.exit(hops.rpc(() => socket, Effect.fail(error))); + return { exit, attributes: yield* recorded }; + }).pipe(Effect.withSpan("rpc")); + const same = (exit: Exit.Exit) => + Exit.isFailure(exit) && + exit.cause.reasons.length === 1 && + Cause.isFailReason(exit.cause.reasons[0]!) && + exit.cause.reasons[0]!.error === error; + // The classification the failure gets, or that classifying it throws, as without the recording. + const label = (value: unknown) => { + try { + return classify(value); + } catch { + return "throws"; + } + }; + const out = (exit: Exit.Exit) => + Exit.isFailure(exit) && Cause.isFailReason(exit.cause.reasons[0]!) + ? exit.cause.reasons[0]!.error + : undefined; + return { + sameErrorHttp: same(viaHttp), + sameErrorRpc: same(rpc.exit), + classified: label(error) === label(out(viaHttp)) && label(error) === label(out(rpc.exit)), + attributes: httpAttributes, + rpcAttributes: rpc.attributes, + }; + }).pipe(Effect.withSpan("probe")); + +/** An error whose `code` getter throws. */ +const throwingGetter = () => { + const error = new Error("synthetic"); + Object.defineProperty(error, "code", { + get() { + throw new Error("getter"); + }, + }); + return error; +}; +/** A value every property read of which throws. */ +const throwingProxy = () => + new Proxy(new Error("synthetic"), { + get() { + throw new Error("proxy"); + }, + has() { + throw new Error("proxy"); + }, + getPrototypeOf() { + throw new Error("proxy"); + }, + }); + +/** An RPC socket to a WebSocket stub doing `serve`, recorded once it has closed. */ +const wsServer = (serve: (socket: WebSocket) => void) => + Effect.acquireRelease( + Effect.callback((resume) => { + const server: WebSocketServer = new WebSocketServer({ host: "127.0.0.1", port: 0 }, () => + resume(Effect.succeed(server)), + ); + server.on("connection", serve); + }), + (server) => Effect.sync(() => server.close()), + ).pipe(Effect.map((server) => (server.address() as { port: number }).port)); + +/** Run `then` once at least `ms` have passed on this process's clock, however late timers fire. */ +const atLeast = (ms: number, then: () => void) => { + const from = performance.now(); + const check = () => { + const left = ms - (performance.now() - from); + if (left <= 0) then(); + else setTimeout(check, Math.ceil(left)); + }; + setTimeout(check, ms); +}; + +/** + * One RPC session through `hops.rpc`: its work fails once the socket has closed (`transport`), at + * once while the socket is open (`app`), succeeds (`success`), or is interrupted (`interrupt`). + * `onReceived` runs after the client has received a message (the recording has counted it). + */ +const session = ( + port: number, + ending: "transport" | "app" | "success" | "interrupt", + sendOnOpen: boolean, + onReceived?: () => void, +) => + Effect.gen(function* () { + const hops = runtimeHops(); + const socket = new WebSocket(`ws://127.0.0.1:${port}/rpc`, { perMessageDeflate: false }); + const record = hops.track(socket); + const opened = yield* Deferred.make(); + const closed = yield* Deferred.make(); + socket.on("open", () => { + Deferred.doneUnsafe(opened, Exit.void); + if (sendOnOpen) socket.send("hello"); + }); + socket.on("error", () => {}); + socket.on("close", () => Deferred.doneUnsafe(closed, Exit.void)); + if (onReceived !== undefined) socket.on("message", onReceived); + const work = { + transport: Effect.andThen(Deferred.await(closed), Effect.fail(new Error("transport"))), + app: Effect.andThen(Deferred.await(opened), Effect.fail(new Error("app"))), + success: Effect.andThen(Deferred.await(opened), Effect.succeed("ok")), + interrupt: Effect.andThen(Deferred.await(opened), Effect.never), + }[ending]; + const fiber = yield* Effect.forkChild(Effect.exit(hops.rpc(() => record, work))); + if (ending === "interrupt") { + yield* Deferred.await(opened); + yield* Fiber.interrupt(fiber); + } else yield* Fiber.join(fiber); + socket.terminate(); + return { attributes: yield* recorded }; + }).pipe(Effect.withSpan("probe")); + +const run = (name: Case) => { + switch (name) { + case "http-unanswered": + // Closes 120 ms after the request arrived: the time to failure is at least that. + return tcpServer((socket) => + untilHeaders(socket, () => setTimeout(() => socket.destroy(), 120)), + ).pipe(Effect.flatMap((port) => probe(exchange(runtimeHops(), port, "{}")))); + case "http-answer-reset": + return Effect.gen(function* () { + // The server resets only once the client has read the reply's headers. + const headersRead = yield* Deferred.make(); + const port = yield* tcpServer((socket) => + untilHeaders(socket, () => { + socket.write( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 64\r\n\r\n", + ); + Effect.runFork( + Effect.andThen( + Deferred.await(headersRead), + Effect.sync(() => socket.resetAndDestroy()), + ), + ); + }), + ); + return yield* probe( + exchange( + runtimeHops(), + port, + "{}", + Deferred.done(headersRead, Exit.void).pipe(Effect.asVoid), + ), + ); + }); + case "http-reset-half-written": + return tcpServer((socket) => untilHeaders(socket, () => socket.resetAndDestroy())).pipe( + Effect.flatMap((port) => + probe(exchange(runtimeHops(), port, "x".repeat(16 * 1024 * 1024))), + ), + ); + case "http-unread-body": + return tcpServer((socket) => + untilHeaders(socket, () => { + socket.pause(); + socket.end("HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\nok"); + }), + ).pipe( + Effect.flatMap((port) => probe(exchange(runtimeHops(), port, "x".repeat(4 * 1024 * 1024)))), + ); + case "http-idle": + // A successful exchange, 150 ms idle, then one that fails: the failure says how long it was. + return Effect.gen(function* () { + const hops = runtimeHops(); + const good = yield* answering; + const bad = yield* tcpServer((socket) => untilHeaders(socket, () => socket.destroy())); + const first = yield* exchange(hops, good, "{}").pipe(Effect.provide(FetchHttpClient.layer)); + yield* Effect.sleep("150 millis"); + return { + first: Exit.isSuccess(first) ? "answered" : "failed", + ...(yield* probe(exchange(hops, bad, "{}"))), + }; + }); + case "http-restarted": + // A runtime replaced twice, the last time 80 ms before the failure. + return Effect.gen(function* () { + const lastAt = performance.now(); + yield* Effect.sleep("80 millis"); + const hops = runtimeHops({ restarts: () => ({ count: 2, lastAt }), probe: () => ({}) }); + const bad = yield* tcpServer((socket) => untilHeaders(socket, () => socket.destroy())); + return yield* probe(exchange(hops, bad, "{}")); + }); + case "guard-throwing-getter": + return guardCase(throwingGetter()); + case "guard-proxy": + return guardCase(throwingProxy()); + case "guard-throwing-context": + return guardCase(new Error("synthetic"), { + restarts: () => ({ count: 0, lastAt: undefined }), + probe: () => { + throw new Error("context"); + }, + }); + case "guard-throwing-socket": + return guardCase(new Error("synthetic"), undefined, { + ended: () => true, + attributes: () => { + throw new Error("socket"); + }, + }); + case "context-checked": + // A probe context with keys and values the recording must refuse, and an odd error tag. + return guardCase( + Object.assign(new Error("synthetic"), { + _tag: "has spaces in it", + code: "sk_live_0123456789abcdef0123456789abcd", + syscall: "aws_secret_key_x", + }), + { + restarts: () => ({ count: 0, lastAt: undefined }), + probe: () => ({ + "executor.runtime.probe.sequence": 7, + "executor.runtime.probe.outcome": "timeout", + "executor.runtime.probe.in_flight": true, + "executor.runtime.http.route": "/elsewhere", + // Keys under the recording's own prefix that the hop never writes itself. + "executor.runtime.forged": "forged", + "executor.runtime.http.forged": 1, + "other.prefix": 1, + "executor.runtime.probe.long": "x".repeat(200), + "executor.runtime.probe.secret": "sk_live_0123456789abcdef0123456789abcd", + "executor.runtime.probe.object": { nested: true }, + "executor.runtime.probe.huge": 1e12, + }), + }, + ); + case "code-long": + // Node's longer codes, 26 and 28 characters, are kept. A code of Bun's word form longer than + // the 40 characters a value may have is recorded as `other`, not dropped. + return Effect.gen(function* () { + const stream = yield* guardCase( + Object.assign(new Error("synthetic"), { code: "ERR_STREAM_PREMATURE_CLOSE" }), + ); + const status = yield* guardCase( + Object.assign(new Error("synthetic"), { code: "UND_ERR_RESPONSE_STATUS_CODE" }), + ); + const long = yield* guardCase( + Object.assign(new Error("synthetic"), { + code: "ConnectionRefusedHostUnreachableTemporarily", + }), + ); + return { + attributes: stream.attributes, + rpcAttributes: status.attributes, + longAttributes: long.attributes, + }; + }); + case "ws-refused": + // Nothing listens on the port: the socket never opens. + // The listener is closed (its socket released at once) before the client connects. + return Effect.scoped(tcpServer(() => {})).pipe( + Effect.flatMap((port) => session(port, "transport", false)), + ); + case "ws-closed-before-upgrade": + // The server accepts the connection and closes it before answering the upgrade. + return tcpServer((socket) => untilHeaders(socket, () => socket.destroy())).pipe( + Effect.flatMap((port) => session(port, "transport", false)), + ); + case "ws-close-before-message": + return wsServer((socket) => socket.close(4001)).pipe( + Effect.flatMap((port) => session(port, "transport", false)), + ); + case "ws-close-after-message": + // Answers the client's message. The client, at least 30 ms on its own clock after it has + // received the reply, asks the stub to close: the order is a fact, not a delay. + return Effect.gen(function* () { + let served: WebSocket | undefined; + const port = yield* wsServer((socket) => { + served = socket; + socket.once("message", () => socket.send("reply")); + }); + return yield* session(port, "transport", true, () => + atLeast(30, () => served?.close(1011)), + ); + }); + case "rpc-app-failure": + return wsServer(() => {}).pipe(Effect.flatMap((port) => session(port, "app", false))); + case "rpc-success": + return wsServer(() => {}).pipe(Effect.flatMap((port) => session(port, "success", false))); + case "rpc-interrupt": + return wsServer(() => {}).pipe(Effect.flatMap((port) => session(port, "interrupt", false))); + } +}; + +const name = process.argv[process.argv.indexOf("--case") + 1] as Case; +if (!cases.includes(name)) throw new Error(`--case must be one of ${cases.join(", ")}`); +Effect.runPromise(Effect.scoped(run(name)).pipe(Effect.timeout("30 seconds"))).then( + (result) => { + console.log(JSON.stringify(result)); + process.exit(0); + }, + (error) => { + console.error(String(error)); + process.exit(1); + }, +); diff --git a/packages/sdk/scripts/webhook-sources-fixture.ts b/packages/sdk/scripts/webhook-sources-fixture.ts new file mode 100644 index 0000000000..4dd55bea10 --- /dev/null +++ b/packages/sdk/scripts/webhook-sources-fixture.ts @@ -0,0 +1,542 @@ +/** + * Local-only probe of the order in which webhook-source reconciliation serves saved intent, and of + * the lease that keeps two sweeps off one source. It runs the SDK's real `makeWebhookSources` on + * PGlite in memory. Apps have no deployment, so a served source becomes `needs-setup` without any + * external call; the stubbed resources only list subscriptions (none) and can hold one source's + * listing, which keeps that source's lease held by a known sweep. Prints, as JSON, what each case + * named (`--case NAME`, repeatable, or `--all`) found, for `webhook-source-order.spec.ts` to judge. + * Each case runs on a database of its own, a copy of one migrated once (`rowsAtOpen` says each + * opened empty), so no other case's or + * scenario's sources are ever in its queue; the cases run one after another in one process. The + * whole run is a few seconds of CPU, so a busy host stays far from the scenario's time limit. + * Nothing here depends on timing: each case counts sweeps and reads stored state. + * + * - `relink-while-new-arrive`: an existing source is linked again while every sweep also finds new + * sources. The round in which it is served, or none within the bound. + * - `configure-while-new-arrive`: the same, saving its intent by configuring it. + * - `relink-during-lease`: a source is linked again while a sweep holds its lease. Whether a second + * sweep claims it meanwhile, and whether it is served again once the first sweep finishes. + * - `outside-failures-then-new`: eight sources fail outside their attempt at every run, one new + * source joins after them. The sweep in which it is served, and how often the failing ones ran. + * - `relinks-before-each-sweep-then-new`: one new source; before every sweep four older sources are + * linked again. The sweep in which the new one is served. + * - `two-sweeps-race`: two sweeps both read the pending queue before either claims. How many claims. + * - `setup-completed-while-new-arrive`: a failed source holding a retry time an hour ahead has its + * setup completed while every sweep also finds new sources. The round in which it is served. + * - `waiting-push-after-setup`: a source's setup is completed while the waiting check of that source + * runs, between its read and its conditional write. Whether that write pushed the source ahead. + * - `intent-plan`: 2,000 pending and 2,000 served sources; how the pending query is planned, in + * `leaseUntil` order and, for comparison, in `createdAt` order. + * + * The clock moves one millisecond at every read, so when each source joined the queue is a fact + * of the order of the case's writes. + */ +import { cpSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Clock, Deferred, Duration, Effect, Fiber, Layer } from "effect"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import { pgliteLayer } from "fumadb-effect/pglite"; +import { SqlClient } from "effect/sql"; +import { makeExecutorStorage } from "../src/core.ts"; +import { database } from "../src/implementation/database.ts"; +import { hostCrypto } from "../src/implementation/host-crypto.ts"; +import { makeWebhookSources, setupCompleted } from "../src/implementation/webhook-sources.ts"; +import { appSlug } from "../src/contracts/app-slug.ts"; + +const cases = [ + "relink-while-new-arrive", + "configure-while-new-arrive", + "relink-during-lease", + "outside-failures-then-new", + "relinks-before-each-sweep-then-new", + "two-sweeps-race", + "intent-plan", + "setup-completed-while-new-arrive", + "waiting-push-after-setup", +] as const; +type Case = (typeof cases)[number]; +/** The cases named by `--case` (repeatable), or every case with `--all`, in that order. */ +const selected: readonly Case[] = process.argv.includes("--all") + ? cases + : process.argv.flatMap((arg, i) => (process.argv[i - 1] === "--case" ? [arg as Case] : [])); +for (const name of selected) + if (!cases.includes(name)) throw new Error(`--case must be one of ${cases.join(", ")}`); +if (selected.length === 0) throw new Error("Name a case with --case, or pass --all."); + +/** Sweeps the starvation cases allow before they report that the source was not served. */ +const rounds = 20; +/** Sources one sweep serves, as the host's tick asks for. */ +const limit = 4; + +const owner = "org_fixture"; + +/** + * A clock that moves one millisecond at every read, so that two writes always store different + * times, in the order they were made: when a source joined the queue is a fact of the case, not of + * how fast it ran. + */ +let ticks = Date.UTC(2026, 10, 16); +const tickingClock: Clock.Clock = { + currentTimeMillisUnsafe: () => ++ticks, + currentTimeMillis: Effect.sync(() => ++ticks), + currentTimeNanosUnsafe: () => BigInt(++ticks) * 1_000_000n, + currentTimeNanos: Effect.sync(() => BigInt(++ticks) * 1_000_000n), + monotonicTimeNanosUnsafe: () => BigInt(++ticks) * 1_000_000n, + monotonicTimeNanos: Effect.sync(() => BigInt(++ticks) * 1_000_000n), + sleep: (duration) => + Effect.promise( + () => new Promise((resolve) => setTimeout(resolve, Duration.toMillis(duration))), + ), +}; +let serial = 0; +const id = (prefix: string) => + `${prefix}_00000000-0000-4000-8000-${String(++serial).padStart(12, "0")}`; + +/** One case, on a database of its own: no other case's sources are ever in its queue. */ +const program = (name: Case) => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* makeExecutorStorage({ provider: "postgresql" }).pipe( + Effect.provideService(SqlClient.SqlClient, sql), + ); + yield* storage.migrate; + const db = database(storage); + const crypto = yield* hostCrypto; + // The provider every fixture account belongs to. + yield* db.create("providers", { + id: "prv_fixture", + definition: {}, + } as never) as Effect.Effect; + + /** Apps whose listing dies. */ + const dying = new Set(); + /** How many times each app's subscriptions were listed: once per claimed reconciliation. */ + const listed = new Map(); + /** An app whose listing waits for `release`, keeping its source's lease held. */ + let holding: + | { + readonly app: string; + readonly reached: Deferred.Deferred; + readonly release: Deferred.Deferred; + } + | undefined; + /** Called inside the waiting check, between its read and its conditional write. */ + let duringWaitingCheck: Effect.Effect | undefined; + let waitingChecks = 0; + const resources = { + accountNeedingReconnect: () => + Effect.gen(function* () { + waitingChecks += 1; + const during = duringWaitingCheck; + duringWaitingCheck = undefined; + if (during !== undefined) yield* during; + // An account must reconnect: the source keeps waiting. + return "acc_00000000-0000-4000-8000-ffffffffffff"; + }), + webhooks: { + list: ({ app }: { readonly app: string }) => + Effect.gen(function* () { + listed.set(app, (listed.get(app) ?? 0) + 1); + // A defect, outside the reconciliation's attempt: its release puts the source back. + if (dying.has(app)) return yield* Effect.die(new Error("listing died")); + if (holding !== undefined && holding.app === app) { + const held = holding; + holding = undefined; + yield* Deferred.succeed(held.reached, undefined); + yield* Deferred.await(held.release); + } + return []; + }), + }, + }; + const sources = makeWebhookSources(db, crypto, undefined, resources as never); + + /** An app without a deployment and an account its owner holds. */ + const appWithAccount = Effect.gen(function* () { + const app = id("app"); + const account = id("acc"); + const createdAt = new Date(); + yield* db.create("apps", { + id: app, + code: id("code"), + repository: null, + owner, + name: `Fixture ${serial}`, + slug: appSlug(`Fixture ${serial}`), + activeDeployment: null, + copiedFrom: null, + createdAt, + } as never) as Effect.Effect; + yield* db.create("accounts", { + id: account, + owner, + provider: "prv_fixture", + method: "key", + label: "fixture", + encryptedCredentials: new Uint8Array(), + createdAt, + } as never) as Effect.Effect; + return { app, account }; + }); + const status = (source: { readonly app: string; readonly account: string }) => + ( + db.findFirst("webhookSources", { + where: (b) => + b.and(b("app", "=", source.app as never), b("account", "=", source.account as never)), + }) as Effect.Effect + ).pipe(Effect.map((row) => (row as { status?: string } | null)?.status)); + const link = (source: { readonly app: string; readonly account: string }) => + sources.link({ app: source.app, account: source.account } as never); + + /** One source, served once, then its intent saved again while new sources keep arriving. */ + const whileNewArrive = ( + saveAgain: (source: { app: string; account: string }) => Effect.Effect, + ) => + Effect.gen(function* () { + const target = yield* appWithAccount; + yield* link(target); + yield* sources.tick(limit); + const before = yield* status(target); + yield* saveAgain(target); + const pendingAfterSave = yield* status(target); + let servedInRound: number | null = null; + for (let round = 1; round <= rounds && servedInRound === null; round++) { + // A full sweep's worth of new sources arrives before each sweep. + for (let i = 0; i < limit; i++) yield* link(yield* appWithAccount); + yield* sources.tick(limit); + if ((yield* status(target)) !== "pending") servedInRound = round; + } + return { before, pendingAfterSave, servedInRound, rounds, limit }; + }); + + switch (name) { + case "relink-while-new-arrive": + return yield* whileNewArrive(link); + case "configure-while-new-arrive": + return yield* whileNewArrive((source) => + Effect.gen(function* () { + const saved = (yield* db.findFirst("webhookSources", { + where: (b) => + b.and( + b("app", "=", source.app as never), + b("account", "=", source.account as never), + ), + }) as Effect.Effect) as { id: string; actor: string | null; revision: number }; + return yield* sources.configureSource({ + app: source.app, + account: source.account, + expected: { id: saved.id, actor: saved.actor, revision: saved.revision }, + config: { probe: true }, + } as never); + }), + ); + case "outside-failures-then-new": { + // Eight sources served once, linked again, and each failing outside its attempt from then + // on; one new source after them. Where it is served, and how often the failing ones run. + const old = []; + for (let i = 0; i < 8; i++) old.push(yield* appWithAccount); + for (const source of old) yield* link(source); + while ((yield* Effect.forEach(old, status)).some((s) => s === "pending")) + yield* sources.tick(limit); + for (const source of old) { + dying.add(source.app); + yield* link(source); + } + const fresh = yield* appWithAccount; + yield* link(fresh); + let servedInSweep: number | null = null; + for (let sweep = 1; sweep <= rounds && servedInSweep === null; sweep++) { + // The host runs each sweep on its own; one whose reconciliation died does not stop it. + yield* Effect.exit(sources.tick(limit)); + if ((yield* status(fresh)) !== "pending") servedInSweep = sweep; + } + const claims = old.map((source) => (listed.get(source.app) ?? 0) - 1); + return { servedInSweep, rounds, limit, maxOldClaims: Math.max(...claims) }; + } + case "relinks-before-each-sweep-then-new": { + // Four sources served once; one new source; before every sweep the four are linked again. + const old = []; + for (let i = 0; i < limit; i++) old.push(yield* appWithAccount); + for (const source of old) yield* link(source); + while ((yield* Effect.forEach(old, status)).some((s) => s === "pending")) + yield* sources.tick(limit); + const fresh = yield* appWithAccount; + yield* link(fresh); + let servedInSweep: number | null = null; + for (let sweep = 1; sweep <= rounds && servedInSweep === null; sweep++) { + for (const source of old) yield* link(source); + yield* sources.tick(limit); + if ((yield* status(fresh)) !== "pending") servedInSweep = sweep; + } + return { servedInSweep, rounds, limit }; + } + case "two-sweeps-race": { + // Two sweeps both read the pending queue before either claims anything: each waits after + // its read until the other has read too. Then both try to claim the same source. + const target = yield* appWithAccount; + yield* link(target); + const arrived = yield* Deferred.make(); + let reads = 0; + const barrier = Effect.gen(function* () { + reads += 1; + if (reads === 2) yield* Deferred.succeed(arrived, undefined); + if (reads <= 2) yield* Deferred.await(arrived); + }); + const racing = new Proxy(db, { + get: (target, property, receiver) => { + const value = Reflect.get(target, property, receiver); + if (property !== "findMany" || typeof value !== "function") return value; + return (table: string, options: { readonly limit?: number }) => { + const read = value.call(target, table, options) as Effect.Effect; + return table === "webhookSources" && options?.limit !== undefined + ? Effect.tap(read, () => barrier) + : read; + }; + }, + }); + const racers = makeWebhookSources(racing as never, crypto, undefined, resources as never); + yield* Effect.all([racers.tick(limit), racers.tick(limit)], { concurrency: "unbounded" }); + return { readsBeforeClaims: reads, claims: listed.get(target.app) ?? 0 }; + } + case "intent-plan": { + // 2,000 served sources and, after them, 2,000 pending ones that joined one millisecond apart: + // how the pending query is planned. One statement writes them, as linking them one by one + // would, so the case's time does not grow with the rows it needs. + const base = ticks; + yield* sql.unsafe( + `insert into executor_webhook_sources + (id, app, owner, account, actor, accounts, config, revision, status, failure, reconciled_deployment, lease, lease_until, created_at, retries) + select 'whs_plan_' || i, 'app_plan_' || i, '${owner}', 'acc_plan_' || i, null, '{}', '{}', 1, + case when i <= 2000 then 'ready' else 'pending' end, null, null, null, + to_timestamp((${base} + i) / 1000.0), to_timestamp((${base} + i) / 1000.0), 0 + from generate_series(1, 4000) as i`, + ); + ticks = base + 4000; + yield* sql.unsafe("analyze executor_webhook_sources"); + const plan = (order: string) => + sql + .unsafe( + `explain (analyze, format json) select * from executor_webhook_sources where status = 'pending' and lease is null order by ${order} asc limit ${limit}`, + ) + .pipe( + Effect.map((rows) => { + const root = (rows as unknown as Array>)[0]?.[ + "QUERY PLAN" + ] as Array<{ + Plan: PlanNode; + }>; + const nodes: PlanNode[] = []; + const walk = (node: PlanNode) => { + nodes.push(node); + for (const child of node.Plans ?? []) walk(child); + }; + walk(root[0]!.Plan); + return { + nodes: nodes.map((node) => node["Node Type"]), + index: + nodes.find((node) => node["Index Name"] !== undefined)?.["Index Name"] ?? null, + rowsRead: Math.max(...nodes.map((node) => node["Actual Rows"] ?? 0)), + sorted: nodes.some((node) => node["Node Type"] === "Sort"), + }; + }), + ); + return { + pending: 2000, + leaseUntil: yield* plan("lease_until"), + createdAt: yield* plan("created_at"), + }; + } + case "setup-completed-while-new-arrive": { + // A source that failed and holds a retry time an hour ahead; a person completes its setup + // while every sweep also finds a full sweep's worth of new sources. + const target = yield* appWithAccount; + yield* link(target); + yield* sources.tick(limit); + yield* db.updateMany("webhookSources", { + where: (b: never) => (b as (...args: unknown[]) => unknown)("app", "=", target.app), + set: { + status: "needs-setup", + failure: "configuration", + retries: 8, + leaseUntil: new Date(ticks + 3_600_000), + }, + } as never) as Effect.Effect; + yield* setupCompleted(db, target as never); + const pendingAfterSetup = yield* status(target); + let servedInRound: number | null = null; + for (let round = 1; round <= rounds && servedInRound === null; round++) { + for (let i = 0; i < limit; i++) yield* link(yield* appWithAccount); + yield* sources.tick(limit); + if ((yield* status(target)) !== "pending") servedInRound = round; + } + return { pendingAfterSetup, servedInRound, rounds, limit }; + } + case "waiting-push-after-setup": { + // A source waiting on its accounts and due for its check, of an app with a deployment that + // declares no accounts (so its accounts resolve and the check runs). While the check runs, + // a person completes the source's setup. The check's own write must not then push it. + const target = yield* appWithAccount; + const version = id("dpl"); + const app = (yield* db.findFirst("apps", { + where: (b: never) => (b as (...args: unknown[]) => unknown)("id", "=", target.app), + } as never) as Effect.Effect) as { code: string }; + yield* db.create("deployments", { + id: version, + code: app.code, + owner, + build: "bld_00000000-0000-4000-8000-000000000001", + requirements: { accounts: {} }, + createdAt: new Date(), + sourceCommit: null, + fileCount: 1, + } as never) as Effect.Effect; + yield* db.updateMany("apps", { + where: (b: never) => (b as (...args: unknown[]) => unknown)("id", "=", target.app), + set: { activeDeployment: version }, + } as never) as Effect.Effect; + yield* link(target); + yield* db.updateMany("webhookSources", { + where: (b: never) => (b as (...args: unknown[]) => unknown)("app", "=", target.app), + set: { + status: "needs-setup", + failure: "accounts", + retries: 1, + leaseUntil: new Date(ticks - 1000), + }, + } as never) as Effect.Effect; + duringWaitingCheck = Effect.orDie(Effect.asVoid(setupCompleted(db, target as never))); + yield* sources.tick(limit); + const row = (yield* db.findFirst("webhookSources", { + where: (b: never) => (b as (...args: unknown[]) => unknown)("app", "=", target.app), + } as never) as Effect.Effect) as { + status: string; + retries: number; + leaseUntil: Date; + }; + return { + waitingChecks, + status: row.status, + retries: row.retries, + pushedAhead: row.leaseUntil.getTime() > ticks, + }; + } + case "relink-during-lease": { + const target = yield* appWithAccount; + yield* link(target); + const reached = yield* Deferred.make(); + const release = yield* Deferred.make(); + holding = { app: target.app, reached, release }; + // The first sweep claims the source and waits inside its reconciliation, lease held. + const first = yield* Effect.forkChild(sources.tick(limit)); + yield* Deferred.await(reached); + // Linked again while that sweep holds the lease. + yield* link(target); + // A second sweep meanwhile, with other intent beside it. + yield* link(yield* appWithAccount); + yield* sources.tick(limit); + const claimsWhileHeld = listed.get(target.app) ?? 0; + // The first sweep finishes; the link it did not see must not be lost. + yield* Deferred.succeed(release, undefined); + yield* Fiber.await(first); + const afterFirst = yield* status(target); + yield* sources.tick(limit); + return { + claimsWhileHeld, + afterFirst, + claimsAfter: listed.get(target.app) ?? 0, + finalStatus: yield* status(target), + }; + } + } + }); + +/** One node of a JSON query plan, as `explain (format json)` returns it. */ +interface PlanNode { + readonly "Node Type": string; + readonly "Index Name"?: string; + readonly "Actual Rows"?: number; + readonly Plans?: ReadonlyArray; +} + +/** The schema, migrated once on disk; each case opens a copy of it, a database of its own. */ +const migrated = (dataDir: string) => + Effect.scoped( + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const storage = yield* makeExecutorStorage({ provider: "postgresql" }).pipe( + Effect.provideService(SqlClient.SqlClient, sql), + ); + yield* storage.migrate; + }), + ).pipe(Effect.provide(pgliteLayer({ dataDir }))); + +/** + * Rows a case's database holds when the case opens it: zero for a database of its own. Each copy is + * a separate directory, opened by its own PGlite instance, so no write of one case reaches another. + */ +const rowsAtOpen = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const [row] = yield* sql.unsafe<{ rows: number }>( + `select ((select count(*) from executor_webhook_sources) + (select count(*) from executor_apps) + + (select count(*) from executor_accounts) + (select count(*) from executor_providers))::int as rows`, + ); + return row?.rows ?? -1; +}); + +/** + * Each case in turn, each on its own copy of the migrated database, in one process. A case whose + * database already holds rows is not run: `rowsAtOpen` reports it. + */ +const all = Effect.acquireUseRelease( + Effect.sync(() => mkdtempSync(join(tmpdir(), "webhook-sources-fixture-"))), + (directory) => + Effect.gen(function* () { + const template = join(directory, "template"); + yield* migrated(template); + const cases: Record = {}; + const opened: Record = {}; + for (const name of selected) { + const dataDir = join(directory, name); + cpSync(template, dataDir, { recursive: true }); + cases[name] = yield* Effect.scoped( + Effect.gen(function* () { + opened[name] = yield* rowsAtOpen; + return opened[name] === 0 ? yield* program(name) : null; + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + pgliteLayer({ dataDir }), + NodeCrypto.layerSecureRandom, + NodeCrypto.layerDigest, + ), + ), + ); + } + return { rowsAtOpen: opened, cases }; + }), + (directory) => Effect.sync(() => rmSync(directory, { recursive: true, force: true })), +); + +Effect.runPromise( + all.pipe( + Effect.provideService(Clock.Clock, tickingClock), + Effect.timeout("600 seconds"), + ) as never, +).then( + (result) => { + console.log(JSON.stringify(result)); + process.exit(0); + }, + (error) => { + console.error( + String(error).slice(0, 2000), + JSON.stringify( + error?.cause ?? error, + Object.getOwnPropertyNames(error?.cause ?? error), + ).slice(0, 1500), + ); + process.exit(1); + }, +); diff --git a/packages/sdk/src/contracts/engine-faults.ts b/packages/sdk/src/contracts/engine-faults.ts new file mode 100644 index 0000000000..3cd4655f7a --- /dev/null +++ b/packages/sdk/src/contracts/engine-faults.ts @@ -0,0 +1,69 @@ +/** + * Engine call faults for E2E runs only, from `EXECUTOR_TEST_WORKFLOW_ENGINE_FAULTS` (this list as + * JSON). A Node host that starts its own workflow runtime (local, and self-host's development + * entry) reads it only with `EXECUTOR_ENVIRONMENT=e2e`; Cloud only with emulated providers, which + * only loopback Cloud and `test-e2e-` stages accept. Any other run has none. + * + * Each fault fails a run's first `failures` calls of one kind to one authored step name, `before` + * the engine sees the call or `after` it answered (for a sleep, after the engine waited). It fails + * with the error an engine would throw for its `answer`, which the host classifies as it would the + * engine's own: `internal`, a transient internal error; `paused` and `terminated`, the local + * engine's "Aborting engine: …" for a paused or terminated instance; `relayed`, a failure carrying + * our marker, as the engine relays a step's own failure. + */ +import { Schema } from "effect"; +import { WorkflowFailure } from "apps/contracts"; +import { workflowFailureMessage } from "./workflow-errors.ts"; + +export const EngineFault = Schema.Struct({ + /** The engine's name for the call: a sleep, a sleepUntil, or a step of one of these kinds. */ + call: Schema.Literals(["sleep", "until", "do", "mutation", "query"]), + step: Schema.String.check(Schema.isMaxLength(100)), + failures: Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), + when: Schema.Literals(["before", "after"]), + answer: Schema.Literals(["internal", "paused", "terminated", "relayed"]), +}); +export type EngineFault = typeof EngineFault.Type; +export const EngineFaults = Schema.Array(EngineFault); + +/** How many calls each configured fault has failed, by run and by the call it fails. */ +const faulted = new Map(); +/** + * The fault that fails this call to the engine, counting it, or undefined once its failures are + * spent. `name` is the engine's name for the call, `::`. + */ +export const engineFaultFor = ( + faults: ReadonlyArray | null | undefined, + run: string, + name: string, +) => { + for (const fault of faults ?? []) { + if (!name.startsWith(`${fault.call}:${fault.step}:`)) continue; + const key = `${run}\u0000${fault.call}:${fault.step}`; + const count = faulted.get(key) ?? 0; + if (count >= fault.failures) return undefined; + faulted.set(key, count + 1); + return fault; + } + return undefined; +}; +/** The error an engine would throw for the fault's answer. */ +export const engineFaultError = (fault: EngineFault): Error => { + switch (fault.answer) { + case "internal": + return new Error("internal error"); + case "paused": + return new Error("Aborting engine: User called pause"); + case "terminated": + return new Error("Aborting engine: User called terminate"); + case "relayed": + return new Error( + workflowFailureMessage(new WorkflowFailure({ reason: "execution", retryable: true })), + ); + } +}; +/** The engine call `work`, failed by `fault` before the engine sees it or after it answered. */ +export const engineFaultCall = (fault: EngineFault, work: () => Promise): Promise => + fault.when === "before" + ? Promise.reject(engineFaultError(fault)) + : work().then(() => Promise.reject(engineFaultError(fault))); diff --git a/packages/sdk/src/contracts/executor.ts b/packages/sdk/src/contracts/executor.ts index 7a276a180c..36959f7a63 100644 --- a/packages/sdk/src/contracts/executor.ts +++ b/packages/sdk/src/contracts/executor.ts @@ -6,7 +6,7 @@ import { type WebhookSourceDispatcher, } from "./webhooks.ts"; /** The shared Executor interface and remote client options; projected from ExecutorApi. */ -import { Context, type Effect, type Redacted, type Stream, Schema } from "effect"; +import { Context, type Effect, type Redacted, type Result, type Stream, Schema } from "effect"; import type { HttpApi, HttpApiEndpoint, HttpApiGroup } from "effect/http-api"; import type { WebhookSetupApi } from "./webhook-setup.ts"; import type { ExecutorApi } from "./http.ts"; @@ -145,17 +145,22 @@ export interface ResourceLifecycle { appUses: readonly AppAccountUse[], ) => Effect.Effect, StorageError>; /** - * Optional: `appResolving` for several apps at once, such as the apps one catalog search reads, - * in one call. Returns each check's passing accounts, in order, as `appResolving` would, or - * `undefined` for a check `appResolving` would fail. + * {@link appResolving} for several apps of one owner at once, for the `CurrentActor`, such as + * every app a search reads: a host reads them in one statement. Each app's entry, in order, is + * what `appResolving` returns for it, or its refusal. Failing the whole call, such as for + * storage, says nothing about any app: the SDK then checks each on its own. */ - readonly appsResolving?: ( - checks: ReadonlyArray<{ - readonly app: { readonly id: AppId; readonly owner: OwnerId }; + readonly appsResolving: ( + owner: OwnerId, + apps: ReadonlyArray<{ + readonly id: AppId; readonly accounts: readonly Pick[]; readonly appUses: readonly AppAccountUse[]; }>, - ) => Effect.Effect | undefined>, StorageError>; + ) => Effect.Effect< + ReadonlyArray, StorageError>>, + StorageError + >; /** * Recheck authority to remove what an app registered upstream, such as a webhook registration, * with the accounts it registered with, before acquiring their credentials and again after any diff --git a/packages/sdk/src/contracts/oauth.ts b/packages/sdk/src/contracts/oauth.ts index e349eb92d8..8c417ccfd9 100644 --- a/packages/sdk/src/contracts/oauth.ts +++ b/packages/sdk/src/contracts/oauth.ts @@ -83,12 +83,33 @@ export const OAuthResourceIndicator = Schema.Union([ ]); export type OAuthResourceIndicator = typeof OAuthResourceIndicator.Type; +/** + * The operator's client matches a sign-in's authorization server, but may not ask for some of the + * scopes the sign-in requests. `scopes` are those, as the provider declares them; `instead` names, + * for each one the client records a replacement for, the allowed scopes that cover it, which grant + * more than it does. + */ +export const OperatorScopeRefusal = Schema.Struct({ + /** The client's label, such as "Executor for Google". */ + client: Schema.NonEmptyString, + scopes: Schema.NonEmptyArray(Schema.String), + instead: Schema.Array( + Schema.Struct({ scope: Schema.String, scopes: Schema.NonEmptyArray(Schema.String) }), + ), +}); +export type OperatorScopeRefusal = typeof OperatorScopeRefusal.Type; + const clientSetup = { /** `automatic`: a client metadata document or dynamic registration can supply a client. */ mode: Schema.Literals(["automatic", "client-required"]), scopes: Schema.Array(Schema.String), /** Signing in uses the instance's own OAuth client, named here; see `FirstPartyOAuthClient`. */ firstParty: Schema.optionalKey(Schema.Struct({ label: Schema.NonEmptyString })), + /** + * The instance's own OAuth client matches the service but may not ask for these scopes, so this + * sign-in needs another client. Only with `client-required`. + */ + operatorScopeRefusal: Schema.optionalKey(OperatorScopeRefusal), /** The resource indicator sign-in sends, or why it sends none. */ resource: OAuthResourceIndicator, }; @@ -429,11 +450,15 @@ export type FirstPartyOAuthClientId = typeof FirstPartyOAuthClientId.Type; * An OAuth client the instance's operator provides, such as one-click Google sign-in. It is offered * for a provider method only when the method's authorization and token endpoints are exactly the * client's and the scopes it asks for are among `allowedScopes`; scopes the method only discovered - * in the resource's advertised list are narrowed to `allowedScopes` instead. Signing in, renewal and - * revocation then use only the operator's server configuration, and the access token it issues - * is managed: it reaches app code sealed, goes only as `placement` says, and a token response - * reporting a scope outside `allowedScopes` is refused. Its secret never leaves the host: grants - * keep only its ID, so rotating or removing it applies to every account. + * in the resource's advertised list are narrowed to `allowedScopes` instead. When the endpoints + * match but a requested scope is one of its `heldScopes`, the sign-in is refused before it reaches + * the service (`OAuthSetupFailed` with `operator_scope_refused`) unless another automatic client + * signs in. + * Signing in, renewal and revocation then use only the operator's server configuration, and the + * access token it issues is managed: it reaches app code sealed, goes only as `placement` says, + * and a token response reporting a scope outside `allowedScopes` and `heldScopes` is refused. Its + * secret never leaves the host: grants keep only its ID, so rotating or removing it applies to + * every account. */ export const FirstPartyOAuthClient = Schema.Struct({ id: FirstPartyOAuthClientId, @@ -447,6 +472,16 @@ export const FirstPartyOAuthClient = Schema.Struct({ defaultScopes: Schema.Array(ScopeToken), /** Every scope a sign-in may ask for and a grant may hold. Defaults to `defaultScopes`. */ allowedScopes: Schema.optionalKey(Schema.Array(ScopeToken)), + /** + * Scopes a grant may hold that no sign-in asks for: ones the service does not let this client ask + * for, which grants it issued earlier hold. Those grants keep renewing; a sign-in, a reconnect + * included, that asks for one is refused before it reaches the service. The refusal leads with + * the user's own client, then names `instead`, when it has any: allowed scopes that cover the + * held one and grant more than it, which an app may ask for only if its user accepts that. + */ + heldScopes: Schema.optionalKey( + Schema.Array(Schema.Struct({ scope: ScopeToken, instead: Schema.Array(ScopeToken) })), + ), placement: ManagedPlacement, /** * Withheld from new sign-ins: offered only to reconnect an account whose grant it issued. Its @@ -462,9 +497,11 @@ export const FirstPartyOAuthClient = Schema.Struct({ ), Schema.makeFilter( (client) => - [...client.defaultScopes, ...(client.allowedScopes ?? [])].every( - (scope) => !scope.includes(client.server.scopeSeparator ?? " "), - ), + [ + ...client.defaultScopes, + ...(client.allowedScopes ?? []), + ...(client.heldScopes ?? []).flatMap((held) => [held.scope, ...held.instead]), + ].every((scope) => !scope.includes(client.server.scopeSeparator ?? " ")), { message: "A scope cannot contain the server's scope separator." }, ), Schema.makeFilter( @@ -473,12 +510,31 @@ export const FirstPartyOAuthClient = Schema.Struct({ client.defaultScopes.every((scope) => client.allowedScopes?.includes(scope) === true), { message: "Every default scope must be allowed." }, ), + Schema.makeFilter( + (client) => + (client.heldScopes ?? []).every( + (held) => + !allowedScopes(client).includes(held.scope) && + held.instead.every((scope) => allowedScopes(client).includes(scope)), + ), + { message: "A held scope cannot be allowed, and every scope it names instead must be." }, + ), ); export type FirstPartyOAuthClient = typeof FirstPartyOAuthClient.Type; /** The scopes a client allows: `allowedScopes`, or its defaults when it lists none. */ -export const allowedScopes = (client: FirstPartyOAuthClient) => - client.allowedScopes ?? client.defaultScopes; +export function allowedScopes(client: { + readonly defaultScopes: readonly string[]; + readonly allowedScopes?: readonly string[]; +}) { + return client.allowedScopes ?? client.defaultScopes; +} + +/** The scopes a grant the client issued may hold: those it allows, and its held scopes. */ +export const grantableScopes = (client: FirstPartyOAuthClient) => [ + ...allowedScopes(client), + ...(client.heldScopes ?? []).map((held) => held.scope), +]; /** Network transport and client identity belong to the product hosting this SDK. */ export interface OAuthOptions { @@ -518,6 +574,59 @@ export interface OAuthOptions { readonly firstPartyClients?: readonly FirstPartyOAuthClient[]; } +/** At most five scopes, each at most 120 characters, and how many more there are. */ +const scopeList = (scopes: ReadonlyArray) => { + const shown = scopes + .slice(0, 5) + .map((scope) => (scope.length > 120 ? `${scope.slice(0, 119)}…` : scope)); + const more = scopes.length - shown.length; + return more > 0 ? `${shown.join(", ")} and ${more} more` : shown.join(", "); +}; + +/** + * The operator's client matches the service but may not ask for some of the requested scopes. The + * scopes are the app's own declaration, shown to that app's user. Recovery leads with the user's + * own client, which asks for the same scopes. When every refused scope has a replacement it then + * names them, saying they grant more, since a replacement covers what it replaces. + */ +const operatorScopeRefused = ( + refusal: OperatorScopeRefusal | undefined, + callback: string, +): ErrorPresentation => { + const ownClient = + "create an OAuth client in the service’s developer console that allows Executor’s callback URL, and connect with its client ID and secret (in the dashboard, Enter client details; through the API, pass `client: { clientId, clientSecret }` to the connection’s `oauth/start`)."; + const unchanged = + "Do not start the same sign-in again unchanged: it is refused the same way. The refusal changes no saved account."; + if (refusal === undefined) + return { + title: "Sign-in needs your own OAuth client", + description: + "This instance’s own OAuth client for the service can’t ask for every permission this app requests, so Executor didn’t start the sign-in.", + recovery: { + action: "Enter your own OAuth client for this service.", + instructions: `The instance's own OAuth client matches the provider's authorization server but may not request every scope the provider declares, so Executor refused the sign-in before redirecting. Compare the provider's declared scopes with what the instance's client allows, or ${ownClient} ${unchanged}${callback}`, + }, + }; + const { client, scopes, instead } = refusal; + const one = scopes.length === 1; + const replaced = scopes.every((scope) => instead.some((entry) => entry.scope === scope)); + const swaps = instead.map((entry) => `${entry.scopes.join(" ")} instead of ${entry.scope}`); + return { + title: `${client} can’t ask for ${one ? "this permission" : "these permissions"}`, + description: `This app asks for ${scopeList(scopes)}, which ${client} isn’t approved to request, so Executor didn’t start the sign-in.`, + recovery: { + action: replaced + ? `Enter your own OAuth client for this service. Or, if broader access is acceptable, change the app to ask for ${scopeList(instead.flatMap((entry) => entry.scopes))} instead, which grants more than ${scopeList(instead.map((entry) => entry.scope))}.` + : `Enter your own OAuth client for this service, or change the app to ask only for permissions ${client} allows.`, + instructions: `The provider's OAuth method requests ${scopes.join(" ")}. ${client}, this instance's own OAuth client, matches the provider's authorization and token endpoints but may not request ${one ? "that scope" : "those scopes"}, so Executor refused the sign-in before redirecting to the service. First choice: ${ownClient.charAt(0).toUpperCase()}${ownClient.slice(1)} A client of the person's own can ask for these scopes without changing the app. ${ + swaps.length === 0 + ? "Otherwise, change the provider's declared scopes to ones the instance's client allows, redeploy the app and start a new sign-in." + : `Second choice, only if the person accepts broader access: change the provider's declared scopes to ask for ${swaps.join("; ")}. Each replacement covers the scope it replaces and grants more access than it, so tell the person what more it allows and get their agreement before changing the app. Then redeploy the app and start a new sign-in, which ${client} signs in.` + } ${unchanged}${callback}`, + }, + }; +}; + /** * OAuth setup failed without exposing upstream bodies, URLs containing codes, or secrets. The * service's own error, when it sent one, is kept apart in `serviceError`. @@ -547,15 +656,18 @@ export const OAuthSetupFailed = UserFacingError.define({ "invalid_redirect", "token_exchange", "unsupported", + "operator_scope_refused", ]), /** Executor's own public callback, which some services must approve. Forms show it with client entry. */ callbackUrl: Schema.optional(HttpUrl), + /** With `operator_scope_refused`: the operator's client and the scopes it may not ask for. */ + operatorScopes: Schema.optional(OperatorScopeRefusal), cause: Schema.optional(OAuthFailureCause), serviceError: Schema.optional(OAuthServiceError), retryAfter: RetryAfter, }, recorded: ({ reason, cause }) => oauthRecorded("OAuth setup failed", reason, cause), - presentation: ({ reason, callbackUrl, cause, serviceError, retryAfter }) => { + presentation: ({ reason, callbackUrl, cause, serviceError, retryAfter, operatorScopes }) => { // Forms that open client entry already show the callback, so only the fix prompt repeats it. const callback = callbackUrl === undefined ? "" : ` Executor’s callback URL is ${callbackUrl}.`; return withCause( @@ -705,6 +817,7 @@ export const OAuthSetupFailed = UserFacingError.define({ "Compare the app’s provider definition with the service’s supported OAuth flow and Executor’s supported configuration. Update the app to a documented compatible method. Do not replace required authentication with an unauthenticated connection.", }, }, + operator_scope_refused: operatorScopeRefused(operatorScopes, callback), } satisfies Record )[reason], cause, @@ -714,6 +827,7 @@ export const OAuthSetupFailed = UserFacingError.define({ }); /** Setup failures that a user-supplied OAuth client can resolve, so forms open client entry. */ export const oauthClientEntryReasons: ReadonlySet = new Set([ + "operator_scope_refused", "invalid_client", "client_not_approved", "client_registration_required", diff --git a/packages/sdk/src/contracts/runtime.ts b/packages/sdk/src/contracts/runtime.ts index cb6824f5c7..b18e81a68d 100644 --- a/packages/sdk/src/contracts/runtime.ts +++ b/packages/sdk/src/contracts/runtime.ts @@ -21,7 +21,7 @@ import { RecordedMessage } from "@executor-js/utils/recorded-message"; import { BuildStage, SourceLocation, type BuildMemoryExceeded } from "./version.ts"; import type { CodeFiles } from "./source.ts"; import { BuildId, Json } from "./shared.ts"; -import type { BlobKey, BlobRange } from "./blobs.ts"; +import type { BlobKey, BlobRange, BlobStoreError } from "./blobs.ts"; import type { HostApiCheck } from "./host-api.ts"; /** @@ -348,7 +348,17 @@ export const runtimeFailures: Record = { */ export class RuntimeProtocolFailed extends Schema.TaggedError()( "RuntimeProtocolFailed", - { reason: Schema.optional(RuntimeFailure), message: Schema.optional(BuildMessage) }, + { + reason: Schema.optional(RuntimeFailure), + message: Schema.optional(BuildMessage), + /** + * Nothing of the failed exchange left the host: its connection never formed, so the runtime + * received no work. Only the host's own transport sets it, from its socket's state, never from + * an error's text, so an app cannot produce it. Its `reason` stays `disconnected`, the value + * older readers know; they drop this field. + */ + notSent: Schema.optional(Schema.Literal(true)), + }, ) { get [RecordedMessage]() { return this.reason === undefined && !this.message @@ -418,7 +428,9 @@ export interface Runtime { | RuntimeBuildFailed | RuntimeProtocolUnsupported | RuntimeAppsDependencyMissing - | BuildMemoryExceeded, + | BuildMemoryExceeded + // Keeping the build failed in storage: nothing about the app, and building again may succeed. + | BlobStoreError, Requirements >; readonly asset?: (input: { diff --git a/packages/sdk/src/contracts/source.ts b/packages/sdk/src/contracts/source.ts index 4e6ddc5206..fa95fc2bbe 100644 --- a/packages/sdk/src/contracts/source.ts +++ b/packages/sdk/src/contracts/source.ts @@ -395,6 +395,16 @@ export const Branch = Schema.String.check( * read lists by their blob (`SourceStoredFile`) are not held and do not count here. */ export const sourceLimits = { files: 4096, bytes: 16 * 1024 * 1024 } as const; +/** + * The most bytes one of the app's own files the build compiles as JavaScript or TypeScript may + * hold. Files of the npm packages it imports are not limited and compile however large they are. + * esbuild compiles each file in a single synchronous stretch on the thread that serves the host's + * requests; this bounds the work it does on any one of the app's files, not a whole build, which + * with several large files, or a large package file, can still hold that thread for seconds. JSON files are outside this limit on purpose: the app's own server JSON imports + * become modules of their own, and JSON in packages or the browser build goes through esbuild's + * JSON loader. `ui/public/` files are never compiled. `sourceLimits` still bounds the whole source. + */ +export const compiledFileLimit = 2_000_000; /** * Public files under `ui/public/`, which builds never load and deploys copy one at a time: * diff --git a/packages/sdk/src/contracts/tools.ts b/packages/sdk/src/contracts/tools.ts index f610885a02..8d789c83ca 100644 --- a/packages/sdk/src/contracts/tools.ts +++ b/packages/sdk/src/contracts/tools.ts @@ -120,6 +120,13 @@ export interface ToolListOptions extends CatalogReadOptions { * and one page reads it once. A caller that sends pages on, such as an HTTP route, omits it. */ readonly unboundedPages?: true; + /** + * The apps one search reads together, this read's app among them. A host may read their state + * in shared statements, but never with an app outside the batch: a search waits only on the apps + * it reaches, so one app's slow or locked rows hold up no search that does not reach it. Without + * it the read takes its own state alone. + */ + readonly searchBatch?: ReadonlyArray; } export { ElicitationFailed, @@ -170,9 +177,39 @@ export const InvocationPrefetch = Context.Reference< } | undefined >("executor/InvocationPrefetch", { defaultValue: () => undefined }); +/** + * Host-only: the reads `tools.searchable` would make for many targets of one owner, made ahead in + * a fixed number of statements ({@link PrefetchHost}'s `searchable`). `value` is opaque to the + * host. `tools.searchable` uses an entry only for the app, version and accounts it was made for, + * under the same actor, and only once; any other read, and a read made ahead that failed in + * storage, reads live. Never supplied by authored code. + */ +export const SearchablePrefetch = Context.Reference( + "executor/SearchablePrefetch", + { defaultValue: () => undefined }, +); +/** What {@link PrefetchHost}'s `searchable` read, for {@link SearchablePrefetch}. */ +export interface SearchablePrefetched { + readonly owner: OwnerId; + readonly value: unknown; +} /** Host-only reads made ahead for a tool call, outside the public HTTP/Promise facade. */ export const PrefetchHost = Symbol("executor.PrefetchHost"); export interface PrefetchHost { + /** + * The reads `tools.searchable` makes for each target, after the host's own checks resolved its + * version and accounts: the apps and accounts, each target's kept search rows, the actor's app + * rechecks and the account grants, each kind in one statement for every target together. It + * never fails: a target whose reads did not answer is read live by `tools.searchable`. + */ + readonly searchable: (input: { + readonly owner: OwnerId; + readonly targets: ReadonlyArray<{ + readonly app: AppId; + readonly version: VersionId; + readonly accounts: AppAccounts; + }>; + }) => Effect.Effect; /** * The reads `apps.get` and then `tools.call` make for one app, as effects of one statement each * that never fail, for the host to send in one batch beside its own checks. Without `known`, @@ -288,8 +325,13 @@ export const mayHaveWrittenPresentation = (presented: ErrorPresentation): ErrorP export const mayHaveWritten = ( schema: Schema.Codec, failure: A, -): A => - Schema.decodeUnknownSync(schema)({ ...Schema.encodeSync(schema)(failure), mayHaveWritten: true }); +): A => { + // A failure that may have written never also says that nothing was sent. + const { notSent: _, ...encoded } = Schema.encodeSync(schema)(failure) as I & { + readonly notSent?: unknown; + }; + return Schema.decodeUnknownSync(schema)({ ...encoded, mayHaveWritten: true }); +}; /** Why a running tool could not complete its user interaction. */ const elicitationFailures = { @@ -917,6 +959,11 @@ export const EvaluationRuntime = Schema.Union([ kind: Schema.Literal("worker"), /** The failure the runtime's error matched, absent when the runtime reported none. */ failure: Schema.optional(RuntimeFailure), + /** + * The host's connection to the runtime never formed, so nothing was sent to it (`failure` is + * then `disconnected`, as older readers expect; they drop this field). + */ + notSent: Schema.optional(Schema.Literal(true)), }), Schema.Struct({ kind: Schema.Literal("build") }), Schema.Struct({ @@ -951,6 +998,16 @@ const retryOnce = (then: string, instructions: string) => ({ const reportReference = "If it fails again, report the failure with the reference shown beside this error, if any, and check what the app's recent changes run while it lists its tools."; +/** What a runtime that could not be reached says: nothing was sent to the app. */ +const unreachablePresentation = { + description: "Executor could not connect to the app runtime, so nothing was sent to the app.", + recovery: retryOnce( + "check that Executor's app runtime is running.", + "Executor could not open its connection to the app runtime, so nothing was sent to the app. Executor's log records when the app runtime stops or is replaced.", + ), + retryable: true, +}; + /** What the app's Worker failure says and what to do about it. */ const workerPresentation = (failure: RuntimeFailure | undefined) => { const stopped = "The app’s Worker stopped without an answer from the app’s code"; @@ -969,9 +1026,14 @@ const workerPresentation = (failure: RuntimeFailure | undefined) => { case "too-large": return { description, + // One retry can help: a first load can do work the next reuses, such as compiling a large + // API definition into the app's cache. The limit is matched from text, and a later attempt + // can still pass, so a repeat proves nothing about the cause: the copy stops unchanged + // retries and says what to check. The host does not know the app's apps version here, so + // rebuilding is advised only for an app on an older one. recovery: retryOnce( - "reduce the work the app does to list its tools.", - "A resource limit matches this failure. Find what the app's factory and tool loaders do while listing, such as fetching or parsing a large API definition or building many tools, and reduce it: expose fewer tools, or keep a large definition in the app's cache.", + "stop retrying it unchanged and investigate. If the app was built on an older apps version than this host’s, rebuild it on this host’s version and deploy it again; if it already runs this host’s version, or still fails, reduce the work the app does to list its tools.", + "A resource limit matches this failure. A first load can do work the next one reuses, such as compiling a large API definition into the app's cache, so one retry may succeed. If it fails the same way, stop retrying the unchanged request, even though a later attempt might succeed, and investigate what listing the app's tools costs. Compare the apps version in the app's package.json with framework.release, the version this host runs. If the app's version is older, for example a catalog app installed before a framework fix, pin this host's version and deploy the app again: an older apps version can spend far more work listing a large API. If the versions are the same, or the rebuilt app still fails, find what the app's factory and tool loaders do while listing, such as fetching or parsing a large API definition or building many tools, and reduce it: expose fewer tools, or keep a large definition in the app's cache.", ), retryable: true, }; @@ -1042,9 +1104,10 @@ const runtimePresentation = (runtime: EvaluationRuntime): CausePresentation => { switch (runtime.kind) { case "worker": case "build": { - const { description, ...rest } = workerPresentation( - runtime.kind === "build" ? "build" : runtime.failure, - ); + const { description, ...rest } = + runtime.kind === "worker" && runtime.notSent === true + ? unreachablePresentation + : workerPresentation(runtime.kind === "build" ? "build" : runtime.failure); return { title: "Tools could not be loaded", description: loaded(description), ...rest }; } case "protocol": @@ -1091,6 +1154,7 @@ const evaluationRuntime = (error: unknown): EvaluationRuntime | undefined => { return { kind: "worker", ...(error.reason === undefined ? {} : { failure: error.reason }), + ...(error.notSent === true ? { notSent: true as const } : {}), }; if (Schema.is(RuntimeBuildUnavailable)(error)) return { kind: "build" }; if (Schema.is(RuntimeProtocolUnsupported)(error)) @@ -1716,19 +1780,41 @@ export const ToolCallFailed = UserFacingError.define({ * Worker, which can produce it, so the match is a likely cause, never a confirmed one. */ rpcLimitMatched: Schema.optional(Schema.Literal(true)), + /** + * Repeating this call is safe: the exchange that failed never left the host (its connection to + * the app runtime never formed), and either it was the call's only hand-off to the app's code, + * or the app confirms the call only reads. A confirmed read repeated after a credential renewal + * keeps it although its first run reached the app: that run only read. Never set together with + * `mayHaveWritten`. + */ + notSent: Schema.optional(Schema.Literal(true)), mayHaveWritten: MayHaveWritten, }, - recorded: ({ failure, mcp, response, rpcLimitMatched }) => - rpcLimitMatched === true - ? "The tool's failure matched the Workers RPC size limit's message" - : mcp !== undefined - ? `The tool failed: ${recordedMcp(mcp)}` - : failure !== undefined - ? `The tool failed: ${recordedApp(failure)}` - : response !== undefined - ? "The tool failed after starting" - : "The tool failed without further detail", - presentation: callPresentation(({ reason, response, failure, mcp, rpcLimitMatched }) => { + recorded: ({ failure, mcp, response, rpcLimitMatched, notSent }) => + notSent === true + ? "The tool call was not sent: the app runtime could not be reached" + : rpcLimitMatched === true + ? "The tool's failure matched the Workers RPC size limit's message" + : mcp !== undefined + ? `The tool failed: ${recordedMcp(mcp)}` + : failure !== undefined + ? `The tool failed: ${recordedApp(failure)}` + : response !== undefined + ? "The tool failed after starting" + : "The tool failed without further detail", + presentation: callPresentation(({ reason, response, failure, mcp, rpcLimitMatched, notSent }) => { + // The host's own fact, never the app's: the call did not leave the host. + if (notSent === true) + return { + title: "Tool call was not sent", + description: reason, + recovery: { + action: "Try the call again.", + instructions: + "Executor could not open its connection to the app runtime, so it sent nothing of this call and the tool did not run. Repeating the call is safe. If it fails the same way again, the app runtime may have stopped; Executor's log records when it stops or is replaced.", + }, + retryable: true, + }; if (mcp !== undefined) return mcpFailurePresentation(mcp); // App code can produce the runtime's text, so the match names a likely cause and keeps a read's // bounded retry. Executor sets no size limit of its own on a result. diff --git a/packages/sdk/src/contracts/workflow-errors.ts b/packages/sdk/src/contracts/workflow-errors.ts index 0dc3f16588..c02dd01d6d 100644 --- a/packages/sdk/src/contracts/workflow-errors.ts +++ b/packages/sdk/src/contracts/workflow-errors.ts @@ -29,6 +29,80 @@ export const workflowFailureMessage = (failure: WorkflowFailure) => { detail === undefined ? "" : ` ${Schema.encodeSync(DetailJson)(detail)}` }`; }; +/** Whether a native error carries our failure marker, so the failure is ours and not the engine's. */ +export const hasWorkflowFailureMarker = (error: unknown) => + Schema.is(WorkflowFailure)(error) || (error instanceof Error && marker.test(error.message)); + +/** + * Where a call from a run's body to the workflow engine stopped: the platform's engine itself, the + * runner's call to Executor's host, which did not answer or answered unreadably, or the runner. + */ +export type EngineCallHop = "platform" | "host" | "runner"; +/** An engine call failure travels as these fields, which every release since detail existed reads. */ +const engineCallName = "WorkflowEngineCallFailed"; +const engineCallForm = + /^hop=(platform|host|runner); class=([A-Za-z]{1,40}); code=([A-Za-z0-9_.-]{1,40})$/; +const word = (value: string) => (/^[A-Za-z0-9_.-]{1,40}$/.test(value) ? value : "other"); +/** + * A call to the engine that failed in transit or in the engine, not in the app's code: retryable, + * and recorded only as its hop, its error's standard class name and a classified code. Never the + * platform's message text, which can repeat the app's step name or input. + */ +export const engineCallFailure = (hop: EngineCallHop, errorClass: string, code: string) => + new WorkflowFailure({ + reason: "engine", + retryable: true, + errorName: engineCallName, + message: `hop=${hop}; class=${/^[A-Za-z]{1,40}$/.test(errorClass) ? errorClass : "other"}; code=${word(code)}`, + }); +/** The hop, class and code of an engine call failure, or undefined for any other failure. */ +export const engineCallOf = (failure: WorkflowFailure) => { + if (failure.reason !== "engine" || !failure.retryable || failure.errorName !== engineCallName) + return undefined; + const match = engineCallForm.exec(failure.message); + if (match === null) return undefined; + return { hop: match[1] as EngineCallHop, errorClass: match[2]!, code: match[3]! }; +}; + +/** + * What an engine's error says, as a code, from its message alone; the message is never kept. + * The local engine (miniflare's `workflows/binding.worker.js`) aborts a paused, restarted or + * terminated instance's calls with "Aborting engine: …", and refuses a `sleepUntil` in the past. + * Cloudflare's engine names instance states as `instance.`. Neither documents a full list, + * so a message no rule recognises is `unclassified`. + */ +export const engineErrorCode = (error: unknown): string => { + const message = error instanceof Error ? error.message : ""; + if (message.startsWith("Aborting engine:")) return "aborted"; + if (/sleep until a time in the past/i.test(message)) return "past"; + const instance = /instance\.([a-z_]+)/.exec(message)?.[0]; + if (instance !== undefined) return instance; + if (/terminat/i.test(message)) return "terminated"; + if (/internal error|internal_server/i.test(message)) return "internal"; + if (/overloaded|too many requests|rate.?limit/i.test(message)) return "overloaded"; + if (/network connection lost|disconnected/i.test(message)) return "disconnected"; + if (/timed out|timeout/i.test(message)) return "timeout"; + return "unclassified"; +}; +/** + * The codes of a call that failed on its way, not by the engine's choice: an internal error or + * overload the engine reports as passing, a lost connection, or a host that never answered. Any + * other code, an aborted, terminated or paused instance, a refusal, an instance state, or a + * failure no rule recognises, is the engine's answer and is not repeated. + */ +const transientCodes: ReadonlySet = new Set([ + "internal", + "overloaded", + "disconnected", + "timeout", + "unanswered", +]); +/** Whether a failed engine call is safe and worth calling again: a transient failure in transit. */ +export const transientEngineCall = (failure: WorkflowFailure) => { + const call = engineCallOf(failure); + return call !== undefined && transientCodes.has(call.code) ? call : undefined; +}; + /** Native engines can wrap exception names; recover only our validated marker and detail. */ export const decodeWorkflowFailure = (error: unknown): WorkflowFailure => { if (Schema.is(WorkflowFailure)(error)) return error; diff --git a/packages/sdk/src/core.ts b/packages/sdk/src/core.ts index 849f4bab6d..9c9fc275ed 100644 --- a/packages/sdk/src/core.ts +++ b/packages/sdk/src/core.ts @@ -57,6 +57,13 @@ export { type WorkflowDriver, } from "./contracts/workflow-runtime.ts"; export { decodeWorkflowFailure, workflowFailureMessage } from "./contracts/workflow-errors.ts"; +export { nativeEngineFailure } from "./implementation/native-engine-failure.ts"; +export { + EngineFaults, + engineFaultError, + engineFaultFor, + type EngineFault, +} from "./contracts/engine-faults.ts"; export { AppRepositoryRecovery } from "./implementation/initial-source.ts"; export { asciiJson } from "./implementation/ascii-json.ts"; diff --git a/packages/sdk/src/implementation/apps.ts b/packages/sdk/src/implementation/apps.ts index de45117884..d939acd9c6 100644 --- a/packages/sdk/src/implementation/apps.ts +++ b/packages/sdk/src/implementation/apps.ts @@ -60,7 +60,7 @@ import { import type { Registry } from "../contracts/registry.ts"; import { resolvePublication } from "./registry.ts"; -import type { BlobStorage } from "../contracts/blobs.ts"; +import { BlobStoreError, type BlobStorage } from "../contracts/blobs.ts"; import { readVersionSource, writeVersionSource } from "./version-source.ts"; import { readInitialSource, writeInitialSource } from "./initial-source.ts"; import { @@ -236,15 +236,28 @@ const appWriteFailure = (app: Pick) => (er /** Insert app identity while preserving name and derived-slug uniqueness errors. */ /** * A new active version re-registers the app's managed webhooks; removed sources keep removing. It is - * new intent, so each source runs again with no failures counted. + * new intent, so each source runs again with no failures counted. A source without a lease joins + * the pending queue now (its `leaseUntil`, which `tick` orders pending sources by); one whose lease + * is held keeps it, and its holder queues it again when it finishes. */ const webhookSourcesPending = (db: Query, app: AppId) => - query(() => - db.updateMany("webhookSources", { - where: (b) => b.and(b("app", "=", app), b("status", "!=", "removing")), - set: { status: "pending", failure: null, retries: 0 }, - }), - ); + Effect.gen(function* () { + const joined = new Date(yield* Clock.currentTimeMillis); + yield* query(() => + db.updateMany("webhookSources", { + where: (b) => + b.and(b("app", "=", app), b("status", "!=", "removing"), b("lease", "is", null)), + set: { status: "pending", failure: null, retries: 0, leaseUntil: joined }, + }), + ); + yield* query(() => + db.updateMany("webhookSources", { + where: (b) => + b.and(b("app", "=", app), b("status", "!=", "removing"), b("lease", "is not", null)), + set: { status: "pending", failure: null, retries: 0 }, + }), + ); + }); export const createApp = (db: Query, app: StoredApp) => { // Storage keeps the column's former name. @@ -333,7 +346,11 @@ export const makeApps = ( reason: error.message, message: error.message, }) - : versionBuildFailed(owner, name, error), + : // Storage failed while keeping the build: nothing about the app, and + // deploying again may succeed. + Schema.is(BlobStoreError)(error) + ? new StorageError() + : versionBuildFailed(owner, name, error), ), ); // Each requirement records its scope; a name is used in one scope only. diff --git a/packages/sdk/src/implementation/create.ts b/packages/sdk/src/implementation/create.ts index 8b67120369..f36b07bd53 100644 --- a/packages/sdk/src/implementation/create.ts +++ b/packages/sdk/src/implementation/create.ts @@ -34,7 +34,7 @@ import { makeOwners } from "./owners.ts"; import { makeSchedules } from "./schedules.ts"; import { makeTools } from "./tools.ts"; import { makeSkills } from "./skills.ts"; -import { toEffectRuntime } from "./runtime.ts"; +import { toEffectRuntime, withTracedPuts } from "./runtime.ts"; import { database } from "./database.ts"; import { makeOAuth } from "./oauth.ts"; import { makeDeclarationCache, makeDeclarations } from "./declarations.ts"; @@ -52,6 +52,8 @@ export const createExecutor = ( const credentials = options.credentials ?? (yield* aesGcmCredentials(options.secret, globalThis.crypto)); const db = database(options.database); + // Each write of the executor's own (sources, public files) has a span with its kind and size. + const blobs = withTracedPuts(options.blobs); const sources = gitSourceStorage(options.git); const origin = options.origin ?? hostedExecutorOrigin; const catalog = makeRegistry( @@ -59,7 +61,7 @@ export const createExecutor = ( origin, db, sources, - options.blobs, + blobs, ); const toolListings = { ...defaultToolListingPolicy, ...options.cache?.toolListings }; const cache = options.cache?.memory ?? makeDeclarationCache(); @@ -79,6 +81,7 @@ export const createExecutor = ( Effect.mapError(() => new RuntimeProtocolFailed({ reason: "data" })), ), }; + // The runtime traces its own store's writes (toEffectRuntime); it gets the store as given. const base = toEffectRuntime(options.runtime, options.blobs, cache); const runtime: Runtime = { ...base, @@ -104,6 +107,7 @@ export const createExecutor = ( background: options.background, resolveAccount: oauth.resolveSelected, accountsUsable: oauth.usableAll, + accountsUsableEach: oauth.usableEach, crypto, lifecycle: options.hooks, hostApi: options.hostApi, @@ -136,7 +140,7 @@ export const createExecutor = ( sources, options.git, catalog.reads, - options.blobs, + blobs, options.markdown, options.hooks, ), @@ -217,13 +221,13 @@ export const createExecutor = ( request: (input, request) => Effect.gen(function* () { const app = yield* storedApp(db, input); - yield* initializeAppRepository(db, sources, options.blobs, app); + yield* initializeAppRepository(db, sources, blobs, app); return yield* options.git.request(app.code, request); }), recover: recoverAppRepositories({ database: options.database, sources, - blobs: options.blobs, + blobs, }), }, [StorageHost]: { @@ -235,7 +239,10 @@ export const createExecutor = ( read: options.database.reactivity.read, write: options.database.reactivity.write, }, - [PrefetchHost]: { invocation: tools.prefetchInvocation }, + [PrefetchHost]: { + invocation: tools.prefetchInvocation, + searchable: tools.prefetchSearchable, + }, [SavedOAuthClientsHost]: makeSavedOAuthClients(db, credentials, crypto, options.hooks, [ ...(options.oauth?.savedClientCallbacks ?? []), ...(options.oauth?.previousRedirectUris ?? []), @@ -281,7 +288,7 @@ export const createExecutor = ( owners: makeOwners(db), publications: catalog.publications, registry: catalog.registry, - skills: makeSkills(db, runtime, crypto, declarations, options.blobs), + skills: makeSkills(db, runtime, crypto, declarations, blobs), webhooks: { ...webhooks.webhooks, sources: webhookSources.sources, diff --git a/packages/sdk/src/implementation/declarations.ts b/packages/sdk/src/implementation/declarations.ts index 0c638ac969..69c5c1fd7a 100644 --- a/packages/sdk/src/implementation/declarations.ts +++ b/packages/sdk/src/implementation/declarations.ts @@ -20,7 +20,7 @@ import { import type { HostApi, ResourceLifecycle } from "../contracts/executor.ts"; import type { AppRequirements } from "../contracts/apps.ts"; import { appCacheNamespace, type HostApiCheck } from "../contracts/host-api.ts"; -import { type AccountId, StorageError } from "../contracts/shared.ts"; +import { type AccountId, type OwnerId, StorageError } from "../contracts/shared.ts"; import type { makeOAuth } from "./oauth.ts"; import { makeHandoff } from "./handoff.ts"; import { resolve, type InvocationSnapshot } from "./tools.ts"; @@ -122,6 +122,7 @@ export const makeDeclarations = (options: { readonly background: BackgroundWork | undefined; readonly resolveAccount: ReturnType["resolveSelected"]; readonly accountsUsable: ReturnType["usableAll"]; + readonly accountsUsableEach: ReturnType["usableEach"]; readonly crypto: HostCrypto; readonly lifecycle: ResourceLifecycle | undefined; readonly hostApi: HostApi | undefined; @@ -159,81 +160,80 @@ export const makeDeclarations = (options: { * would stop it: a kept result is never served for an account a live read would refuse, such * as an OAuth grant that needs reconnecting. */ + /** What {@link authorize} checks for a state: its accounts, by how each use is authorized. */ const checksOf = (state: InvocationSnapshot) => { const selected = state.selections.flatMap(({ required, accounts }) => accounts.map((account) => ({ account, provider: required.definition })), ); - const accounts = selected.map(({ account }) => account); // Each use is authorized on its own: by the actor's use where a user requirement uses the // account, and by the app link for each app requirement it fills. const { user: linked, app: appUses } = accountAuthority(state.selections); return { + selected, linked, appUses, + /** Whether the recheck allowed every account the state runs with. */ permitted: (allowed: ReadonlySet) => - accounts.every((account) => allowed.has(account.id)) - ? Effect.void - : Effect.fail(new StorageError()), - usable: options.accountsUsable(selected), + selected.every(({ account }) => allowed.has(account.id)), }; }; const authorize = (state: InvocationSnapshot) => Effect.gen(function* () { const lifecycle = options.lifecycle; - const { linked, appUses, permitted, usable } = checksOf(state); + const checks = checksOf(state); // The actor's use of the app and of every account is one read, alongside the grant checks. yield* Effect.all( [ lifecycle === undefined ? Effect.void - : lifecycle.appResolving(state.app, linked, appUses).pipe(Effect.flatMap(permitted)), - usable, + : lifecycle + .appResolving(state.app, checks.linked, checks.appUses) + .pipe( + Effect.flatMap((allowed) => + checks.permitted(allowed) ? Effect.void : Effect.fail(new StorageError()), + ), + ), + options.accountsUsable(checks.selected), ], { concurrency: "unbounded", discard: true }, ); }); /** - * `authorize` for many states, with each state's outcome. A product that offers - * `appsResolving` rechecks the actor's use of every app and account in one call; the same checks - * decide each state. + * {@link authorize} for several states of one owner, each outcome in order: the actor's rechecks + * of every app in one call (`appsResolving`) beside one read of every account's grant. A state + * the recheck refuses fails as `authorize` fails for it, before its grants are considered. + * Failing the whole call, such as for storage, says nothing about any state. */ - const authorizeEach = (states: ReadonlyArray) => + const authorizeAll = (owner: OwnerId, states: ReadonlyArray) => Effect.gen(function* () { - const many = options.lifecycle?.appsResolving; - if (many === undefined || states.length < 2) - return yield* Effect.forEach(states, (state) => Effect.result(authorize(state)), { - concurrency: "unbounded", - }); + const lifecycle = options.lifecycle; const checks = states.map(checksOf); - const allowed = yield* Effect.result( - many( - states.map((state, index) => ({ - app: state.app, - accounts: checks[index]?.linked ?? [], - appUses: checks[index]?.appUses ?? [], - })), - ), - ); - return yield* Effect.forEach( - checks, - (check, index) => - Effect.result( - Effect.all( - [ - Effect.suspend(() => { - if (Result.isFailure(allowed)) return Effect.fail(allowed.failure); - const passed = allowed.success[index]; - return passed === undefined - ? Effect.fail(new StorageError()) - : check.permitted(passed); - }), - check.usable, - ], - { concurrency: "unbounded", discard: true }, - ), - ), + const [rechecked, usable] = yield* Effect.all( + [ + lifecycle === undefined || states.length === 0 + ? Effect.succeed(undefined) + : lifecycle.appsResolving( + owner, + states.map((state, index) => ({ + id: state.app.id, + accounts: checks[index]?.linked ?? [], + appUses: checks[index]?.appUses ?? [], + })), + ), + options.accountsUsableEach(checks.map(({ selected }) => selected)), + ], { concurrency: "unbounded" }, ); + return checks.map( + (check, index): Result.Result>> => { + const recheck = rechecked?.[index]; + if (recheck !== undefined) { + if (Result.isFailure(recheck)) return Result.fail(recheck.failure); + if (!check.permitted(recheck.success)) return Result.fail(new StorageError()); + } + return usable[index] ?? Result.fail(new StorageError()); + }, + ); }); /** A result another process or isolate kept, if the host keeps results beyond this one. */ const recall = (app: string, id: string) => @@ -303,7 +303,7 @@ export const makeDeclarations = (options: { hostApi: (version: Parameters[1]) => versionHostApi(options.hostApi, version), authorize, - authorizeEach, + authorizeAll, recall, persist, /** diff --git a/packages/sdk/src/implementation/listings.ts b/packages/sdk/src/implementation/listings.ts index 07809e8fbf..a6edf72b2e 100644 --- a/packages/sdk/src/implementation/listings.ts +++ b/packages/sdk/src/implementation/listings.ts @@ -1,5 +1,5 @@ /** Evaluated tool listings, reused across requests through the declaration store. */ -import { Cause, Clock, Effect, Exit, Fiber, Option, Result, Semaphore } from "effect"; +import { Cause, Clock, Effect, Exit, Fiber, Option, Result, Schema, Semaphore } from "effect"; import { defaultToolListingPolicy, durableHeadStartMillis, @@ -20,12 +20,13 @@ import { type ToolRouter, } from "../contracts/tools.ts"; import type { AppAccounts } from "../contracts/apps.ts"; -import type { VersionId } from "../contracts/shared.ts"; +import { OAuthReconnectRequired } from "../contracts/oauth.ts"; +import type { OwnerId, VersionId } from "../contracts/shared.ts"; import type { Declarations } from "./declarations.ts"; import { makeHandoff } from "./handoff.ts"; import { decodeListing, shareListing } from "./listing-json.ts"; import type { makeOAuth } from "./oauth.ts"; -import { resolve, type InvocationSnapshot } from "./tools.ts"; +import { reconnectReported, resolve, type InvocationSnapshot } from "./tools.ts"; import type { ToolSearch } from "./tool-search.ts"; /** Every tool of one app for one invocation state, sorted by name. */ @@ -61,6 +62,17 @@ type Evaluate = ( context: Effect.Success>, ) => Effect.Effect; +/** + * What `searchable` reads for one state before it decides whether kept rows can be served, read + * ahead together with other states' (see `searchablePrefetch`): the evaluation running when its + * rows were read, the rows, and the outcome of the access check that precedes serving them. + */ +export interface SearchableAhead { + readonly running: PendingLoad | undefined; + readonly kept: Effect.Success> | undefined; + readonly access: Result.Result>>; +} + /** How an in-process caller reads a listing. */ interface ListingRead extends ToolListOptions { /** @@ -667,7 +679,41 @@ export const makeListings = (options: { ); if (!(yield* background(work))) cache.renewed(); }); + /** + * The reads `searchable` makes for each state before it decides whether kept rows can be served, + * for many states of one owner at once, in order: each state's running evaluation is read and + * its app's changes followed, as `searchable` does first, then every state's rows in one + * statement beside the access checks of all of them (`authorizeAll`). Undefined for every state + * when no listing may be served kept, as `searchable` then reads nothing either. A failure of + * either read fails the whole call; nothing is decided from it. + */ + const searchablePrefetch = (owner: OwnerId, states: ReadonlyArray) => + Effect.gen(function* () { + if (policy.maxStaleMillis <= 0 || states.length === 0) + return states.map((): SearchableAhead | undefined => undefined); + const reads = yield* Effect.forEach(states, (state) => + Effect.map(options.declarations.key("tools.list", state), (id) => ({ + state, + listing: ToolSearchListing.make(id), + running: cache.pending(id), + })), + ); + yield* Effect.forEach( + new Set(states.map((state) => state.app.id)), + (app) => options.declarations.followChanges(app), + { concurrency: "unbounded", discard: true }, + ); + const [kept, access] = yield* Effect.all( + [search.listings(reads), options.declarations.authorizeAll(owner, states)], + { concurrency: "unbounded" }, + ); + return reads.map(({ running }, index): SearchableAhead | undefined => { + const outcome = access[index]; + return outcome === undefined ? undefined : { running, kept: kept[index], access: outcome }; + }); + }).pipe(Effect.withSpan("sdk.tools.searchable.prefetch")); return { + searchablePrefetch, /** The host API revision this version's invocations are bound to; listings are keyed by it. */ hostApi: options.declarations.hostApi, read: (state: InvocationSnapshot, evaluate: Evaluate, read: ListingRead = {}) => @@ -732,26 +778,48 @@ export const makeListings = (options: { * later one. A write that fails is `ToolSearchUnavailable`: the rows that are kept, if any, are * not this listing's. */ - searchable: (state: InvocationSnapshot, evaluate: Evaluate, read: ListingRead = {}) => + searchable: ( + state: InvocationSnapshot, + evaluate: Evaluate, + read: ListingRead = {}, + /** These reads made ahead for the state by `searchablePrefetch`, in place of its own. */ + ahead?: SearchableAhead, + ) => Effect.gen(function* () { const id = yield* options.declarations.key("tools.list", state); const listing = ToolSearchListing.make(id); const now = yield* Clock.currentTimeMillis; // Read before the rows: an evaluation that ends while they are read wrote them first. - const running = cache.pending(id); + const running = ahead === undefined ? cache.pending(id) : ahead.running; // Rows from before the app's cached data last changed are not served; see `followChanges`. - yield* options.declarations.followChanges(state.app.id); + if (ahead === undefined) yield* options.declarations.followChanges(state.app.id); // The checks kept rows are served after run beside their read; only kept rows wait on them. const [kept, access] = policy.maxStaleMillis <= 0 ? ([undefined, undefined] as const) - : yield* Effect.all( - [ - search.listing(state, listing), - Effect.result(options.declarations.authorize(state)), - ], - { concurrency: "unbounded" }, - ); + : ahead !== undefined + ? ([ahead.kept, ahead.access] as const) + : yield* Effect.all( + [ + search.listing(state, listing), + Effect.result(options.declarations.authorize(state)), + ], + { concurrency: "unbounded" }, + ); + yield* Effect.annotateCurrentSpan("executor.search.read_ahead", ahead !== undefined); + // The access check fails for an account that must reconnect exactly when resolving it + // would (see `usableAll`): neither kept rows nor an evaluation could be served, so fail + // now rather than evaluate the app only to fail the same way, naming the reconnect as + // resolving would. Only a read ahead decides this: its check rechecks the actor's app + // access before any grant, as resolving does, while the live check runs both together and + // could report a reconnect to an actor who lost the app. + if ( + ahead !== undefined && + access !== undefined && + Result.isFailure(access) && + Schema.is(OAuthReconnectRequired)(access.failure) + ) + return yield* Effect.fail(reconnectReported(state)(access.failure)); // Rows missing, stale or removed are filled from an evaluation through the shared gate, // which keeps no decoded listing: rows are what later searches read. Readers of the same // listing share one evaluation, which the load bound stops like any other once nobody @@ -840,7 +908,10 @@ export const makeListings = (options: { * at once. A state whose rows are missing, stale, superseded or newer in memory, or whose * checks fail, is `undefined`: its `searchable` read writes them or reports why. Never evaluates. */ - searchableKept: (states: ReadonlyArray) => + searchableKept: ( + states: ReadonlyArray, + ahead?: ReadonlyArray, + ) => Effect.gen(function* () { if (policy.maxStaleMillis <= 0 || states.length === 0) return states.map(() => undefined); const ids = yield* Effect.forEach(states, (state) => @@ -850,7 +921,8 @@ export const makeListings = (options: { state, listing: ToolSearchListing.make(ids[index] ?? ""), })); - const kept = yield* search.listings(wanted); + const kept = + ahead === undefined ? yield* search.listings(wanted) : ahead.map((entry) => entry.kept); const now = yield* Clock.currentTimeMillis; const current: Array<{ readonly index: number; @@ -858,7 +930,7 @@ export const makeListings = (options: { readonly routers: ReadonlyArray; }> = []; for (const [index, { state, listing }] of wanted.entries()) { - const found = kept.get(listing); + const found = kept[index]; if (found === undefined || found.superseded !== null) continue; if (now - found.at >= policy.maxStaleMillis || cache.outdated(state.app.id, found.at)) continue; @@ -867,9 +939,16 @@ export const makeListings = (options: { continue; current.push({ index, state, routers: found.routers }); } - const checked = yield* options.declarations.authorizeEach( - current.map(({ state }) => state), - ); + const owner = states[0]?.app.owner; + const checked = + ahead !== undefined + ? current.map(({ index }) => ahead[index]?.access) + : owner === undefined + ? [] + : yield* options.declarations.authorizeAll( + owner, + current.map(({ state }) => state), + ); const served: Array< | { readonly listing: ToolSearchListing; readonly routers: ReadonlyArray } | undefined diff --git a/packages/sdk/src/implementation/native-engine-failure.ts b/packages/sdk/src/implementation/native-engine-failure.ts new file mode 100644 index 0000000000..7527ec4d35 --- /dev/null +++ b/packages/sdk/src/implementation/native-engine-failure.ts @@ -0,0 +1,18 @@ +/** What a failed call to a native workflow engine becomes in a run, on every host. */ +import { + decodeWorkflowFailure, + engineCallFailure, + engineErrorCode, + hasWorkflowFailureMarker, +} from "../contracts/workflow-errors.ts"; +import { errorClassOf } from "./runtime-hop.ts"; + +/** + * A failure carrying our marker is ours, relayed by the engine, and passes on as itself. Any other + * is the platform's answer: its standard class name and a code read from its message + * (`engineErrorCode`), never the message. + */ +export const nativeEngineFailure = (error: unknown) => + hasWorkflowFailureMarker(error) + ? decodeWorkflowFailure(error) + : engineCallFailure("platform", errorClassOf(error), engineErrorCode(error)); diff --git a/packages/sdk/src/implementation/oauth.ts b/packages/sdk/src/implementation/oauth.ts index 360724f5e1..62df2b085e 100644 --- a/packages/sdk/src/implementation/oauth.ts +++ b/packages/sdk/src/implementation/oauth.ts @@ -51,6 +51,8 @@ import { OAuthFirstPartyAttempt, OAuthServer, allowedScopes, + grantableScopes, + type OperatorScopeRefusal, type FirstPartyOAuthClient, type FirstPartyOAuthClientId, isFirstPartyAttempt, @@ -658,12 +660,15 @@ export const makeOAuth = ( .flatMap((part) => (separator === undefined ? [part] : part.split(separator))) .filter((granted) => granted !== ""); }; - /** Whether a token response for the operator's client reports only scopes the client allows. */ + /** + * Whether a token response for the operator's client reports only scopes a grant of it may hold: + * those it allows, and its held scopes, which grants it issued earlier keep through renewal. + */ const grantAllowed = (id: FirstPartyOAuthClientId, scope: string | undefined) => { const client = firstPartyClients.get(id); if (client === undefined) return false; - const allowed = allowedScopes(client); - return (grantedScopes(client, scope) ?? []).every((granted) => allowed.includes(granted)); + const grantable = grantableScopes(client); + return (grantedScopes(client, scope) ?? []).every((granted) => grantable.includes(granted)); }; /** * The operator's client and authorization server as configured now, or undefined when the @@ -758,6 +763,10 @@ export const makeOAuth = ( * Scopes discovered only in the resource's advertised list are what it offers, not what it * requires, so they are narrowed to `allowedScopes`; the client is offered when any remain. * An `unlisted` client is offered only to reconnect an account whose grant it issued, `reconnecting`. + * A client whose endpoints match but which may not ask for every requested scope is not offered. + * When one of those is a held scope, `refused` names every scope it may not ask for, so a + * sign-in with no other automatic client can say why rather than send the person to a service + * that would refuse the client. */ const firstPartyClient = ( discovered: { @@ -767,9 +776,16 @@ export const makeOAuth = ( readonly scopesAdvertised?: true | undefined; }, reconnecting?: FirstPartyOAuthClientId, - ) => { - if (discovered.grant !== "authorization_code") return undefined; + ): { + readonly offered?: { + readonly client: FirstPartyOAuthClient; + readonly scopes: readonly string[]; + }; + readonly refused?: OperatorScopeRefusal; + } => { + if (discovered.grant !== "authorization_code") return {}; const { server } = discovered; + let refused: OperatorScopeRefusal | undefined; const same = (declared: string | undefined, configured: string) => declared !== undefined && new URL(declared).href === new URL(configured).href; for (const client of firstPartyClients.values()) { @@ -783,13 +799,27 @@ export const makeOAuth = ( if (discovered.scopesAdvertised === true && discovered.scopes.length > 0) { const scopes = discovered.scopes.filter((scope) => allowed.includes(scope)); if (scopes.length === 0) continue; - return { client, scopes }; + return { offered: { client, scopes } }; } const scopes = discovered.scopes.length === 0 ? client.defaultScopes : discovered.scopes; - if (!scopes.every((scope) => allowed.includes(scope))) continue; - return { client, scopes }; + const [first, ...others] = scopes.filter((scope) => !allowed.includes(scope)); + if (first === undefined) return { offered: { client, scopes } }; + // Only a held scope, which the client once asked for, makes the client's refusal the + // answer; any other scope outside its menu leaves it simply not offered. + const held = (client.heldScopes ?? []).map((entry) => entry.scope); + if (![first, ...others].some((scope) => held.includes(scope))) continue; + refused ??= { + client: client.label, + scopes: [first, ...others], + instead: (client.heldScopes ?? []).flatMap((held) => { + const [replacement, ...rest] = held.instead; + return [first, ...others].includes(held.scope) && replacement !== undefined + ? [{ scope: held.scope, scopes: [replacement, ...rest] }] + : []; + }), + }; } - return undefined; + return refused === undefined ? {} : { refused }; }; const encrypt = (identity: AccountId | OAuthAttemptId | OAuthClientId, value: unknown) => decode(JsonObject, value).pipe( @@ -1186,11 +1216,17 @@ export const makeOAuth = ( // A registered client the caller is refused still counts, as a start finds it too. const existing = yield* existingAutomatic(input.owner, resolved); // The operator's own client, after the owner's registered clients and before CIMD and DCR. - const firstParty = existing === undefined ? firstPartyClient(discovered) : undefined; + const operating = existing === undefined ? firstPartyClient(discovered) : {}; + const firstParty = operating.offered; const mode = existing === undefined && firstParty === undefined && registration === "manual" ? ("client-required" as const) : ("automatic" as const); + // Said only when no automatic client signs in, as a start then fails with it. + const refusal = + mode === "client-required" && operating.refused !== undefined + ? { operatorScopeRefusal: operating.refused } + : {}; const resource = firstParty !== undefined ? sentResource(firstParty.client.server.resource, "operator") @@ -1221,6 +1257,7 @@ export const makeOAuth = ( : []; return { mode, + ...refusal, scopes: firstParty?.scopes ?? discovered.scopes, grant: "authorization_code", issuer: HttpUrl.make(resolved.issuer), @@ -1464,7 +1501,8 @@ export const makeOAuth = ( kind: "stored" as const, ...(yield* authorizedClientUse(existing.row, existing.secret, input, resolved)), }; - const firstParty = firstPartyClient(discovered, reconnecting); + const operating = firstPartyClient(discovered, reconnecting); + const firstParty = operating.offered; if (firstParty !== undefined) return { kind: "firstParty" as const, firstParty }; if (registration === "client_id_metadata_document") { const url = @@ -1479,7 +1517,13 @@ export const makeOAuth = ( return { kind: "entered" as const, client, callback: clientCallback(client, redirect) }; } if (discovered.grant !== "authorization_code" || registration !== "dynamic") - return yield* new OAuthClientUnavailable(input); + return yield* operating.refused === undefined + ? new OAuthClientUnavailable(input) + : new OAuthSetupFailed({ + reason: "operator_scope_refused", + operatorScopes: operating.refused, + ...(redirect === undefined ? {} : { callbackUrl: HttpUrl.make(redirect.href) }), + }); if (redirect === undefined) return yield* new OAuthSetupFailed({ reason: "invalid_redirect" }); const client = yield* protocol @@ -1517,6 +1561,15 @@ export const makeOAuth = ( return { kind: "stored" as const, ...(yield* useStoredClient(used, input, resolved)) }; }).pipe(Effect.withSpan("oauth.clients.automatic")); + /** + * Whether an automatic choice found no client to sign in with: none fits the service, or the + * operator's client fits it but may not ask for a held scope the sign-in requests. A reconnect then keeps the + * client its account signed in with in either case. + */ + const noAutomaticClient = (error: unknown): error is OAuthClientUnavailable | OAuthSetupFailed => + Schema.is(OAuthClientUnavailable)(error) || + (Schema.is(OAuthSetupFailed)(error) && error.reason === "operator_scope_refused"); + /** * The client a sign-in uses. Without a choice, a reconnect keeps the stored client its account * signed in with, while that client exists and the product admits the caller to it; otherwise @@ -1595,13 +1648,13 @@ export const makeOAuth = ( // fall back to the copy the grant holds of the same client. if (kept.source === "manual") return yield* keep; return yield* automaticChoice(input, resolved, reconnecting).pipe( - Effect.catchTag("OAuthClientUnavailable", () => keep), + Effect.catchIf(noAutomaticClient, () => keep), ); } // No stored client holds or held the grant's copy, so reusing it uses no stored client's // credentials. Completion checks again, in case one is stored meanwhile. return yield* automaticChoice(input, resolved, reconnecting).pipe( - Effect.catchTag("OAuthClientUnavailable", (error) => + Effect.catchIf(noAutomaticClient, (error) => reusable === undefined ? Effect.fail(error) : Effect.succeed({ @@ -2833,38 +2886,32 @@ export const makeOAuth = ( ), ); + /** One account a usability check covers, with the provider definition it signs in with. */ + type UsableCheck = { + readonly account: StoredAccount; + readonly provider: ProviderDefinition; + }; + /** The distinct accounts among `checks` that sign in with something other than secrets. */ + const grantedIds = (checks: ReadonlyArray) => [ + ...new Set( + checks + .filter(({ account, provider }) => provider.auth[account.method]?.type !== "secrets") + .map(({ account }) => account.id), + ), + ]; + /** The grant rows of these accounts, in one query; none when it names none. */ + const usableGrants = (ids: readonly AccountId[]) => + (ids.length === 0 + ? Effect.succeed([]) + : query(() => db.findMany("oauthGrants", { where: (b) => b("id", "in", ids) })) + ).pipe(Effect.map((rows) => new Map(rows.map((row) => [row.id, row] as const)))); /** - * Fail as `resolve` would when a stored grant can no longer release credentials, without - * renewing it or releasing anything. A grant that renewal would replace still passes; its - * outcome is known only once a live resolve renews it. A grant that needs reconnecting is - * recorded as the span's outcome, as `resolve` records it, and fails only after the span. - * - * Each of `checks` is checked in order, and the first that fails decides the failure. Their - * grants are read in one query and one span covers the whole check, so checking a selection - * costs the same however many accounts it names. + * The check of {@link usableAll} over grant rows already read: the first of `checks` whose grant + * can no longer release credentials decides the outcome, as one `oauth.usable` span records it. */ - const usableAll = ( - checks: ReadonlyArray<{ - readonly account: StoredAccount; - readonly provider: ProviderDefinition; - }>, - ) => + const usableWith = (checks: ReadonlyArray, grants: ReadonlyMap) => Effect.gen(function* () { yield* Effect.annotateCurrentSpan("oauth.usable.accounts", checks.length); - const granted = [ - ...new Set( - checks - .filter(({ account, provider }) => provider.auth[account.method]?.type !== "secrets") - .map(({ account }) => account.id), - ), - ]; - const rows = - granted.length === 0 - ? [] - : yield* query(() => - db.findMany("oauthGrants", { where: (b) => b("id", "in", granted) }), - ); - const grants = new Map(rows.map((row) => [row.id, row] as const)); for (const { account, provider } of checks) { // The provider of the account checked last: on failure, the one that failed. yield* Effect.annotateCurrentSpan("oauth.provider.id", account.provider); @@ -2894,6 +2941,27 @@ export const makeOAuth = ( Schema.is(OAuthReconnectRequired)(checked) ? Effect.fail(checked) : Effect.void, ), ); + /** + * Fail as `resolve` would when a stored grant can no longer release credentials, without + * renewing it or releasing anything. A grant that renewal would replace still passes; its + * outcome is known only once a live resolve renews it. A grant that needs reconnecting is + * recorded as the span's outcome, as `resolve` records it, and fails only after the span. + * + * Each of `checks` is checked in order, and the first that fails decides the failure. Their + * grants are read in one query, so checking a selection costs the same however many accounts it + * names. + */ + const usableAll = (checks: ReadonlyArray) => + Effect.flatMap(usableGrants(grantedIds(checks)), (grants) => usableWith(checks, grants)); + /** + * {@link usableAll} for several selections at once, such as every app a search reads: the grants + * of all of them are read in one query, then each selection is checked on its own and its outcome + * returned in order. Only that read fails the whole call. + */ + const usableEach = (selections: ReadonlyArray>) => + Effect.flatMap(usableGrants(grantedIds(selections.flat())), (grants) => + Effect.forEach(selections, (checks) => Effect.result(usableWith(checks, grants))), + ); /** The accounts the product still authorizes; undefined when it has no lifecycle. */ const authorized = (accounts: readonly StoredAccount[]) => @@ -3227,6 +3295,7 @@ export const makeOAuth = ( resolveSelected, renewRejected, usableAll, + usableEach, revokeRemoved, }; }; diff --git a/packages/sdk/src/implementation/remote-app-runner.ts b/packages/sdk/src/implementation/remote-app-runner.ts index 4738e628d9..98fee81f65 100644 --- a/packages/sdk/src/implementation/remote-app-runner.ts +++ b/packages/sdk/src/implementation/remote-app-runner.ts @@ -36,6 +36,8 @@ import { } from "./app-runner.ts"; import { invocationBuildLoader } from "./app-runtime.ts"; import { asciiJson } from "./ascii-json.ts"; +import { engineCallFailure } from "../contracts/workflow-errors.ts"; +import { errorClassOf } from "./runtime-hop.ts"; type Callback = (input: unknown) => Promise; @@ -150,6 +152,25 @@ const settle = (call: () => Promise) => ), ); +/** + * A sleep sent to the host that runs the engine. The host's own answer passes through; a call the + * host never answered, or answered unreadably, is an engine call failure at the host hop. + */ +const settleSleep = (call: () => Promise) => + Effect.tryPromise({ + try: async () => await call(), + catch: (error) => engineCallFailure("host", errorClassOf(error), "unanswered"), + }).pipe( + Effect.flatMap((reply) => + Schema.decodeUnknownEffect(WorkflowRpcResult)(reply).pipe( + Effect.mapError(() => engineCallFailure("host", "other", "unreadable")), + ), + ), + Effect.flatMap((result) => + result.ok ? Effect.succeed(result.value) : Effect.fail(result.error), + ), + ); + /** The caller's side: callbacks over this invocation's capabilities, released with `signal`. */ export const remoteCapabilities = (capabilities: AppCapabilities, signal: AbortSignal) => Effect.gen(function* () { @@ -223,11 +244,11 @@ const localCapabilities = (remote: RemoteCapabilities, load: () => Promise - settle(() => + settleSleep(() => workflow.sleep(name, Schema.encodeSync(WorkflowDuration)(duration)), ).pipe(Effect.asVoid), sleepUntil: (name, timestamp) => - settle(() => workflow.sleepUntil(name, timestamp)).pipe(Effect.asVoid), + settleSleep(() => workflow.sleepUntil(name, timestamp)).pipe(Effect.asVoid), }, resolve: () => settle(() => workflow.context()).pipe( diff --git a/packages/sdk/src/implementation/runtime-connect.ts b/packages/sdk/src/implementation/runtime-connect.ts new file mode 100644 index 0000000000..d34190b0ce --- /dev/null +++ b/packages/sdk/src/implementation/runtime-connect.ts @@ -0,0 +1,221 @@ +/** + * One repeat of an exchange with the local app runtime (workerd) whose connection never formed, so + * nothing of it left the host. On Windows, Bun (and libuv) connect to a loopback address with one + * SYN and no retransmission, so a SYN-ACK slower than about a quarter of a second fails the connect + * with `ETIMEDOUT` while the runtime is healthy. Repeating such an exchange cannot repeat any work: + * none was sent. Anything that may have reached the runtime is never repeated. + * + * Each repeat re-runs the exchange from its own first statement, so whatever the exchange checks + * before it connects is checked again before the second connect. + * + * What the span records: the plain hop attributes (`executor.runtime.rpc.*`, `.http.*`) describe + * the attempt that decided the outcome. A repeated exchange adds `connect_attempts` (2), + * `connect_outcome` (`opened`, `answered` or `failed`) and the first attempt's facts under + * `first_attempt.` (`error_code`, `failed_ms`). The first attempt's own hop attributes are held + * while it runs, and reach the span unchanged only when it is not repeated. + */ +import { Cause, Data, Deferred, Effect, Exit, Predicate, type Scope } from "effect"; +import { RuntimeProtocolFailed } from "../contracts/runtime.ts"; +import { + annotateChecked, + codeOf, + HopRecording, + platformError, + type HopAttributes, +} from "./runtime-hop.ts"; + +/** An RPC session whose WebSocket closed or failed before it opened: nothing was sent on it. */ +export class SocketNeverOpened extends Data.TaggedError("SocketNeverOpened")<{ + readonly code: string | undefined; + readonly failedMs: number; +}> {} + +/** The minimal view of a `ws` client socket that tells whether it opened. */ +interface OpeningSocket { + once(event: "open" | "close", listener: () => void): unknown; + once(event: "error", listener: (error: unknown) => void): unknown; +} + +/** + * Whether one RPC session's socket opens. `watch` is called with the socket as soon as it is made, + * before anything can be sent on it; `opened` succeeds once it opens, or fails with + * `SocketNeverOpened` when it closes or errors first. `neverOpened` is that fact, read from the + * socket itself: the repeat is decided from it, not from whichever error the session ends with. + */ +export interface Opening { + readonly watch: (socket: OpeningSocket) => void; + readonly opened: Effect.Effect; + readonly neverOpened: () => SocketNeverOpened | undefined; +} + +const opening = Effect.gen(function* () { + const outcome = yield* Deferred.make(); + let never: SocketNeverOpened | undefined; + return { + watch: (socket: OpeningSocket) => { + const created = performance.now(); + let opened = false; + let code: string | undefined; + socket.once("open", () => { + opened = true; + Deferred.doneUnsafe(outcome, Effect.void); + }); + socket.once("error", (error) => { + code = Predicate.hasProperty(error, "code") ? codeOf(error.code) : undefined; + }); + socket.once("close", () => { + if (opened) return; + never = new SocketNeverOpened({ code, failedMs: Math.round(performance.now() - created) }); + Deferred.doneUnsafe(outcome, Effect.fail(never)); + }); + }, + opened: Deferred.await(outcome), + neverOpened: () => never, + } satisfies Opening; +}); + +const interrupted = (exit: Exit.Exit) => + Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause); + +/** Run `attempt` with its hop attributes held rather than written, returning both. */ +const held = (attempt: Effect.Effect) => + Effect.gen(function* () { + const attributes: Record = {}; + const exit = yield* Effect.exit( + attempt.pipe( + Effect.provideService(HopRecording, (recorded: HopAttributes) => { + Object.assign(attributes, recorded); + }), + ), + ); + return { exit, attributes }; + }); + +/** The span attributes of a repeated connect: the first attempt's failure and how the repeat ended. */ +const repeated = ( + prefix: "rpc" | "http", + first: { readonly code: string | undefined; readonly failedMs: number }, + outcome: "opened" | "answered" | "failed", +) => + annotateChecked({ + [`executor.runtime.${prefix}.connect_attempts`]: 2, + [`executor.runtime.${prefix}.connect_outcome`]: outcome, + [`executor.runtime.${prefix}.first_attempt.error_code`]: first.code ?? "none", + [`executor.runtime.${prefix}.first_attempt.failed_ms`]: first.failedMs, + }); + +/** + * The failure of an RPC exchange whose socket never opened, twice: the runtime was not reached. + * Its reason is `disconnected`, the value clients already know for a lost connection; `notSent` + * says nothing was sent, and older clients drop it. + */ +const unreachable = Effect.annotateCurrentSpan({ + "executor.runtime.failure": "unreachable", + "executor.runtime.cause": "Executor could not connect to the app runtime", +}).pipe( + Effect.andThen(Effect.fail(new RuntimeProtocolFailed({ reason: "disconnected", notSent: true }))), +); + +/** + * Run an RPC exchange, `attempt`, which waits for its socket to open before it sends anything. When + * its socket never opened, run it once more, from its start, at once. When that socket never opens + * either, fail as `unreachable` with `notSent`. Whether to repeat is read from the socket's own + * state (`Opening.neverOpened`), so it holds whatever error the attempt ends with. A failure after + * the socket opened is the exchange's own and is never repeated. + */ +export const repeatUnopened = ( + attempt: (opening: Opening) => Effect.Effect, + options: { + /** + * Runs before every connect, the repeat's included: the place for a check that must hold + * before the host connects at all, such as whether it has given the runtime up. + */ + readonly beforeEachConnect?: Effect.Effect; + } = {}, +): Effect.Effect | E2 | RuntimeProtocolFailed, R | R2> => + Effect.gen(function* () { + const before = options.beforeEachConnect ?? Effect.void; + yield* before; + const firstOpening = yield* opening; + const first = yield* held(attempt(firstOpening)); + const never = firstOpening.neverOpened(); + if (never === undefined || interrupted(first.exit)) { + yield* Effect.annotateCurrentSpan(first.attributes); + return yield* first.exit; + } + yield* before; + const secondOpening = yield* opening; + const second = yield* Effect.exit(attempt(secondOpening)); + const unopened = secondOpening.neverOpened() !== undefined; + yield* repeated("rpc", never, unopened ? "failed" : "opened"); + if (unopened && Exit.isFailure(second) && !interrupted(second)) { + // A reason the session chose for itself, other than a lost connection, is kept; only the + // fact that nothing was sent is added. + const own = Exit.findErrorOption(second); + return yield* own._tag === "Some" && + own.value instanceof RuntimeProtocolFailed && + own.value.reason !== undefined && + own.value.reason !== "disconnected" + ? Effect.fail(new RuntimeProtocolFailed({ reason: own.value.reason, notSent: true })) + : unreachable; + } + return yield* second; + // An attempt fails with `SocketNeverOpened` only when its socket never opened, and then the + // second attempt's failure is replaced above: it never reaches the caller. + }) as Effect.Effect | E2 | RuntimeProtocolFailed, R | R2>; + +/** + * Whether an HTTP exchange's failure is provably its connect failing, so that no byte of the request + * was written. Only facts the platform sets on its error are read, never its text: + * + * - Node (undici over `node:net`): the socket error's `syscall` is `connect`, or undici's own connect + * time-out (`UND_ERR_CONNECT_TIMEOUT`). Both come before the request is written. + * - Bun: `fetch` reports `ECONNREFUSED` only for a connect that failed (`on_connect_error` in its + * HTTP client; a closed connection is `ECONNRESET`). Its connect time-out on Windows is + * `ETIMEDOUT`, but so is its socket idle time-out after the request was written, and nothing on + * the error tells them apart, so a Bun `ETIMEDOUT` is not repeated. + */ +export const failedAtConnect = (error: unknown): boolean => { + if (!Predicate.hasProperty(error, "_tag") || error._tag !== "HttpClientError") return false; + if (!Predicate.hasProperty(error, "reason") || !Predicate.hasProperty(error.reason, "_tag")) + return false; + if (error.reason._tag !== "TransportError") return false; + const platform = platformError(error); + if (!Predicate.isObject(platform)) return false; + const syscall = Predicate.hasProperty(platform, "syscall") ? platform.syscall : undefined; + const code = Predicate.hasProperty(platform, "code") ? platform.code : undefined; + return syscall === "connect" || code === "ECONNREFUSED" || code === "UND_ERR_CONNECT_TIMEOUT"; +}; + +/** + * Run an HTTP exchange and, when it fails at connect (`failedAtConnect`), run it once more from its + * start, at once. Any other failure, and the repeat's own, is the exchange's. + */ +export const repeatAtConnect = ( + exchange: Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + const started = performance.now(); + const first = yield* held(exchange); + const error = Exit.isFailure(first.exit) ? Exit.findErrorOption(first.exit) : undefined; + if (error === undefined || error._tag === "None" || !failedAtConnect(error.value)) { + yield* Effect.annotateCurrentSpan(first.attributes); + return yield* first.exit; + } + const platform = platformError(error.value); + const firstFacts = { + code: Predicate.hasProperty(platform, "code") ? codeOf(platform.code) : undefined, + failedMs: Math.round(performance.now() - started), + }; + const second = yield* Effect.exit(exchange); + yield* repeated("http", firstFacts, Exit.isSuccess(second) ? "answered" : "failed"); + return yield* second; + }); + +/** + * An HTTP exchange in its own scope, repeated once from its start when it fails at connect + * (`repeatAtConnect`). Each attempt has a scope of its own. + */ +export const repeatingAtConnect = ( + exchange: Effect.Effect, +): Effect.Effect> => repeatAtConnect(Effect.scoped(exchange)); diff --git a/packages/sdk/src/implementation/runtime-hop.ts b/packages/sdk/src/implementation/runtime-hop.ts new file mode 100644 index 0000000000..16204af1b0 --- /dev/null +++ b/packages/sdk/src/implementation/runtime-hop.ts @@ -0,0 +1,404 @@ +/** + * How far an exchange with the local app runtime (workerd) got before it failed, recorded on the + * current span so a failed call can be told apart: a reply lost as its connection ended, a runtime + * that never answered, a session that never opened or closed before or after it carried anything, + * and a runtime that was just replaced. Only measurement: nothing here sends, waits, retries or + * changes an outcome, and nothing here can change a failure. Every attribute is built inside a + * guard; if building them throws, the span records one fixed marker and the failure passes on as + * the same object. + * + * Only shapes are recorded, and every value passes one check before it reaches the span: a fixed + * route and method, error tags, a platform error's standard class name, its `code` and `syscall` + * when they have the shape a platform gives them, its `errno`, a WebSocket close code, whether a + * close reason was given, message counts, times and the runtime's restart count. No message text, + * URL beyond the route's fixed path, header or payload is recorded, so none of an app's or a + * customer's data can reach telemetry through these attributes. + */ +import { Cause, Context, Effect, Predicate } from "effect"; + +/** Span attribute values a recording carries: shapes only. */ +export type HopAttributes = Readonly>; + +/** + * The host's view of its runtime at the moment an exchange fails, recorded beside the exchange's + * own attributes. `probe` is a supervisor's recent probe events: only keys under + * `executor.runtime.probe.` with a number, a boolean or a short word as their value are kept. + */ +export interface RuntimeView { + /** How many times the runtime was replaced, and when the last replacement finished. */ + readonly restarts: () => { readonly count: number; readonly lastAt: number | undefined }; + readonly probe: () => unknown; +} +const noRuntimeView: RuntimeView = { + restarts: () => ({ count: 0, lastAt: undefined }), + probe: () => ({}), +}; + +/** What the span records when building the attributes threw: nothing else. */ +const recordingFailed: HopAttributes = { "executor.runtime.recording": "failed" }; + +/** Error class names the platform and the language define. Any other name is recorded as `other`. */ +const standardErrorNames: ReadonlySet = new Set([ + "Error", + "TypeError", + "RangeError", + "SyntaxError", + "AggregateError", + "AbortError", + "TimeoutError", + "NetworkError", + "SocketError", + "ConnectTimeoutError", + "HeadersTimeoutError", + "BodyTimeoutError", + "RequestAbortedError", + "DOMException", +]); + +/** + * A platform error code: Node's and libuv's upper-case codes (`ECONNRESET`, `UND_ERR_SOCKET`, + * `ERR_STREAM_PREMATURE_CLOSE`), or Bun's words (`ConnectionRefused`, `FailedToOpenSocket`). + */ +const codeForm = /^(?:[A-Z][A-Z0-9_]{1,39}|(?:[A-Z][a-z]{1,15}){1,5})$/; +/** A system call: one short lower-case word (`connect`, `read`, `write`, `getaddrinfo`). */ +const syscallForm = /^[a-z]{2,16}$/; +/** An error's `_tag`: one word in upper camel case (`HttpClientError`, `TransportError`). */ +const tagForm = /^[A-Z][A-Za-z0-9]{1,39}$/; +/** + * A value of the recording's own: a word of up to 40 characters, as long as the longest code or tag + * the forms above accept (Node's and undici's codes run to about 30), or the fixed route. + */ +const valueForm = /^(?:[A-Za-z][A-Za-z0-9_-]{0,39}|\/[a-z]{1,23})$/; +/** A value of a supervisor's probe: a word of up to 24 characters. */ +const probeValueForm = /^[A-Za-z][A-Za-z0-9_-]{0,23}$/; +/** Every key the recording writes. */ +const keyForm = /^executor\.runtime\.[a-z_]+(?:\.[a-z_]+){0,2}$/; +/** The only keys a supervisor's probe context may write. */ +const probeKeyForm = /^executor\.runtime\.probe\.[a-z_]+(?:\.[a-z_]+)?$/; + +/** `value` if it is a bounded finite number, a boolean or a string `form` accepts. */ +const shape = (value: unknown, form: RegExp): string | number | boolean | undefined => + typeof value === "boolean" + ? value + : typeof value === "number" + ? Number.isFinite(value) && Math.abs(value) < 1e9 + ? value + : undefined + : typeof value === "string" && form.test(value) + ? value + : undefined; + +/** + * Only well-formed keys with well-formed values: the check every recorded attribute passes. A + * string of the recording's own that fails it is recorded as `other`, so a reader knows a value was + * there; a probe's is dropped. Numbers out of range and other values are dropped. + */ +const checked = (attributes: Record, probe = false): HopAttributes => { + const kept: Record = {}; + for (const [key, value] of Object.entries(attributes)) { + if (!(probe ? probeKeyForm : keyForm).test(key)) continue; + const checkedValue = shape(value, probe ? probeValueForm : valueForm); + if (checkedValue !== undefined) kept[key] = checkedValue; + else if (typeof value === "string" && !probe) kept[key] = "other"; + } + return kept; +}; + +/** The recording's attributes, or the fixed marker if building them threw. */ +const guarded = (build: () => HopAttributes): HopAttributes => { + try { + return build(); + } catch { + return recordingFailed; + } +}; +/** + * Where a hop's attributes go instead of the current span, while an attempt that may be repeated + * runs (`runtime-connect.ts`). Unset, they go on the span. Attributes about an exchange's attempt + * belong in `annotateChecked` (or the hops' own recording), which honours it: written to the span + * directly, a first attempt's attributes would stay on a span the repeat rescued. + */ +export const HopRecording = Context.Reference<((attributes: HopAttributes) => void) | undefined>( + "executor/HopRecording", + { defaultValue: () => undefined }, +); +const annotate = (build: () => HopAttributes) => + Effect.flatMap(HopRecording, (hold) => + Effect.suspend(() => + hold === undefined + ? Effect.annotateCurrentSpan(guarded(build)) + : Effect.sync(() => hold(guarded(build))), + ), + ); + +/** Attributes of the recording's own, checked and guarded, on the current span (`runtime-connect.ts`). */ +export const annotateChecked = (attributes: Record) => + annotate(() => checked(attributes)); +/** An error's standard class name, or `other` for any name an app or library chose. */ +export const errorClassOf = (error: unknown): string => + error instanceof Error && standardErrorNames.has(error.name) ? error.name : "other"; +/** A platform error code in one of a platform's forms, otherwise `other`. */ +export const codeOf = (value: unknown): string => { + const code = shape(value, codeForm); + return typeof code === "string" ? code : "other"; +}; + +/** The runtime's restarts and a supervisor's probe events, each checked; a throwing probe is marked. */ +const hostAttributes = (view: RuntimeView, now: number): HopAttributes => { + const { count, lastAt } = view.restarts(); + let probe: HopAttributes; + try { + const value = view.probe(); + probe = Predicate.isObject(value) ? checked(value as Record, true) : {}; + } catch { + probe = { "executor.runtime.probe.recording": "failed" }; + } + return { + ...probe, + ...checked({ + "executor.runtime.host.restarts": count, + "executor.runtime.host.restarted_ms": + lastAt === undefined ? undefined : Math.round(now - lastAt), + }), + }; +}; + +const tagOf = (value: unknown) => { + const tag = Predicate.hasProperty(value, "_tag") ? value._tag : undefined; + return typeof tag === "string" ? (tagForm.test(tag) ? tag : "other") : undefined; +}; + +/** + * The innermost platform error under an Effect HTTP client error: its reason's cause, followed + * through `cause` links (an undici `TypeError: fetch failed` carries the socket's error as its cause). + */ +export const platformError = (error: unknown): unknown => { + let current: unknown = + Predicate.hasProperty(error, "reason") && Predicate.hasProperty(error.reason, "cause") + ? error.reason.cause + : error; + for (let depth = 0; depth < 4; depth++) { + if (!Predicate.hasProperty(current, "cause") || current.cause === undefined) break; + if (Predicate.hasProperty(current, "code") && typeof current.code === "string") break; + current = current.cause; + } + return current; +}; + +/** A platform error's shape: its standard class name, `code`, `errno` and `syscall`. */ +const platformShape = (prefix: string, error: unknown): Record => { + if (!(error instanceof Error)) return {}; + const code = Predicate.hasProperty(error, "code") ? error.code : undefined; + const errno = Predicate.hasProperty(error, "errno") ? error.errno : undefined; + const syscall = Predicate.hasProperty(error, "syscall") ? error.syscall : undefined; + return { + [`${prefix}.error_name`]: standardErrorNames.has(error.name) ? error.name : "other", + ...(code === undefined ? {} : { [`${prefix}.code`]: shape(code, codeForm) ?? "other" }), + ...(typeof errno === "number" && Number.isInteger(errno) ? { [`${prefix}.errno`]: errno } : {}), + ...(syscall === undefined + ? {} + : { [`${prefix}.syscall`]: shape(syscall, syscallForm) ?? "other" }), + }; +}; + +/** The stage an HTTP exchange with the runtime failed at. */ +type HttpStage = "request" | "status" | "response-body"; + +/** A minimal view of a `ws` client socket: the events and method the tracker reads. */ +interface TrackedSocket { + on(event: "open" | "message", listener: () => void): unknown; + on(event: "close", listener: (code: number, reason: Buffer) => void): unknown; + on(event: "error", listener: (error: Error) => void): unknown; + send: (...args: never[]) => unknown; +} + +/** What one RPC session's socket did, as far as its shape goes. */ +export interface SocketRecord { + /** Whether the socket has closed or reported an error: a transport failure. */ + readonly ended: () => boolean; + /** The span attributes describing the socket, measured from its creation to now. */ + readonly attributes: () => Record; +} + +/** Whether a failure is the host's own "the connection to the runtime was lost". */ +const lostConnection = (cause: Cause.Cause) => + cause.reasons.some( + (reason) => + Cause.isFailReason(reason) && + Predicate.hasProperty(reason.error, "_tag") && + reason.error._tag === "RuntimeProtocolFailed" && + Predicate.hasProperty(reason.error, "reason") && + reason.error.reason === "disconnected", + ); + +/** The recording for one runtime: its HTTP routes, its RPC sessions, and its view of itself. */ +export const runtimeHops = (view: RuntimeView = noRuntimeView) => { + /** When the last HTTP exchange with this runtime ended, to say how long its origin was idle. */ + let lastHttpEnded: number | undefined; + + /** + * Run one HTTP exchange with the runtime and, when it fails, record how far it got: `route`, + * `method`, the `stage` (`request`: no response headers arrived; `status`: headers arrived with + * an unexpected status; `response-body`: headers arrived and reading the body failed), the time + * to failure and to the headers, how long the origin had been idle since the previous exchange + * ended, the status, the error tags and the platform error's shape. `send` sends the request; + * `read` checks and reads the response it returns. + * + * A fetch client does not report when its connection opened, whether it reused a pooled one, or + * when its request was fully written, so those cannot be told apart here: a failure before any + * response headers is `request`, the platform error's code says what the socket saw (Bun retries + * a reused connection's reset only for GET, so a POST reports it as `ECONNRESET`), and `idle_ms` + * says whether a pooled connection could have outlived the runtime's 5-second idle limit. + */ + const http = ( + route: "/workflow" | "/evaluated", + send: Effect.Effect, + read: (response: R) => Effect.Effect, + ): Effect.Effect => + Effect.suspend(() => { + const started = performance.now(); + const idleMs = lastHttpEnded === undefined ? undefined : started - lastHttpEnded; + let headersMs: number | undefined; + let status: number | undefined; + const record = (stage: HttpStage, error: unknown) => + annotate(() => { + const now = performance.now(); + return { + ...hostAttributes(view, now), + ...checked({ + "executor.runtime.http.route": route, + // Every runtime hop is a POST: the method a reused connection's reset is not retried for. + "executor.runtime.http.method": "POST", + "executor.runtime.http.stage": stage, + "executor.runtime.http.failed_ms": Math.round(now - started), + "executor.runtime.http.idle_ms": + idleMs === undefined ? undefined : Math.round(idleMs), + "executor.runtime.http.headers_ms": headersMs, + "executor.runtime.http.status": status, + "executor.runtime.http.error": tagOf(error), + "executor.runtime.http.reason": Predicate.hasProperty(error, "reason") + ? tagOf(error.reason) + : undefined, + ...platformShape("executor.runtime.http", platformError(error)), + }), + }; + }); + return send.pipe( + Effect.tapError((error) => record("request", error)), + Effect.flatMap((response) => { + headersMs = Math.round(performance.now() - started); + status = response.status; + return read(response).pipe( + Effect.tapError((error) => + record(response.status === 200 ? "response-body" : "status", error), + ), + ); + }), + Effect.ensuring( + Effect.sync(() => { + lastHttpEnded = performance.now(); + }), + ), + ); + }); + + /** + * Watch an RPC session's WebSocket: whether it opened and when, when it closed and with which + * code, whether a close reason was given (never its text), the code of any `error` event, how + * many messages were sent (attempts: each call to `send`) and received, and when the last one + * arrived. Listening and counting only: the socket's `send` is wrapped to count and otherwise + * called unchanged, and no listener throws. + */ + const track = (socket: TrackedSocket): SocketRecord => { + const created = performance.now(); + let openedAt: number | undefined; + let closedAt: number | undefined; + let lastReceivedAt: number | undefined; + let closeCode: unknown; + let closeReason: "none" | "other" | undefined; + let errorCode: string | undefined; + let sent = 0; + let received = 0; + socket.on("open", () => { + openedAt = performance.now(); + }); + socket.on("message", () => { + received++; + lastReceivedAt = performance.now(); + }); + socket.on("close", (code, reason) => { + closedAt = performance.now(); + closeCode = code; + try { + closeReason = + reason === undefined || reason === null || String(reason) === "" ? "none" : "other"; + } catch { + closeReason = "other"; + } + }); + socket.on("error", (error) => { + try { + const code = Predicate.hasProperty(error, "code") ? error.code : undefined; + errorCode = code === undefined ? "none" : String(shape(code, codeForm) ?? "other"); + } catch { + errorCode = "other"; + } + }); + const send = socket.send.bind(socket) as (...args: never[]) => unknown; + socket.send = (...args: never[]) => { + sent++; + return send(...args); + }; + return { + ended: () => closedAt !== undefined || errorCode !== undefined, + attributes: () => { + const now = performance.now(); + return { + "executor.runtime.rpc.opened": openedAt !== undefined, + "executor.runtime.rpc.open_ms": + openedAt === undefined ? undefined : Math.round(openedAt - created), + "executor.runtime.rpc.closed_ms": + closedAt === undefined ? undefined : Math.round(closedAt - created), + "executor.runtime.rpc.failed_ms": Math.round(now - created), + "executor.runtime.rpc.last_received_ms": + lastReceivedAt === undefined ? undefined : Math.round(now - lastReceivedAt), + "executor.runtime.rpc.close_code": closeCode, + "executor.runtime.rpc.close_reason": closeReason, + "executor.runtime.rpc.error_code": errorCode, + "executor.runtime.rpc.sent": sent, + "executor.runtime.rpc.received": received, + }; + }, + }; + }; + + /** + * Run `work` on an RPC session and, when it fails because the transport did (the socket closed or + * errored, or the host reports the connection lost), record the socket's shape. An app's own + * failure on a healthy session, a success and an interruption record nothing. + */ + const rpc = ( + socket: () => SocketRecord | undefined, + work: Effect.Effect, + ): Effect.Effect => + work.pipe( + Effect.tapCause((cause) => { + if (Cause.hasInterruptsOnly(cause)) return Effect.void; + const record = socket(); + if (record === undefined) return Effect.void; + let transport: boolean; + try { + transport = record.ended() || lostConnection(cause); + } catch { + transport = false; + } + if (!transport) return Effect.void; + return annotate(() => ({ + ...hostAttributes(view, performance.now()), + ...checked(record.attributes()), + })); + }), + ); + + return { http, track, rpc }; +}; diff --git a/packages/sdk/src/implementation/runtime.ts b/packages/sdk/src/implementation/runtime.ts index f6cc4cfc50..1144412c04 100644 --- a/packages/sdk/src/implementation/runtime.ts +++ b/packages/sdk/src/implementation/runtime.ts @@ -18,7 +18,7 @@ import { import type { DeclarationCache } from "../contracts/declarations.ts"; import type { CodeFiles } from "../contracts/source.ts"; import type { BuildId, Json } from "../contracts/shared.ts"; -import { BlobStore, type BlobStorage } from "../contracts/blobs.ts"; +import { BlobStore, type BlobKey, type BlobStorage } from "../contracts/blobs.ts"; const NativeRuntime = Symbol("executor.Runtime"); @@ -167,6 +167,48 @@ export const createAppRuntime = (options: { }; }; +/** + * What a stored object is, from the shape of its key alone: a key names an app, version or build, + * so the span records the kind and never the key. + */ +export const blobKind = (key: string) => + key.startsWith("frameworks/") + ? "framework" + : /^bld_[^/]+(\/worker)?\.json$/u.test(key) + ? "build" + : /^bld_[^/]+\/ui\//u.test(key) + ? "build-ui" + : /^bld_[^/]+\/public\.json$/u.test(key) + ? "public-manifest" + : /^deployments\/[^/]+\/source\.json$/u.test(key) + ? "version-source" + : /^app-source\/[^/]+\/initial\.json$/u.test(key) + ? "initial-source" + : key.startsWith("app-public/") + ? "public-file" + : key.startsWith("team-icons/") + ? "team-icon" + : "other"; + +/** + * One write, in a `storage.blob.put` span: the object's kind and size in bytes, and whether it was + * stored. A host's store adds how many attempts it made and its storage's failure code. + */ +const tracedPut = (blobs: BlobStorage, key: BlobKey, body: Uint8Array) => + blobs.put(key, body).pipe( + Effect.tap(() => Effect.annotateCurrentSpan("storage.blob.outcome", "stored")), + Effect.tapError(() => Effect.annotateCurrentSpan("storage.blob.outcome", "failed")), + Effect.withSpan("storage.blob.put", { + attributes: { "storage.blob.size": body.byteLength, "storage.blob.kind": blobKind(key) }, + }), + ); + +/** A store whose writes each have a span (`tracedPut`); its other operations are unchanged. */ +export const withTracedPuts = (blobs: BlobStorage): BlobStorage => ({ + ...blobs, + put: (key, body) => tracedPut(blobs, key, body), +}); + /** * Provide the host store without changing caller cancellation, tracing, or resource scopes. App * cache changes the runtime reports forget that app's kept results in `declarations`. @@ -203,12 +245,7 @@ export const toEffectRuntime = ( Effect.tap((found) => Effect.annotateCurrentSpan("storage.blob.found", found)), Effect.withSpan("storage.blob.exists"), ), - put: (key, body) => - blobs.put(key, body).pipe( - Effect.withSpan("storage.blob.put", { - attributes: { "storage.blob.size": body.byteLength }, - }), - ), + put: (key, body) => tracedPut(blobs, key, body), remove: (key) => blobs.remove(key).pipe(Effect.withSpan("storage.blob.remove")), removeAll: (keys) => blobs.removeAll(keys).pipe( diff --git a/packages/sdk/src/implementation/sleep-until.ts b/packages/sdk/src/implementation/sleep-until.ts new file mode 100644 index 0000000000..ec39f8451c --- /dev/null +++ b/packages/sdk/src/implementation/sleep-until.ts @@ -0,0 +1,38 @@ +/** + * A run's `sleepUntil` a time that has passed is over: the run goes on without asking the engine. + * + * The engine refuses a `sleepUntil` whose time is in the past ("You can't sleep until a time in + * the past", miniflare's `binding.worker.js`, the engine Cloudflare ships for local runs). A body + * that computes its time and then reaches the engine late, or whose first call failed and is + * repeated after the time, would otherwise fail its run for waiting too little. Skipping the call + * is the same on a replay: the time has passed then too, and each call has its own engine name. + */ +import { Clock, Effect } from "effect"; +import type { WorkflowFailure } from "apps/contracts"; +import { engineCallOf, transientEngineCall } from "../contracts/workflow-errors.ts"; + +/** + * Whether a failed call's time having passed ends the sleep: only for a failure in transit + * (`transientEngineCall`, the one classification of what is transient) or the engine's refusal of a + * time in the past. Any other failure is the engine's answer, such as a pause or a termination that + * arrives as the wait ends, and reaches the body as it would from a `sleep`. + */ +const overOnceTimePassed = (failure: WorkflowFailure) => + transientEngineCall(failure) !== undefined || engineCallOf(failure)?.code === "past"; + +/** Call the engine only while `timestamp` is ahead; see `overOnceTimePassed` for a failed call. */ +export const untilTime = ( + timestamp: number, + call: () => Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + const passed = Effect.annotateCurrentSpan("executor.workflow.call.passed", true); + if ((yield* Clock.currentTimeMillis) >= timestamp) return yield* passed; + return yield* call().pipe( + Effect.catch((failure) => + Effect.flatMap(Clock.currentTimeMillis, (now) => + now >= timestamp && overOnceTimePassed(failure) ? passed : Effect.fail(failure), + ), + ), + ); + }); diff --git a/packages/sdk/src/implementation/tool-search-store.ts b/packages/sdk/src/implementation/tool-search-store.ts index 8d4ab79a06..a720d29171 100644 --- a/packages/sdk/src/implementation/tool-search-store.ts +++ b/packages/sdk/src/implementation/tool-search-store.ts @@ -59,6 +59,16 @@ const Kept = Schema.Struct({ routers: Schema.Array(ToolRouter), superseded: Schema.NullOr(Schema.Number), }); +/** + * {@link Kept} with the listing and target it belongs to, for a read of several listings. Rows + * written before targets were recorded have none; they match no wanted target, as `listing`'s + * `target =` filter skips them. + */ +const KeptOf = Schema.Struct({ + ...Kept.fields, + listing: Schema.String, + target: Schema.NullOr(Schema.String), +}); const Listed = Schema.Struct({ listing: Schema.String }); /** A ranked page as one row: every part of it read in the same statement. */ const Ranked = Schema.Struct({ @@ -217,35 +227,35 @@ export const makeToolSearchStore = (sql: SqlClient.SqlClient) => { failed, ), /** - * `listing` for many targets in one statement: the kept rows of each listing whose row is for - * its target, by listing. + * The kept rows of each of these listings of its target, in order, read in one statement; + * undefined where none is kept. Each answers as `listing` would for it alone. */ listings: ( wanted: ReadonlyArray<{ readonly listing: ToolSearchListing; readonly target: string }>, - ): Effect.Effect, StorageError> => + ) => wanted.length === 0 - ? Effect.succeed(new Map()) + ? Effect.succeed([]) : sql`select listing, target, at::float8 as at, digest, routers, superseded::float8 as superseded from executor_tool_search_listings where listing in ${sql.in([...new Set(wanted.map((entry) => entry.listing))])}`.pipe( - Effect.flatMap( - Schema.decodeUnknownEffect( - Schema.Array( - Schema.Struct({ ...Kept.fields, listing: Schema.String, target: Schema.String }), - ), - ), + Effect.flatMap(Schema.decodeUnknownEffect(Schema.Array(KeptOf))), + Effect.map((rows) => + wanted.map((entry) => { + const row = rows.find( + (candidate) => + candidate.listing === entry.listing && candidate.target === entry.target, + ); + return row === undefined + ? undefined + : { + at: row.at, + digest: row.digest, + routers: row.routers, + superseded: row.superseded, + }; + }), ), - Effect.map((rows) => { - const targets = new Map(wanted.map((entry) => [entry.listing, entry.target])); - return new Map( - rows.flatMap(({ listing, target, ...kept }) => - targets.get(ToolSearchListing.make(listing)) === target - ? [[listing, kept] as const] - : [], - ), - ); - }), failed, ), /** diff --git a/packages/sdk/src/implementation/tool-search.ts b/packages/sdk/src/implementation/tool-search.ts index 29fef3d0e1..bae25957e3 100644 --- a/packages/sdk/src/implementation/tool-search.ts +++ b/packages/sdk/src/implementation/tool-search.ts @@ -211,7 +211,9 @@ export const makeToolSearch = (options: { /** The kept rows of the state's listing, if any. */ listing: (state: InvocationSnapshot, listing: ToolSearchListing) => Effect.flatMap(stateTarget(state), (target) => store.listing(listing, target)), - /** `listing` for many states in one read, by listing. */ + /** + * The kept rows of each state's listing, in order, read in one statement; see `listing`. + */ listings: ( wanted: ReadonlyArray<{ readonly state: InvocationSnapshot; diff --git a/packages/sdk/src/implementation/tools.ts b/packages/sdk/src/implementation/tools.ts index 28722eaf71..520f2d4365 100644 --- a/packages/sdk/src/implementation/tools.ts +++ b/packages/sdk/src/implementation/tools.ts @@ -103,6 +103,7 @@ import { ToolCallKindObserved, type PrefetchedInvocation, type PrefetchHost, + SearchablePrefetch, type SearchableTools, } from "../contracts/tools.ts"; import type { ExecutorDatabase } from "./storage.ts"; @@ -129,7 +130,7 @@ import { selectionAccountIds, type SelectionAccountsRead, } from "./selection.ts"; -import type { Listings, ToolListing } from "./listings.ts"; +import type { Listings, SearchableAhead, ToolListing } from "./listings.ts"; import { pageStart, pageTools } from "./tool-pages.ts"; import { ownsDatabase, sharesRuntime } from "../contracts/apps.ts"; @@ -142,6 +143,17 @@ const readApp = (db: Query, app: AppId) => }), ); +/** {@link readApp} for several apps, in one statement; an app without a row is not returned. */ +const readApps = (db: Query, apps: readonly AppId[]) => + apps.length === 0 + ? Effect.succeed([]) + : query(() => + db.findMany("apps", { + join: (b) => b.deployment(), + where: (b) => b("id", "in", apps), + }), + ); + /** * The reads {@link snapshot} makes, made ahead and at once. The account read stands in only when it * named the accounts the call names; otherwise the snapshot reads again, in order. A read that @@ -490,6 +502,13 @@ const appAccountReconnect = }); }; +/** + * The reconnect a failed access check reports for this state, named as resolving the state's + * accounts would name it: an account the app uses as its own carries its app requirement. + */ +export const reconnectReported = (state: Effect.Success>) => + appAccountReconnect(distinctSelection(state)); + /** * Resolve current credentials, to run the app, without holding a database transaction open. It * rechecks the actor's app access. `ahead` stands in for the app recheck only when it succeeded, @@ -853,6 +872,44 @@ const failureSource = Match.type().pipe( const observeFailure = (source: CallFailureSource) => Effect.map(CallFailureSources, (observe) => observe?.(source)); +/** + * A call's failure as `runtimeFailure` reports it, except a runtime exchange the host's transport + * never connected: nothing of it was sent, and the failure says so (`notSent`). + */ +const toolCallFailure = + ( + identity: { app: AppId; version: VersionId; tool: ToolName }, + state: InvocationSnapshot, + renewal: Renewal | undefined, + ) => + (failure: CallError) => + failure._tag === "RuntimeProtocolFailed" && failure.notSent === true + ? new ToolCallFailed({ + ...identity, + reason: + "Executor could not connect to the app runtime, so it did not send this tool call.", + notSent: true, + }) + : runtimeFailure(identity, state, renewal)(failure); +/** + * Whether a failure says that nothing of its exchange left the host. It holds for the call only + * when that exchange was the only one in the call to reach the app's code (`entries`): an earlier + * one, such as the catalog read for the call's kind, may have run code that wrote. + */ +const notSentOnly = (failure: CallFailure) => + failure._tag === "ToolCallFailed" && failure.notSent === true; +/** + * A call whose exchange never left the host after an earlier one of the same call reached the app: + * it may have partly run, and its failure says so rather than that the call was not sent. + */ +const partlyRun = ({ app, version, tool }: ToolCallFailed) => + new ToolCallFailed({ + app, + version, + tool, + reason: + "Executor could not reach the app runtime to finish this tool call. An earlier step of the call reached the app, so the call may have partly run.", + }); /** * The error a failed call reports to its caller, such as an agent. A call that may write, a * mutation or a call that names no kind, may already have made its change once the host handed any of @@ -866,14 +923,15 @@ const observeFailure = (source: CallFailureSource) => * (`confirmedRead`), keep their own classification. A query only its upstream calls one, such as * an MCP tool with `readOnlyHint`, may write like any other call. */ -const callFailure = - (read: boolean, enteredApp: boolean) => +export const callFailure = + (read: boolean, enteredApp: boolean, entries: number) => (failure: CallFailure): CallFailure => - read || !enteredApp + read || !enteredApp || (entries === 1 && notSentOnly(failure)) ? failure : Match.value(failure).pipe( Match.tagsExhaustive({ - ToolCallFailed: (failure) => mayHaveWritten(ToolCallFailed, failure), + ToolCallFailed: (failure) => + mayHaveWritten(ToolCallFailed, notSentOnly(failure) ? partlyRun(failure) : failure), AppProviderFailed: (failure) => mayHaveWritten(AppProviderFailed, failure), AppEvaluationFailed: (failure) => mayHaveWritten(AppEvaluationFailed, failure), ToolElicitationFailed: (failure) => mayHaveWritten(ToolElicitationFailed, failure), @@ -888,6 +946,25 @@ const callFailure = }), ); +/** + * One call's hand-offs to the app's code, as the host reports them through `AppCodeEntered`: the + * catalog read for the call's kind, the call itself, and a confirmed read's repeat after a renewal. + * `earlier` counts those that came before this tracker, such as the request a resume continues, + * whose setup and approval check ran in the app. `callFailure` judges the call's failure against + * how many there were, so a call whose last exchange never left the host says it was not sent only + * when that exchange was its only one. + */ +export const appEntries = (onEntered: () => void, earlier = 0) => { + let entries = earlier; + return { + entered: () => { + entries += 1; + onEntered(); + }, + callFailure: (read: boolean, enteredApp: boolean) => callFailure(read, enteredApp, entries), + }; +}; + /** * Report a defect that a call's failure replaces for its caller: the host's error reporter, such as * Sentry, captures it, and the call's span records it as the tracer records a defect it ends with. @@ -1288,6 +1365,123 @@ export const makeTools = ( }); return { leaves, take }; }; + /** + * One target's reads made ahead for `tools.searchable` by {@link prefetchSearchable}: the target + * they were made for, the actor they ran as, the invocation state its snapshot read, and what + * the listing store read for it. Taken at most once. + */ + interface SearchableEntry { + readonly target: { + readonly app: AppId; + readonly version: VersionId; + readonly accounts: AppAccounts; + }; + readonly actor: string | undefined; + readonly state: InvocationSnapshot; + readonly ahead: SearchableAhead; + } + /** The opaque value of a {@link SearchablePrefetch}: its entries by app, each taken once. */ + class SearchableReads { + readonly entries: Map; + constructor(entries: Map) { + this.entries = entries; + } + /** The entry made for exactly this app, version and accounts, removed as it is taken. */ + take(input: { + readonly app: AppId; + readonly version?: VersionId | undefined; + readonly accounts?: AppAccounts | undefined; + }) { + const entry = this.entries.get(input.app); + if ( + entry === undefined || + input.version !== entry.target.version || + input.accounts === undefined || + !sameAccounts(entry.target.accounts, input.accounts) + ) + return undefined; + this.entries.delete(input.app); + return entry; + } + } + /** + * {@link PrefetchHost}'s `searchable`: what `tools.searchable` reads for each target, for all of + * them at once. The apps and the accounts the targets name are read in one statement each, and + * each target's snapshot is made from them as it would make it from its own reads; then every + * target's kept rows, the actor's app rechecks and the grants are read in one statement each + * (`searchablePrefetch`). The host's checks have finished, so each read still follows them. A + * target whose snapshot fails, or whose app is another owner's, has no entry and is read live, + * as is every target when any of these statements fails. + */ + const prefetchSearchable: PrefetchHost["searchable"] = ({ owner, targets }) => + Effect.gen(function* () { + const actor = yield* CurrentActor; + const [apps, accounts] = yield* Effect.all( + [ + readApps(db, [...new Set(targets.map((target) => target.app))]), + readSelectionAccounts(db, [ + ...new Set(targets.flatMap((target) => namedAccounts(target.accounts))), + ]), + ], + { concurrency: "unbounded" }, + ); + const states = yield* Effect.forEach(targets, (target) => { + const ids = new Set(namedAccounts(target.accounts)); + const reads: SnapshotReads = { + app: Result.succeed(apps.find((row) => row.id === target.app) ?? null), + accounts: { + ids: [...ids], + rows: Result.succeed(accounts.filter((row) => ids.has(row.id))), + }, + }; + return snapshot(db, target, reads).pipe( + Effect.withSpan("sdk.invocation.snapshot"), + Effect.option, + Effect.map(Option.filter((state) => state.app.owner === owner)), + Effect.map(Option.map((state) => ({ target, state }))), + ); + }).pipe(Effect.map((found) => found.flatMap(Option.toArray))); + const aheads = yield* listings.searchablePrefetch( + owner, + states.map(({ state }) => state), + ); + const entries = new Map(); + states.forEach(({ target, state }, index) => { + const ahead = aheads[index]; + // One entry per app: the first target keeps it and a later one of the same app reads live. + // `take` still matches the exact version and accounts, so no target gets another's entry. + if (ahead === undefined || entries.has(target.app)) return; + entries.set(target.app, { target, actor, state, ahead }); + }); + return { owner, value: new SearchableReads(entries) }; + }).pipe( + Effect.catchCause(() => Effect.succeed({ owner, value: undefined })), + Effect.withSpan("sdk.tools.searchable.prefetch", { + attributes: { "executor.prefetch.targets": targets.length }, + }), + ); + /** + * The entry the host's {@link SearchablePrefetch} holds for this input, under the same owner and + * actor, taken so no later read reuses it. + */ + const searchableAhead = (input: { + readonly app: AppId; + readonly version?: VersionId | undefined; + readonly accounts?: AppAccounts | undefined; + }) => + Effect.gen(function* () { + const prefetched = yield* SearchablePrefetch; + if (prefetched === undefined || !(prefetched.value instanceof SearchableReads)) + return undefined; + const entry = prefetched.value.take(input); + if ( + entry === undefined || + entry.state.app.owner !== prefetched.owner || + entry.actor !== (yield* CurrentActor) + ) + return undefined; + return entry; + }); // A prefetch answers one call: it is provided around the whole host call, so a later call in // that scope (none today) must read live rather than reuse reads that may be seconds old. const consumedPrefetches = new WeakSet(); @@ -1428,6 +1622,7 @@ export const makeTools = ( ); return { prefetchInvocation, + prefetchSearchable, /** * The first of these accounts whose saved sign-in must reconnect before the app's tools can * run with them; undefined when every account can supply credentials. Reads stored state only: @@ -1500,8 +1695,19 @@ export const makeTools = ( options?: ToolListOptions, ) => Effect.gen(function* () { - const state = yield* listed(input); - const found = yield* listings.searchable(state, listingOf(state), options); + const ahead = yield* searchableAhead(input); + const state = + ahead === undefined + ? yield* listed(input) + : yield* Effect.as( + Effect.annotateCurrentSpan({ + "executor.app.id": ahead.state.app.id, + "executor.deployment.id": ahead.state.version.id, + "executor.build.id": ahead.state.version.build, + }), + ahead.state, + ); + const found = yield* listings.searchable(state, listingOf(state), options, ahead?.ahead); return { listing: found.listing, version: state.version.id, @@ -1525,6 +1731,48 @@ export const makeTools = ( ) => Effect.gen(function* () { if (inputs.length === 0) return []; + // The kept-row fast path and live fallback consume the same request-owned upstream + // prefetch. A miss or refusal leaves its entry for the live path to explain. + const prefetched = yield* SearchablePrefetch; + if (prefetched !== undefined && prefetched.value instanceof SearchableReads) { + const actor = yield* CurrentActor; + const entries = inputs.map((input) => { + const entry = + prefetched.value instanceof SearchableReads + ? prefetched.value.entries.get(input.app) + : undefined; + return entry !== undefined && + entry.actor === actor && + entry.state.app.owner === prefetched.owner && + input.version === entry.target.version && + input.accounts !== undefined && + sameAccounts(input.accounts, entry.target.accounts) + ? entry + : undefined; + }); + const current = entries.flatMap((entry) => (entry === undefined ? [] : [entry])); + const found = yield* listings.searchableKept( + current.map((entry) => entry.state), + current.map((entry) => entry.ahead), + ); + let position = 0; + return entries.map((entry): SearchableTools | undefined => { + if (entry === undefined) return undefined; + const served = found[position++]; + if (served === undefined) return undefined; + if (prefetched.value instanceof SearchableReads) prefetched.value.take(entry.target); + return { + listing: served.listing, + version: entry.state.version.id, + accounts: scopedSelection( + entry.state.version.requirements.accounts, + entry.state.accounts, + entry.state.scope, + ), + routers: served.routers.filter((router) => router.error !== undefined), + }; + }); + } const apps = [...new Set(inputs.map((input) => input.app))]; const rows = yield* query(() => db.findMany("apps", { @@ -1687,11 +1935,9 @@ export const makeTools = ( // Set by the host when it hands this call's work to the app's code: catalog reads for its // kind, and the call itself. let enteredApp = false; - const appCode = { - entered: () => { - enteredApp = true; - }, - }; + const appCode = appEntries(() => { + enteredApp = true; + }); // The caller's kind, confirmed by the app for a query (`confirmedRead`), decides how a // failure is reported and whether a refused call is repeated. For a call that names none, // the catalog only names the kind it is dispatched with: reading the catalog ran the app's @@ -1764,7 +2010,7 @@ export const makeTools = ( ); } yield* observeFailure(failureSource(result.failure)); - return yield* Effect.fail(runtimeFailure(identity, state, renewal)(result.failure)); + return yield* Effect.fail(toolCallFailure(identity, state, renewal)(result.failure)); }).pipe( Effect.catch((failure) => // Only a confirmed read's renewal fails a call itself (`executeRenewing`), and a @@ -1775,7 +2021,9 @@ export const makeTools = ( failure._tag === "AccountRequired" || !enteredApp ? Effect.fail(failure) - : Effect.flatMap(read, (read) => Effect.fail(callFailure(read, enteredApp)(failure))), + : Effect.flatMap(read, (read) => + Effect.fail(appCode.callFailure(read, enteredApp)(failure)), + ), ), // A defect after the hand-off, such as in Executor's own code, does not establish what // the app's code did either. A call that may write reports it as a failure that records @@ -1893,6 +2141,10 @@ export const makeTools = ( ); // Set by the host when it hands this call's work to the app's code, as for a call. let enteredApp = false; + // Approval setup already entered app code, even if this resumed exchange never connects. + const appCode = appEntries(() => { + enteredApp = true; + }, 1); // As for a call, the saved (caller's) kind counts as a read only when the app // confirms it. const read = yield* readOf(state, saved.tool, saved.kind); @@ -1943,7 +2195,7 @@ export const makeTools = ( if (Result.isFailure(result)) { yield* observeFailure(failureSource(result.failure)); return yield* Effect.fail( - runtimeFailure( + toolCallFailure( { app: saved.app, version: saved.version, tool: saved.tool }, state, renewal, @@ -1969,14 +2221,10 @@ export const makeTools = ( !enteredApp ? Effect.fail(failure) : Effect.flatMap(read, (read) => - Effect.fail(callFailure(read, enteredApp)(failure)), + Effect.fail(appCode.callFailure(read, enteredApp)(failure)), ), ), - Effect.provideService(AppCodeEntered, { - entered: () => { - enteredApp = true; - }, - }), + Effect.provideService(AppCodeEntered, appCode), ); }).pipe( // The request is consumed, so its caller receives the call's own failure. diff --git a/packages/sdk/src/implementation/webhook-sources.ts b/packages/sdk/src/implementation/webhook-sources.ts index ddb345d36a..96d1ac7c24 100644 --- a/packages/sdk/src/implementation/webhook-sources.ts +++ b/packages/sdk/src/implementation/webhook-sources.ts @@ -87,6 +87,11 @@ export const setupCompleted = ( }), ); if (row === null || row.status === "removing") return; + // Joins the pending queue now, as `link` does (`queued`): `leaseUntil` of a pending source + // without a lease is when it joined. A failed source's is its retry time, which can be an + // hour ahead. A held lease is left to its holder, which finds the newer revision. + const joining = + row.lease === null ? { leaseUntil: new Date(yield* Clock.currentTimeMillis) } : {}; yield* query(() => tx.updateMany("webhookSources", { where: (b) => b("id", "=", row.id), @@ -95,6 +100,7 @@ export const setupCompleted = ( failure: null, retries: 0, revision: row.revision + 1, + ...joining, }, }), ); @@ -130,6 +136,24 @@ export const makeWebhookSources = ( }, ) => { const now = Clock.currentTimeMillis; + /** + * This service's own name in telemetry, on its sweep's passes and its reads of an app's sources, + * so a pass can be told from another server's that shares a collector. One host process (a + * self-hosted server's actor) reads and sweeps through one service; on Cloud each Worker has its + * own. + */ + const instance = globalThis.crypto.randomUUID(); + /** + * A source joining the pending queue: `leaseUntil` of a pending source without a lease is when it + * joined, so `tick` serves pending sources first in, first out. A source whose lease is held + * keeps it; its holder finds the newer revision when it finishes and queues it at that time. + * Read under the app's lock, which every claim also takes, so the lease cannot change before the + * write that uses this. + */ + const queued = (source: { readonly lease: string | null }) => + Effect.map(now, (time): { readonly leaseUntil?: Date } => + source.lease === null ? { leaseUntil: new Date(time) } : {}, + ); const keyFor = (value: Json) => crypto.digest .digest(Digest.sha256, new TextEncoder().encode(canonical(value))) @@ -427,6 +451,7 @@ export const makeWebhookSources = ( // Acquire/release owns the claim across interruption and defects too. // A completed or superseded claim must never release another worker's lease. yield* lockApp(tx, { app: input.app }); + const queuedAt = new Date(yield* now); yield* query(() => tx.updateMany("webhookSources", { where: (b) => @@ -435,7 +460,9 @@ export const makeWebhookSources = ( b("account", "=", input.account), b("lease", "=", token), ), - set: { lease: null, leaseUntil: new Date(0) }, + // Back of the queue: a source failing this way would otherwise come first + // at every sweep and keep the others waiting. + set: { lease: null, leaseUntil: queuedAt }, }), ); }), @@ -489,7 +516,11 @@ export const makeWebhookSources = ( return yield* Schema.decodeUnknownEffect(Schema.Array(WebhookSourceSummary))( rows.map((row) => ({ ...row, label: labels.get(row.account) })), ).pipe(Effect.mapError(() => new StorageError())); - }), + }).pipe( + Effect.withSpan("sdk.webhooks.sources", { + attributes: { "executor.webhooks.instance": instance }, + }), + ), link: (input: typeof LinkWebhookSource.Type) => transaction(db, (tx) => Effect.gen(function* () { @@ -529,17 +560,19 @@ export const makeWebhookSources = ( revision: 1, reconciledDeployment: null, lease: null, - leaseUntil: new Date(0), + leaseUntil: createdAt, createdAt, }), ); - } else + } else { + const joining = yield* queued(existing); yield* query(() => tx.updateMany("webhookSources", { where: (b) => b("id", "=", existing.id), - set: { ...fields, revision: existing.revision + 1 }, + set: { ...fields, revision: existing.revision + 1, ...joining }, }), ); + } const saved = yield* find(parsed); if (saved === undefined) return yield* new StorageError(); return yield* view(saved); @@ -565,6 +598,7 @@ export const makeWebhookSources = ( existing.revision !== parsed.expected.revision ) return yield* new WebhookConflict(); + const joining = yield* queued(existing); yield* query(() => tx.updateMany("webhookSources", { where: (b) => b("id", "=", existing.id), @@ -575,6 +609,7 @@ export const makeWebhookSources = ( failure: null, retries: 0, revision: existing.revision + 1, + ...joining, }, }), ); @@ -614,7 +649,8 @@ export const makeWebhookSources = ( Effect.gen(function* () { const time = new Date(yield* now); // New saved intent runs immediately, ahead of retries: a backlog of failed or waiting - // sources never delays one someone just changed. + // sources never delays one someone just changed. A pending source without a lease holds + // in `leaseUntil` when it joined the queue (`queued`), so this is first in, first out. const intent = yield* query(() => db.findMany("webhookSources", { where: (b) => b.and(b("status", "=", "pending"), b("lease", "is", null)), @@ -678,6 +714,14 @@ export const makeWebhookSources = ( ), ), ); + // What this pass took, for anyone judging the queue from its traces: a pass that took fewer + // new sources than its limit left none of them behind. + yield* Effect.annotateCurrentSpan({ + "executor.webhooks.instance": instance, + "executor.webhooks.sweep.limit": limit, + "executor.webhooks.sweep.taken": intent.length, + "executor.webhooks.sweep.retries": ready.length, + }); const reconciled = yield* Effect.forEach( [...intent, ...ready], (row) => @@ -692,6 +736,6 @@ export const makeWebhookSources = ( // A full batch of saved intent may have left more behind. Continue while it makes // progress; a batch whose intent all failed waits for the host's next wake. return intent.length >= limit && reconciled.slice(0, intent.length).some(Boolean); - }), + }).pipe(Effect.withSpan("sdk.webhooks.sweep")), }; }; diff --git a/packages/sdk/src/implementation/worker-build-storage.ts b/packages/sdk/src/implementation/worker-build-storage.ts index 108ff817c5..b5ddfee79b 100644 --- a/packages/sdk/src/implementation/worker-build-storage.ts +++ b/packages/sdk/src/implementation/worker-build-storage.ts @@ -2,14 +2,14 @@ * Retained builds store each `apps` framework release once. A build's record holds its own modules * and the identity of the framework it links; loading a build reads both and links them. */ -import { BlobKey, BlobStore } from "../contracts/blobs.ts"; +import { BlobKey, BlobStore, BlobStoreError } from "../contracts/blobs.ts"; import { RuntimeBuildFailed, RuntimeBuildUnavailable } from "../contracts/runtime.ts"; import type { BuildId } from "../contracts/shared.ts"; import type { UiBuildFile } from "../contracts/ui-build.ts"; import type { RuntimeAsset } from "../contracts/runtime.ts"; import { asciiJson } from "./ascii-json.ts"; import { publicFileAsset, readPublicManifest } from "./public-files.ts"; -import { Effect, Option, Schema } from "effect"; +import { Effect, Option, Schedule, Schema } from "effect"; import { type FrameworkIdentity, InlinedWorkerBuild, @@ -21,6 +21,15 @@ import { type WorkerFramework, } from "../contracts/worker-build.ts"; +/** + * A failure of storage itself, which writing again later may not meet: no status, a rate limit, or + * a server error. A store that answers another 4xx refused this write for what it carries (an + * invalid key), and so does a key the key type rejects: those are the app's, and are not repeated. + */ +export const storageFailure = (error: unknown): error is BlobStoreError => + Schema.is(BlobStoreError)(error) && + (error.status === undefined || error.status === 429 || error.status >= 500); + /** Shared with the one-off framework migration, which reads and writes the same objects. */ export const key = (value: string) => Schema.decodeUnknownEffect(BlobKey)(value); /** The release's label leads for operators; the hash alone decides which object a build links. */ @@ -64,7 +73,10 @@ export const assembleWorkerBundle = ( /** * Store the framework once at its content address. Concurrent deploys of the same release write - * identical content, and each put publishes a complete object, so a lost race is harmless. + * identical content, and each put publishes a complete object, so a lost race is harmless. A store + * may refuse a write that loses the race (R2 takes one write a second to a key): the object then + * being there means another build stored these same bytes, which is all this needs. The check is + * repeated for up to 7 s, since the other build's write may still be in flight. */ const retainFramework = (framework: WorkerFramework) => Effect.gen(function* () { @@ -77,7 +89,21 @@ const retainFramework = (framework: WorkerFramework) => "executor.build.framework_stored": stored, }); const retained: RetainedFramework = { ...identity, modules: framework.modules }; - if (!stored) yield* blobs.put(location, yield* encodeJson(RetainedFramework, retained)); + if (!stored) + yield* blobs.put(location, yield* encodeJson(RetainedFramework, retained)).pipe( + Effect.catchIf(storageFailure, (refused) => + // A look that itself fails counts as not found, so the write's own failure is reported. + blobs.exists(location).pipe( + Effect.orElseSucceed(() => false), + Effect.flatMap((found) => (found ? Effect.void : Effect.fail(refused))), + // At most 7 s: the other build's write, uncontended, takes under 2 s. + Effect.retry({ times: 3, schedule: Schedule.exponential("1 second") }), + Effect.tap(() => + Effect.annotateCurrentSpan("executor.build.framework_stored_by", "another"), + ), + ), + ), + ); return retained; }); @@ -123,7 +149,11 @@ export const retainWorkerBuild = ( yield* blobs.put(yield* buildKey(build), body); return { record, framework: retained }; }).pipe( - Effect.mapError(() => new RuntimeBuildFailed({ stage: "retain" })), + // A storage failure says nothing about the app and is reported as one; anything else here (a + // UI file name no key can hold, a key the store refuses) fails the build as it always has. + Effect.mapError((error) => + storageFailure(error) ? error : new RuntimeBuildFailed({ stage: "retain" }), + ), Effect.withSpan("runtime.cloud.retain"), ); diff --git a/packages/sdk/src/implementation/worker-build.ts b/packages/sdk/src/implementation/worker-build.ts index ff5a649509..d3ad76804e 100644 --- a/packages/sdk/src/implementation/worker-build.ts +++ b/packages/sdk/src/implementation/worker-build.ts @@ -17,6 +17,7 @@ import { sourceImports } from "./worker-source-imports.ts"; import { serverSourceMap, sourceLocation } from "./worker-source-map.ts"; import { wasmBuild } from "./worker-wasm-build.ts"; import { jsonModules } from "./worker-json-modules.ts"; +import { compiledFiles } from "./worker-compiled-files.ts"; import { serverFilesModule, serverFilesSource } from "./worker-bridge.ts"; import { workerDependencies } from "./worker-dependencies.ts"; import apps from "apps/package.json" with { type: "json" }; @@ -26,9 +27,13 @@ import apps from "apps/package.json" with { type: "json" }; * framework of its own: every source declares the `apps` release it uses in `dependencies.apps`. * `apps` supplies the package files of one declared release, so a test deployment can build apps * against its own unpublished framework. Any other declared release installs from the registry. + * `tarballFetch` sends every install request off the registry's origin: tarball URL + * dependencies, and tarballs or redirects the registry's metadata sends elsewhere. Requests on the + * registry's origin use the global `fetch`, as everything does without `tarballFetch`. */ export interface WorkerHost { readonly registry?: string; + readonly tarballFetch?: (input: string, init?: RequestInit) => Promise; readonly apps?: { readonly version: string; /** Package-relative paths and contents, as in the published archive. */ @@ -178,6 +183,7 @@ export const compileWorkerApp = ( const wasm = wasmBuild(filesystem, path); const json = jsonModules(files, entry); const sourceMap = serverSourceMap(entry); + const sizes = compiledFiles(filesystem); const compiled = yield* Effect.tryPromise({ try: () => createApp({ @@ -199,9 +205,11 @@ export const compileWorkerApp = ( // own lookup of the asset file then reaches the source resolver. ...(browser === undefined ? [] : [browser.plugin]), sourceImports(filesystem, path), + // Last: it sees only the files the bundler itself loads and compiles. + sizes.plugin, ], }), - catch: (cause) => json.failure() ?? compileFailure(cause), + catch: (cause) => sizes.failure() ?? json.failure() ?? compileFailure(cause), }); const map = sourceMap.map(); if (map === undefined) diff --git a/packages/sdk/src/implementation/worker-compiled-files.ts b/packages/sdk/src/implementation/worker-compiled-files.ts new file mode 100644 index 0000000000..b82aff2d53 --- /dev/null +++ b/packages/sdk/src/implementation/worker-compiled-files.ts @@ -0,0 +1,74 @@ +/** + * Refuse a file too large to compile before esbuild parses it. esbuild-wasm compiles each file in + * a single synchronous stretch on the thread that serves the host's requests, so this bounds the + * work esbuild does on any one file. It does not bound a whole build: one of many large files can + * still hold the thread for several seconds. Only compiling off that thread bounds request delay. + */ +import type { InMemoryFileSystem } from "@cloudflare/worker-bundler"; +import type { Plugin } from "esbuild"; +import { compiledFileLimit } from "../contracts/source.ts"; +import { boundBuildMessage, RuntimeBuildFailed } from "../contracts/runtime.ts"; + +/** + * The bytes `TextEncoder` writes for `text`, which esbuild compiles, counted only until the code + * point that takes the count past `stop`, so a stopped count is at most `stop` plus 4. A high surrogate followed by a low one is one code point of four bytes; + * any other surrogate is written as U+FFFD, three bytes. + */ +const utf8Bytes = (text: string, stop = Number.POSITIVE_INFINITY) => { + let bytes = 0; + for (let at = 0; at < text.length && bytes <= stop; at++) { + const code = text.charCodeAt(at); + if (code < 0x80) bytes += 1; + else if (code < 0x800) bytes += 2; + else if (code >= 0xd800 && code < 0xdc00) { + const next = text.charCodeAt(at + 1); + if (next >= 0xdc00 && next < 0xe000) { + bytes += 4; + at++; + } else bytes += 3; + } else bytes += 3; + } + return bytes; +}; +/** Whether `text` is more than `limit` UTF-8 bytes. */ +const exceeds = (text: string, limit: number) => + // Each UTF-16 unit is at least one byte and at most three, so most files need no count. + text.length > limit || (text.length * 3 > limit && utf8Bytes(text, limit) > limit); + +const byteCount = (count: number) => `${count.toLocaleString("en-US")} bytes`; + +/** + * Applies to the server and browser builds, and must be the last of the build's own plugins: it + * sees only the files no earlier plugin loaded, which the bundler then reads from `filesystem` and + * compiles as JavaScript or TypeScript. Authored server JSON imports become modules of their own + * and browser assets are copied, so neither reaches it. Other JSON (in packages and the browser + * build) and CSS do reach it and are left out on purpose: esbuild loads them with its JSON and CSS + * loaders, outside this limit. Files of installed packages are not limited: they compile as they + * always have, however large. + */ +export const compiledFiles = (filesystem: InMemoryFileSystem) => { + let failure: RuntimeBuildFailed | undefined; + const plugin: Plugin = { + name: "executor-compiled-file-size", + setup(build) { + build.onLoad({ filter: /.*/, namespace: "virtual" }, (args) => { + if (args.path.endsWith(".json") || args.path.endsWith(".css")) return undefined; + // Only the app's own source is limited; installed packages compile as they always have. + if (args.path.startsWith("node_modules/")) return undefined; + const text = filesystem.read(args.path); + if (text === null || !exceeds(text, compiledFileLimit)) return undefined; + const message = boundBuildMessage( + `${args.path} is ${byteCount(utf8Bytes(text))}; a JavaScript or TypeScript file the build compiles can be at most ${byteCount(compiledFileLimit)} (2 MB). Split it into smaller modules, or move data it holds into a JSON file, which this limit does not cover.`, + ); + failure ??= new RuntimeBuildFailed({ + stage: "source", + // A location names at most 1,024 characters of path. + ...(args.path.length <= 1024 ? { location: { file: args.path } } : {}), + message, + }); + return { errors: [{ text: message }] }; + }); + }, + }; + return { plugin, failure: () => failure }; +}; diff --git a/packages/sdk/src/implementation/worker-dependencies.ts b/packages/sdk/src/implementation/worker-dependencies.ts index 972ce7ea80..3f1feae187 100644 --- a/packages/sdk/src/implementation/worker-dependencies.ts +++ b/packages/sdk/src/implementation/worker-dependencies.ts @@ -19,12 +19,17 @@ const Package = Schema.Struct({ * registry URL can carry credentials or a token, and the deployer reading a failed build is not * always the operator, so a failure loses each URL's userinfo, query and fragment. A token the * registry takes as a path segment is kept and reaches the deployer: it cannot be told from a path. + * Requests off the registry's origin, including where its redirects land, use the host's + * `tarballFetch`. The installer words every failure itself: it names the declared package, an + * HTTP status or a fixed reason, never a dependency's name, a URL a server chose, or anything a + * server or an archive sent. */ export const workerDependencies = (filesystem: InMemoryFileSystem, host: WorkerHost) => Effect.gen(function* () { // The installer appends `/` to the registry; a configured trailing slash would request // `//`, which registries such as Verdaccio answer with 404. const registry = host.registry?.replace(/\/+$/u, ""); + const { tarballFetch } = host; const manifest = filesystem.read("package.json"); const dependencies = manifest === null @@ -51,7 +56,11 @@ export const workerDependencies = (filesystem: InMemoryFileSystem, host: WorkerH list: (prefix) => filesystem.list(prefix), flush: () => filesystem.flush(), }, - { transitive, ...(registry === undefined ? {} : { registry }) }, + { + transitive, + ...(registry === undefined ? {} : { registry }), + ...(tarballFetch === undefined ? {} : { tarballFetch }), + }, ), catch: (cause) => new RuntimeBuildFailed({ diff --git a/packages/sdk/src/implementation/worker-workflow-rpc.ts b/packages/sdk/src/implementation/worker-workflow-rpc.ts index ff0864651d..d744751af3 100644 --- a/packages/sdk/src/implementation/worker-workflow-rpc.ts +++ b/packages/sdk/src/implementation/worker-workflow-rpc.ts @@ -7,10 +7,13 @@ import { WorkflowControlCommand, WorkflowRun, WorkflowRunPage, + workflowDurationMillis, type WorkflowExecution, type WorkflowHostControls, type WorkflowRpc, } from "apps/contracts"; +import { repeatedSleep } from "./workflow-engine-calls.ts"; +import { untilTime } from "./sleep-until.ts"; const reply = (effect: Effect.Effect) => effect.pipe( @@ -59,14 +62,27 @@ export const invocationWorkflow = ( ), ); } + // A sleep is repeated when the call to the engine failed in transit; see + // `workflow-engine-calls.ts`. `signal` interrupts it, its calls and its waits. case "sleep": - return yield* execution.driver - .sleep(command.name, command.duration) - .pipe(Effect.as(null)); + return yield* repeatedSleep( + execution.runId, + "sleep", + command.name, + () => execution.driver.sleep(command.name, command.duration), + workflowDurationMillis(command.duration), + ); case "until": - return yield* execution.driver - .sleepUntil(command.name, command.timestamp) - .pipe(Effect.as(null)); + return yield* repeatedSleep( + execution.runId, + "until", + command.name, + () => + untilTime(command.timestamp, () => + execution.driver.sleepUntil(command.name, command.timestamp), + ), + undefined, + ); case "context": { const context = yield* execution.resolve(); return Redacted.value(context.accounts); diff --git a/packages/sdk/src/implementation/workerd-apps.ts b/packages/sdk/src/implementation/workerd-apps.ts index c02bcaadf3..8b7278e024 100644 --- a/packages/sdk/src/implementation/workerd-apps.ts +++ b/packages/sdk/src/implementation/workerd-apps.ts @@ -18,6 +18,7 @@ import { WebSocket as NodeWebSocket } from "ws"; import { newWebSocketRpcSession, type RpcStub } from "capnweb"; import { Cause, + Config, ConfigProvider, Effect, FileSystem, @@ -51,6 +52,21 @@ import { runtimeAdapter } from "./runtime.ts"; import { connectedWorkerdApps, TransportClosed, workerdHostHandler } from "./workerd-client.ts"; import { workerdHostModules } from "./workerd-bundle.ts"; import { appWorkerIdleSeconds, appWorkerLimit } from "./app-worker-residency.ts"; +import { EngineFaults } from "../contracts/engine-faults.ts"; + +/** E2E runs only: engine call faults for the runtime's workflows; see `engine-faults.ts`. */ +const engineFaults = Effect.gen(function* () { + const environment = yield* Config.option(Config.String("EXECUTOR_ENVIRONMENT")); + if (Option.getOrUndefined(environment) !== "e2e") return null; + const configured = yield* Config.option(Config.String("EXECUTOR_TEST_WORKFLOW_ENGINE_FAULTS")); + return Option.isNone(configured) + ? null + : yield* Schema.decodeUnknownEffect(Schema.fromJsonString(EngineFaults))(configured.value).pipe( + Effect.orDie, + ); +}); +import { runtimeHops, type SocketRecord } from "./runtime-hop.ts"; +import { repeatingAtConnect, repeatUnopened, type Opening } from "./runtime-connect.ts"; /** Existing stores need an explicit migration; opening a new empty store would hide retained app data. */ export class WorkerdMigrationRequired extends Schema.TaggedError()( @@ -218,11 +234,13 @@ export const workerdApps = (options: { // matched to one. Only the exit's code and signal are kept: its stderr can quote app output. const services = yield* Effect.context(); let restarts = 0; + let lastRestartAt: number | undefined; const restarted = (exit: { readonly exitCode: number | null; readonly signal: string | null; }) => { restarts += 1; + lastRestartAt = performance.now(); Effect.runForkWith(services)( Effect.logWarning("App runtime restarted").pipe( Effect.annotateLogs({ @@ -265,6 +283,7 @@ export const workerdApps = (options: { "APP_WORKER_IDLE_SECONDS", Option.getOrNull(yield* appWorkerIdleSeconds), ), + JsonBinding.local("WORKFLOW_ENGINE_FAULTS", yield* engineFaults), publicEgressBinding, JsonBinding.local("SELF_ORIGIN", options.selfOrigin?.origin ?? ""), JsonBinding.local("NPM_REGISTRY", options.npmRegistry ?? ""), @@ -277,6 +296,12 @@ export const workerdApps = (options: { }) .pipe(Effect.provideContext(runtimeContext)); const headers = { authorization: `Bearer ${secret}` }; + // How far a failed exchange with the runtime got, with its restarts; see runtime-hop.ts. A + // supervisor's probe events would go in `probe`. + const hops = runtimeHops({ + restarts: () => ({ count: restarts, lastAt: lastRestartAt }), + probe: () => ({}), + }); const websocketUrl = (pathname: string) => { const url = new URL(pathname, origin); url.protocol = "ws:"; @@ -287,8 +312,17 @@ export const workerdApps = (options: { // delivered to the host Worker. The session then waits forever for its result. const connect = (pathname: string) => new NodeWebSocket(websocketUrl(pathname), { headers, perMessageDeflate: false }); + // An RPC exchange sends nothing until its socket has opened. One whose socket never opened is + // run once more from `session`'s start; see runtime-connect.ts. const rpc = ( work: (api: RpcStub, signal: AbortSignal) => Effect.Effect, + ) => + repeatUnopened((opening) => + session(opening, (api, signal) => Effect.andThen(opening.opened, () => work(api, signal))), + ); + const session = ( + opening: Opening, + work: (api: RpcStub, signal: AbortSignal) => Effect.Effect, ) => Effect.scoped( Effect.gen(function* () { @@ -296,9 +330,13 @@ export const workerdApps = (options: { Effect.sync(() => new AbortController()), (controller) => Effect.sync(() => controller.abort()), ); + let socketRecord: SocketRecord | undefined; const peer = yield* Effect.acquireRelease( Effect.sync(() => { const socket = connect("/rpc"); + // Measured only, for a failed call's span: see runtime-hop.ts. + socketRecord = hops.track(socket); + opening.watch(socket); // Registered before Cap'n Web's own listeners, so the session knows its socket // closed by the time it breaks; see `TransportClosed`. let closed = false; @@ -327,7 +365,7 @@ export const workerdApps = (options: { } }).pipe(Effect.catchCause(() => Effect.void)), ); - return yield* work(peer, lifetime.signal); + return yield* hops.rpc(() => socketRecord, work(peer, lifetime.signal)); }), ); const changes = (app: string) => @@ -368,23 +406,29 @@ export const workerdApps = (options: { ); // workerd closes an idle keep-alive connection after 5 seconds, and reconciliation polls // every 5 seconds. A pooled socket can then close while a request is written to it, and - // the fetch fails with a transport error. Each workflow request uses its own connection. + // the fetch fails with a transport error. Each workflow request asks for its connection to be + // closed after the response. Bun can still send it on an idle pooled connection, which only a + // request without that header would leave; none to this origin does. const backend = (operation: "start" | "status" | "terminate", run: WorkflowRunId) => - Effect.scoped( + repeatingAtConnect( Effect.gen(function* () { const request = yield* HttpClientRequest.post(new URL("/workflow", origin), { headers: { ...headers, connection: "close" }, }).pipe(HttpClientRequest.bodyJson({ operation, run })); - const response = yield* http.execute(request); - if (response.status !== 200) return yield* engineFailure(); - return yield* response.json.pipe( - Effect.flatMap(Schema.decodeUnknownEffect(WorkflowBackendState)), + return yield* hops.http("/workflow", http.execute(request), (response) => + Effect.gen(function* () { + if (response.status !== 200) return yield* engineFailure(); + return yield* response.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(WorkflowBackendState)), + ); + }), ); }), ).pipe(Effect.mapError(engineFailure)); - // Each command uses its own connection, like workflow requests above. + // Each command asks for its connection to be closed after the response, like workflow requests + // above. const evaluated = (app: string, command: EvaluatedCommand) => - Effect.scoped( + repeatingAtConnect( Effect.gen(function* () { const url = new URL("/evaluated", origin); url.searchParams.set("app", app); @@ -396,10 +440,13 @@ export const workerdApps = (options: { "application/json", ), ); - const response = yield* http.execute(request); - if (response.status !== 200) return yield* protocolFailure(); - return yield* response.text.pipe( - Effect.flatMap(Schema.decodeUnknownEffect(EvaluatedReplyJson)), + return yield* hops.http("/evaluated", http.execute(request), (response) => + Effect.gen(function* () { + if (response.status !== 200) return yield* protocolFailure(); + return yield* response.text.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(EvaluatedReplyJson)), + ); + }), ); }), ).pipe(Effect.mapError(protocolFailure)); diff --git a/packages/sdk/src/implementation/workerd-entry.ts b/packages/sdk/src/implementation/workerd-entry.ts index 23808756e2..18c37b5a06 100644 --- a/packages/sdk/src/implementation/workerd-entry.ts +++ b/packages/sdk/src/implementation/workerd-entry.ts @@ -42,6 +42,7 @@ import { makeAppWorkerResidency, type AppWorkerResidency, } from "./app-worker-residency.ts"; +import { isPrivateHostname } from "@executor-js/utils/url-policy"; import { compileWorkerApp } from "../workerd-build.ts"; import { assembleWorkerBundle } from "./worker-build-storage.ts"; import { declarationFailed } from "./worker-source-map.ts"; @@ -66,6 +67,13 @@ import { workflowFailureDetail, workflowFailureMessage, } from "../contracts/workflow-errors.ts"; +import { + engineFaultCall, + engineFaultError, + engineFaultFor, + type EngineFault, +} from "../contracts/engine-faults.ts"; +import { nativeEngineFailure } from "./native-engine-failure.ts"; declare const WebSocketPair: { new (): { 0: NativeWebSocket; 1: NativeWebSocket } }; @@ -139,7 +147,13 @@ interface Environment { }; readonly RUNS: Workflow<{ run: string }>; readonly HOST: Fetcher; + /** E2E runs only: engine call faults (`engine-faults.ts`); otherwise null. */ + readonly WORKFLOW_ENGINE_FAULTS?: ReadonlyArray | null; } + +/** A native sleep: a failure without our marker is the platform's, recorded by class only. */ +const nativeSleep = (work: () => Promise) => + Effect.tryPromise({ try: work, catch: nativeEngineFailure }).pipe(Effect.asVoid); const failure = () => new WorkflowFailure({ reason: "engine", retryable: true }); /** The deployer sees the underlying failure; builds bind no accounts. */ const buildFailed = (stage: RuntimeBuildFailed["stage"], cause: unknown) => @@ -184,6 +198,28 @@ export class AppOutbound extends WorkerEntrypoint { }); } } +/** The installer's name for a destination the host's network policy refuses. */ +class TarballDestinationRefused extends Error { + override readonly name = "NetworkRefused"; +} +/** + * How a build reaches anything off its registry's origin when the operator has not allowed + * private fetch: tarball URL dependencies, and tarballs or redirects registry metadata sends + * elsewhere. They go through the public-only network, as app code's own requests do. This + * Worker's global `fetch` uses the `internet` service, which reaches private addresses, and stays + * the path to the registry's own origin, so an operator's private registry keeps working. The + * installer follows each redirect itself and sends every hop here or to the registry. A private + * address named in the URL is refused before sending; one a public name resolves to is refused by + * the network, which workerd reports with an error reference only. + */ +const publicTarballFetch = + (network: HttpService) => + (input: string, init?: RequestInit): Promise => { + const url = URL.parse(input); + if (url === null || isPrivateHostname(url.hostname)) + return Promise.reject(new TarballDestinationRefused()); + return network.fetch(new Request(url, init)); + }; type OutboundLoopback = (options: { readonly props: OutboundProps }) => Fetcher; const OutboundExports = Schema.Struct({ AppOutbound: Schema.declare((value): value is OutboundLoopback => typeof value === "function"), @@ -305,7 +341,12 @@ class AppApi extends RpcTarget { ).pipe(Effect.mapError((cause) => buildFailed("source", cause))); const compiled = yield* compileWorkerApp( request.files, - this.#env.NPM_REGISTRY === "" ? {} : { registry: this.#env.NPM_REGISTRY }, + { + ...(this.#env.NPM_REGISTRY === "" ? {} : { registry: this.#env.NPM_REGISTRY }), + ...(this.#env.APPS_PRIVATE_FETCH + ? {} + : { tarballFetch: publicTarballFetch(this.#env.PUBLIC_FETCH) }), + }, request.public ?? [], ); const { bundle, framework, ui, sourceMap } = compiled; @@ -484,16 +525,23 @@ export class AppWorkflows extends WorkflowEntrypoint Promise) => { + const fault = engineFaultFor(faults, seed.runId, name); + return Effect.andThen( + progress, + nativeSleep(fault === undefined ? work : () => engineFaultCall(fault, work)), + ); + }; const execution: WorkflowExecution = { runId: seed.runId, driver: { - do: (name, options, work) => - native(() => + do: (name, options, work) => { + const fault = engineFaultFor(faults, seed.runId, name); + if (fault?.when === "before") + return Effect.fail(nativeEngineFailure(engineFaultError(fault))); + const call = native(() => nativeStep.do(name, options, (context) => { - // The engine counts an attempt before running it, and after its process is - // killed it marks that attempt failed and runs the step again without checking - // the step's retry limit. A step that keeps killing the process would then - // loop forever, so an attempt past the limit fails the run instead. const attempt = Schema.decodeUnknownOption(EngineStepContext)(context); if ( Option.isSome(attempt) && @@ -519,17 +567,18 @@ export class AppWorkflows extends WorkflowEntrypoint - Effect.andThen( - progress, - native(() => nativeStep.sleep(name, duration)), - ), + ); + return fault === undefined + ? call + : call.pipe( + Effect.flatMap(() => + Effect.fail(nativeEngineFailure(engineFaultError(fault))), + ), + ); + }, + sleep: (name, duration) => sleepCall(name, () => nativeStep.sleep(name, duration)), sleepUntil: (name, timestamp) => - Effect.andThen( - progress, - native(() => nativeStep.sleepUntil(name, timestamp)), - ), + sleepCall(name, () => nativeStep.sleepUntil(name, timestamp)), }, resolve: () => hostRequest(this.env, { operation: "context", run: seed.runId }).pipe( @@ -593,7 +642,20 @@ export class AppWorkflows extends WorkflowEntrypoint + (await this.env.RUNS.get(seed.runId)).status(), + ).pipe( + Effect.map((state) => state.status === "running" || state.status === "waiting"), + Effect.orElseSucceed(() => false), + ))); + if (recorded) yield* hostRequest(this.env, { operation: "finish", run: seed.runId, diff --git a/packages/sdk/src/implementation/workflow-engine-calls.ts b/packages/sdk/src/implementation/workflow-engine-calls.ts new file mode 100644 index 0000000000..3b1060a211 --- /dev/null +++ b/packages/sdk/src/implementation/workflow-engine-calls.ts @@ -0,0 +1,133 @@ +/** + * A run body's sleeps, repeated when the call to the engine failed in transit. + * + * Only a failure the hops classify as transient is repeated (`transientEngineCall`): an internal + * error or overload the engine reports as passing, a lost connection, or a host that never + * answered. The engine's own answers pass through at once: an aborted, paused or terminated + * instance ("Aborting engine: …" from the local engine, `instance.` from Cloudflare's), a + * refusal, a failure carrying our marker, which the engine relays from the app or the host, and any + * message no rule recognises. Step calls are never repeated here: the engine names a step by its + * name and how many times the run called it, so a repeat is a second entry whose work runs again. + * + * A repeat sends the same name and duration. The local engine keys a sleep by its name, its + * duration and how many times the run asked for that pair (`binding.worker.js`, `sleep`). A later + * replay asks for the first of them. For a `sleep(d)` whose first call failed: + * - the engine never saw it: the repeat is the first entry, which a replay finds. No extra wait. + * If that failed call itself outlasted `d`, as a slow failure can a short sleep, the sleep is + * over without an entry, and a later replay waits `d` once more. Up to `d` extra. + * - the engine recorded it and the answer was lost at `t` before `d`: the repeat waits `d` as a + * second entry; a replay finds the first. Up to `t` extra. + * - the answer was lost after `d`: the failed call lasted `d`, so the sleep is over and is not + * repeated (below). No extra wait. + * - the engine counted the call but failed before writing its entry: the repeat waits `d` as a + * second entry, and a later replay finds no first entry and waits `d` again. Up to `d` extra, + * once, on that replay. The run wakes late, never early. + * + * A repeat would wait the whole duration again after an engine that did wait and lost only its + * answer. So a `sleep` whose failed call itself lasted the duration is over instead of repeated: + * the sleep's time has passed. Usually that is an engine that held the call for the sleep; a slow + * failure in transit can also outlast a short sleep, which then also ends, never early, but with + * no engine entry, so a later replay can wait it once more. + * + * Nothing bounds a call itself: a sleep's call lasts as long as the sleep. Cancellation interrupts + * it, and the waits between calls, at once: the call runs in the RPC handler's fiber, which the + * invocation's signal interrupts (`invocationWorkflow`). + */ +import { Clock, Effect, Random, Result } from "effect"; +import { WorkflowFailure } from "apps/contracts"; +import { engineCallOf, transientEngineCall } from "../contracts/workflow-errors.ts"; + +/** Calls a sleep gets in all: the first and three repeats. */ +export const sleepAttempts = 4; +/** The wait before the first repeat. Each later wait doubles; each is jittered to 80-120%. */ +export const firstRepeatMillis = 200; + +/** The authored name inside the engine's step name, `::<64 hex digits>`. */ +const authoredName = (name: string) => /^[a-z]+:([\s\S]*):[0-9a-f]{64}$/.exec(name)?.[1] ?? name; + +/** What the run's owner reads when every call failed: the hops' own words, no platform text. */ +const exhausted = ( + kind: "sleep" | "until", + name: string, + attempts: number, + last: NonNullable>, +) => + new WorkflowFailure({ + reason: "engine", + retryable: true, + step: authoredName(name).slice(0, 200), + errorName: "WorkflowEngineCallFailed", + message: `Executor called the workflow engine ${attempts} times for this ${ + kind === "sleep" ? "sleep" : "sleepUntil" + } and every call failed. The last failure: hop=${last.hop}; class=${last.errorClass}; code=${last.code}.`, + }); + +/** + * One sleep from the run body, with its own span. `durationMillis` is a `sleep`'s duration; a + * `sleepUntil` passes none, as its own check ends it once its time has passed (`sleep-until.ts`). + */ +export const repeatedSleep = ( + run: string, + kind: "sleep" | "until", + name: string, + call: () => Effect.Effect, + durationMillis: number | undefined, +) => + Effect.gen(function* () { + let attempts = 0; + let wait = firstRepeatMillis; + for (;;) { + attempts += 1; + const called = yield* Clock.currentTimeMillis; + const result = yield* Effect.result(call()); + if (Result.isSuccess(result)) { + yield* Effect.annotateCurrentSpan("executor.workflow.call.attempts", attempts); + return null; + } + const failure = result.failure; + const transient = transientEngineCall(failure); + const failed = engineCallOf(failure); + // A call that failed only after lasting the whole sleep: the sleep's time has passed, so it + // is over. Usually an engine waited it and lost only its answer; see the header for the rest. + const lasted = (yield* Clock.currentTimeMillis) - called; + const waited = durationMillis !== undefined && lasted >= durationMillis; + if (transient !== undefined && waited) { + yield* Effect.annotateCurrentSpan({ + "executor.workflow.call.attempts": attempts, + "executor.workflow.call.waited": true, + // How long the failed call lasted: at least the sleep's duration, or it would be repeated. + "executor.workflow.call.waited_ms": lasted, + }); + return null; + } + if (transient === undefined || attempts >= sleepAttempts) { + yield* Effect.annotateCurrentSpan({ + "executor.workflow.call.attempts": attempts, + "executor.workflow.call.outcome": transient === undefined ? "refused" : "failed", + ...(failed === undefined + ? {} + : { + "executor.workflow.call.hop": failed.hop, + "executor.workflow.call.error_class": failed.errorClass, + "executor.workflow.call.code": failed.code, + }), + }); + return yield* transient === undefined || attempts === 1 + ? Effect.fail(failure) + : Effect.fail(exhausted(kind, name, attempts, transient)); + } + const jittered = Math.round(wait * (0.8 + 0.4 * (yield* Random.next))); + // The repeat's own facts: what failed, and the wait the product scheduled before it. + yield* Effect.annotateCurrentSpan({ + [`executor.workflow.call.repeat_${attempts}.hop`]: transient.hop, + [`executor.workflow.call.repeat_${attempts}.code`]: transient.code, + [`executor.workflow.call.repeat_${attempts}.wait_ms`]: jittered, + }); + yield* Effect.sleep(jittered); + wait *= 2; + } + }).pipe( + Effect.withSpan(kind === "sleep" ? "workflow.sleep" : "workflow.sleepUntil", { + attributes: { "executor.run.id": run, "executor.workflow.step": name }, + }), + ); diff --git a/packages/sdk/src/implementation/workflows.ts b/packages/sdk/src/implementation/workflows.ts index a08ac0fa8e..d8256de410 100644 --- a/packages/sdk/src/implementation/workflows.ts +++ b/packages/sdk/src/implementation/workflows.ts @@ -393,9 +393,23 @@ export const makeWorkflowRuns = ( "execution", ).pipe(Effect.result); if (Result.isFailure(result)) { - if (result.failure.reason === "engine" && result.failure.retryable) - return yield* result.failure; const detail = workflowFailureDetail(result.failure); + if (result.failure.reason === "engine" && result.failure.retryable) { + // The engine retries steps, not the body, so this failure ends the run. One whose cause + // is known is recorded now, for its owner to read, while the engine reports the run + // running or waiting. A paused, terminated or other run is left to the engine, and an + // unreadable status to the next status read, as before. + if (detail !== undefined && backend !== undefined) { + const state = yield* backend.status(run).pipe(Effect.result); + if ( + Result.isSuccess(state) && + (state.success.status === "running" || state.success.status === "waiting") + ) + // A failed write leaves the run to the next status read, which records the reason. + yield* finish(run, { ok: false, error: "engine", detail }).pipe(Effect.ignore); + } + return yield* result.failure; + } yield* finish(run, { ok: false, error: result.failure.reason, diff --git a/packages/telemetry/collector/README.md b/packages/telemetry/collector/README.md index ea942acf35..9afd23cd69 100644 --- a/packages/telemetry/collector/README.md +++ b/packages/telemetry/collector/README.md @@ -1,8 +1,8 @@ # executor-collector The telemetry collector bundled with Executor, in Rust. It takes OTLP/HTTP JSON traces and logs -and keeps them in one SQLite file. It is not wired into the product yet: Motel -(`packages/telemetry/motel`) is still the collector every host runs. +and keeps them in one SQLite file. The local server, the desktop app, the self-host image and the +e2e harness all run it; it replaced Motel, the TypeScript collector they ran before. ## Build @@ -31,8 +31,31 @@ change; that job also builds, checks and smoke-tests the linux-x64 release execu `packages/telemetry/scripts/build-collector.ts` builds the executable that ships, for one release target, and is the only way it should be built for packaging (`bun run collector:build` builds for -this machine). It needs rustup, which installs the pinned toolchain and the target's standard -library itself. +this machine). It needs rustup and the pinned toolchain, and never installs either itself: without +them it stops and prints the one command that installs them (`rustup toolchain install 1.100.0 +--profile minimal`, or rustup's installer with that toolchain). `--install-toolchain` installs the +toolchain and the target's standard library, alone or before the steps; CI's `collector` action +runs it. + +A checkout keeps only `packages/telemetry/dist/collector/` (about 7 MB). Cargo's intermediate +files go to a shared cache outside every checkout: `$EXECUTOR_COLLECTOR_CACHE` (absolute), or +`executor-collector` in the platform's cache directory (`$XDG_CACHE_HOME` or `~/.cache` on Linux, +`~/Library/Caches` on macOS, `%LOCALAPPDATA%` on Windows). There `target//` holds one checkout's cargo output (about 160 MB); those not used for 14 days are removed, +and past 3 GiB in all the least recently used go first, never one used in the last hour. Outside CI, +`store///` keeps each finished build, at most 32 a target and none unused for 30 +days: a checkout whose crate has an identity already stored copies that build in a fraction of a +second instead of compiling. Entries are written beside the store and renamed into it, and handed +out by copying, never by linking, so nothing in a checkout can write through to the store. Each +build records its executable's SHA-256 and size in `executable.sha256`; a checkout's build or a +stored one whose executable does not match is not used, and a damaged stored entry is removed. A +partial copy whose writer has exited, or older than an hour, is removed, and partial copies count +against the bound. CI never +reads or writes the store; it builds once a run (`collector` in `checks.yml`, `collector` in the +release workflow) and every job fetches that build. + +Measured on minisforum (16 cores): a cold build 44 s; after one change in the crate 16.6 s; with +the crate unchanged 0.15 s; a new checkout of a crate already stored 0.30 s. ```sh node packages/telemetry/scripts/build-collector.ts [--target ] build check smoke @@ -100,6 +123,7 @@ error. ```sh executor-collector serve --data [--listen 127.0.0.1:0 | --listen unix:] [--max-pending 4] [--max-export-bytes 16777216] [--retention-hours 168] [--max-store-mb 1024] + [--viewer ] ``` Once it is listening it prints one line on stdout, `{"event":"listening","port":N}` for TCP or @@ -138,19 +162,19 @@ replacement made earlier. The check and the removal are two steps, so this holds else replaces files in that directory. The supervisor's private directory, with one collector at a time, guarantees that. -It exits with code 1, before printing the listening line, when it cannot open or create the store -or cannot listen. +It exits with code 1, before printing the listening line, when it cannot open or create the store, +cannot listen, or cannot serve the viewer `--viewer` names (see The viewer). ## Routes -| Route | Answer | -| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `POST /v1/traces` | `{insertedSpans}`: every span of the export, including resent spans that were already stored | -| `POST /v1/logs` | `{insertedLogs}` | -| `GET /api/health` | `{ok, service, version}` | -| `GET /api/ingest` | `{pending, maxPending, maxBytes, refused: {queueFull, tooLarge, invalid, storeFailed, overBound}, refusedBytes, lastRefusedAt, evictions: {passes, traces, spans, logs}, store: {...}}` | +| Route | Answer | +| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `POST /v1/traces` | `{insertedSpans}`: every span of the export, including resent spans that were already stored | +| `POST /v1/logs` | `{insertedLogs}` | +| `GET /api/health` | `{ok, service, version}` | +| `GET /api/ingest` | `{pending, maxPending, maxBytes, refused: {queueFull, tooLarge, invalid, storeFailed, overBound, tooMuchDisk}, refusedBytes, lastRefusedAt, evictions: {passes, traces, spans, logs}, store: {...}}` | -Anything else answers 404 `{error}`. The viewer's files are served by a later change. +With `--viewer`, the viewer's routes too (see The viewer). Anything else answers 404 `{error}`. `/api/ingest` also has `droppedEvents`: span events not stored because their span had more than 128 (see Field mapping), in exports that were stored, kept with the refusal counts. And @@ -184,6 +208,48 @@ store is and whether retention has finished: and `logs` what they removed, all since the collector started. A pass is counted when its first trace is gone, so a trace the caller sees missing means `passes` already counts its pass. +### The viewer + +`--viewer ` serves the trace viewer's build (`packages/telemetry/viewer`, built into +`packages/telemetry/dist/viewer`) at the origin root, beside the API it reads: + +| Route | Answer | +| ------------------------------------------------ | -------------------------------------------------------------------------------------- | +| `GET /`, `/traces`, `/logs`, `/trace/` | `index.html`, `text/html; charset=utf-8`, `Cache-Control: no-cache` | +| `GET /assets/` | the file, typed by its extension, `Cache-Control: public, max-age=31536000, immutable` | + +- **Read once, at start.** Before the store is opened, the collector reads `index.html` and every + regular file under `assets/` (at most four directories deep) into memory. A request is looked up + by its path in that table and answered from memory, shared by every answer, so nothing a request + carries ever becomes a path on disk: `..`, encoded separators (`%2F`, `%5C`, `%2e`), absolute + paths and links in a request reach nothing, no directory is listed, and a request holds no memory + of its own beyond its response's head. A file added or changed after the start is not served + until the collector restarts. Only the routes above are served, `/index.html` included as not + found. +- **The directory is trusted.** It is part of the installation, named by whatever starts the + collector, and whoever can write to it already decides what the viewer serves. Reading it guards + against a directory assembled by mistake, not against someone changing it while the collector + starts. Refused, ending the start: a symbolic link anywhere inside the directory (each file is + opened without following a final link, `O_NOFOLLOW` on Unix, the link itself on Windows, so no + check stands apart from the open), and anything that is not a regular file or a directory (the + opened file's own kind decides; a FIFO does not block the open). Not refused: a hard link, which + is a regular file (package stores and deduplicating file systems install files that way), and + the directory itself being reached through a link. +- **Bounded before it is allocated.** At most `VIEWER_ENTRIES` (4,096) files and directories, + `index.html` and `assets` included, and at most `VIEWER_BYTES` (16 MiB) in all, counting each + file's contents, each entry's stored path (its whole path under `assets/`) and 256 bytes for each + entry; the build is about 0.6 MB in three files. Reading a file stops at what is left of the + bound, so a larger file is not read past it. Past either bound, or with a missing directory or + `index.html` or a file it cannot read, the start ends with exit 1 and `cannot serve the viewer` + in the log, before the data directory is touched. An empty `--viewer` is a usage error (exit 2). +- **Methods.** `GET`, and `HEAD`, which answers the same head without the body. Any other method + on these routes answers 405. +- **Caching.** `index.html` is `no-cache`, so a browser asks again each time and finds the new + asset names after an upgrade. Vite names every asset by a hash of its content, so a name always + holds the same bytes and the browser may keep it for a year. Every file is sent with + `X-Content-Type-Options: nosniff`. +- Without `--viewer` these routes answer 404, as any unknown route. + ### Admission Bodies may be uncompressed or gzip (`Content-Encoding: gzip`), which every OTLP/HTTP server must @@ -532,7 +598,7 @@ the directory at most 105.9 MB. The routes the product's tests, the e2e harness and the viewer call, with Motel's response shapes, limits, clamping and lookback rules (Motel's `domain.ts`, `httpApi.ts`, `httpListPolicy.ts` and -`TelemetryStore.ts` at the revision `packages/telemetry/motel/source.json` pins): +`TelemetryStore.ts` in [`UsefulSoftwareCo/motel`](https://github.com/UsefulSoftwareCo/motel) at `b1783f4d344fd2e0fa584ef00b3e71838a8a1e03`): | Route | Parameters | Answer | | --------------------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------------------------------------- | @@ -727,8 +793,9 @@ deliberate or does not reach any caller: and `groupBy=attr.*` counts every trace as `unknown`. A malformed stats request answers 400 with an `{error}` body; Motel's had none. - **Dropped routes and parameters.** `/api/traces/search`, `/api/spans/:spanId`, - `/api/spans/:spanId/logs`, `/api/logs/stats`, `/api/facets`, `/api/docs`, `/api/ai/*`, - `/trace/:traceId`, `/openapi.json` and `/` answer 404. `attrContains.*`, `parentOperation`, + `/api/spans/:spanId/logs`, `/api/logs/stats`, `/api/facets`, `/api/docs`, `/api/ai/*` and + `/openapi.json` answer 404; `/` and `/trace/:traceId` serve the viewer's page with `--viewer` + and answer 404 without it. `attrContains.*`, `parentOperation`, `status` on searches, `minDurationMs` and `aiText` are ignored (a repeated or malformed `status` or `minDurationMs` answers 400, as in Motel). - **Ids.** A trace or span id in a path or filter must be lowercase hex, as Motel stored and diff --git a/packages/telemetry/collector/src/args.rs b/packages/telemetry/collector/src/args.rs index 45bdf5199f..b8c742fd30 100644 --- a/packages/telemetry/collector/src/args.rs +++ b/packages/telemetry/collector/src/args.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; pub const USAGE: &str = "usage: executor-collector serve --data [--listen <127.0.0.1:port | unix:path>] [--max-pending ] [--max-export-bytes ] [--retention-hours ] [--max-store-mb ] + [--viewer ] --data directory holding telemetry.sqlite (created if missing) --listen address to listen on (default 127.0.0.1:0); port 0 picks a free port @@ -12,6 +13,7 @@ pub const USAGE: &str = "usage: executor-collector serve --data [--listen ones are refused with 413. A store started with a larger one is emptied --retention-hours how long traces and logs are kept after they were last written (default 168) --max-store-mb the most data the store keeps, in MiB; the oldest is evicted first (default 1024) + --viewer directory of the trace viewer's build, served at / (default: none) Once listening, prints one line on stdout: {\"event\":\"listening\",\"port\":N} for TCP or {\"event\":\"listening\",\"path\":P} for a Unix socket."; @@ -30,6 +32,7 @@ pub struct Args { pub max_export_bytes: usize, pub retention_hours: u64, pub max_store_mb: u64, + pub viewer: Option, } pub fn parse(mut args: impl Iterator) -> Result { @@ -44,6 +47,7 @@ pub fn parse(mut args: impl Iterator) -> Result { let mut max_export_bytes = 16 * 1024 * 1024; let mut retention_hours = 168; let mut max_store_mb = 1024; + let mut viewer = None; while let Some(flag) = args.next() { let (name, inline) = match flag.split_once('=') { Some((name, value)) => (name.to_owned(), Some(value.to_owned())), @@ -58,6 +62,8 @@ pub fn parse(mut args: impl Iterator) -> Result { }; match name.as_str() { "--data" => data = Some(PathBuf::from(value)), + "--viewer" if value.is_empty() => return Err("--viewer needs a directory".to_owned()), + "--viewer" => viewer = Some(PathBuf::from(value)), "--listen" => { listen = match value.strip_prefix("unix:") { Some(path) if !path.is_empty() => Listen::Unix(PathBuf::from(path)), @@ -90,6 +96,7 @@ pub fn parse(mut args: impl Iterator) -> Result { max_export_bytes, retention_hours, max_store_mb, + viewer, }) } diff --git a/packages/telemetry/collector/src/http.rs b/packages/telemetry/collector/src/http.rs index f98ecc4ed2..9d26314e10 100644 --- a/packages/telemetry/collector/src/http.rs +++ b/packages/telemetry/collector/src/http.rs @@ -27,6 +27,7 @@ use crate::otlp; use crate::query::{self, LogFilter, Query}; use crate::readers::Readers; use crate::store::{self, Export, Job, Losses, MemoryRelease, Shared, SharedLosses}; +use crate::viewer::Viewer; const REFUSAL_LOG_INTERVAL: Duration = Duration::from_secs(60); @@ -41,6 +42,8 @@ pub struct App { /// The writer's figures: the store's size for admission and `/api/ingest`. pub store: Arc, pub readers: Arc, + /// The trace viewer's files, when `--viewer` names them. + pub viewer: Option>, } /// An admitted export: its permit, held until the export is stored or refused. Dropping it, on @@ -57,7 +60,32 @@ impl Drop for Admitted { } pub fn router(app: Arc) -> Router { - Router::new() + let mut router = Router::new(); + if let Some(viewer) = &app.viewer { + // The viewer's own routes, as its README lists them. `get` answers HEAD too; any other + // method is 405. + let index = { + let viewer = viewer.clone(); + move || { + let viewer = viewer.clone(); + async move { viewer.index() } + } + }; + let asset = { + let viewer = viewer.clone(); + move |Path(name): Path| { + let viewer = viewer.clone(); + async move { viewer.asset(&name) } + } + }; + router = router + .route("/", get(index.clone())) + .route("/traces", get(index.clone())) + .route("/logs", get(index.clone())) + .route("/trace/{trace_id}", get(index)) + .route("/assets/{*name}", get(asset)); + } + router .route("/v1/traces", post(traces)) .route("/v1/logs", post(logs)) .route("/api/health", get(health)) diff --git a/packages/telemetry/collector/src/lib.rs b/packages/telemetry/collector/src/lib.rs index ab421d1bcd..dd22396b01 100644 --- a/packages/telemetry/collector/src/lib.rs +++ b/packages/telemetry/collector/src/lib.rs @@ -15,3 +15,4 @@ pub mod retention; pub mod store; pub mod text; mod unicode61; +pub mod viewer; diff --git a/packages/telemetry/collector/src/main.rs b/packages/telemetry/collector/src/main.rs index 7832d5a13d..b1b908a7d8 100644 --- a/packages/telemetry/collector/src/main.rs +++ b/packages/telemetry/collector/src/main.rs @@ -13,7 +13,7 @@ use tokio::sync::Semaphore; use executor_collector::args::{self, Listen}; use executor_collector::lifecycle::{self, Stop}; -use executor_collector::{connection, http, log, otlp, readers, retention, store}; +use executor_collector::{connection, http, log, otlp, readers, retention, store, viewer}; /// The whole shutdown, from the stop signal until the process exits. The local host kills the /// collector 3 s after asking it to stop (the self-host supervisor waits 15 s); this leaves half a @@ -51,6 +51,14 @@ fn main() -> ExitCode { return ExitCode::FAILURE; } }; + // Before the store: a viewer that cannot be served ends the start, changing nothing. + let viewer = match args.viewer.as_deref().map(viewer::Viewer::load).transpose() { + Ok(viewer) => viewer.map(Arc::new), + Err(error) => { + log::error("cannot serve the viewer", &[("error", error.into())]); + return ExitCode::FAILURE; + } + }; let losses = Arc::new(Mutex::new(store::LossBook::default())); let release = Arc::new(store::MemoryRelease::default()); let shared = Arc::new(store::Shared::default()); @@ -124,6 +132,7 @@ fn main() -> ExitCode { release, store: shared, readers: readers.clone(), + viewer, }); let stopped_at = Arc::new(Mutex::new(None)); let served = runtime.block_on(serve(args.listen.clone(), app, stopped_at.clone())); diff --git a/packages/telemetry/collector/src/viewer.rs b/packages/telemetry/collector/src/viewer.rs new file mode 100644 index 0000000000..47b23d5062 --- /dev/null +++ b/packages/telemetry/collector/src/viewer.rs @@ -0,0 +1,251 @@ +//! The trace viewer's files (`--viewer `), served at the origin root beside the API. +//! +//! The directory is read once, at start: its `index.html` and the regular files under `assets/`, +//! at most `ASSET_DEPTH` directories deep. A request is looked up by name in that table and +//! answered from memory, so nothing a request carries ever becomes a path on disk. +//! +//! The directory is part of the installation and is trusted: whoever can write to it already +//! decides what the viewer serves. Reading it guards against a directory assembled by mistake, not +//! against someone changing it while the collector starts. A symbolic link anywhere inside it, and +//! anything that is not a regular file or a directory, ends the start; each file is opened without +//! following a final symbolic link, so no check stands apart from the open. A hard link is a regular +//! file and is served, and the directory itself may be reached through a link. +//! +//! What is read is bounded before it is allocated: at most `VIEWER_ENTRIES` files and directories, +//! `index.html` and `assets` included, and at most `VIEWER_BYTES` in all, counting each file's +//! contents, each entry's stored path (its whole path under `assets/`) and `ENTRY_BYTES` for each +//! entry. A directory past either bound ends the start, before the data directory is touched. + +use std::collections::HashMap; +use std::path::Path; + +use axum::body::{Body, Bytes}; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; + +/// The most the viewer's files may hold in all, names and entries counted. The build is about +/// 0.6 MB in three files. +pub const VIEWER_BYTES: u64 = 16 * 1024 * 1024; +/// The most files and directories the viewer's directory may hold. The build has three files. +pub const VIEWER_ENTRIES: usize = 4096; +/// What one entry of the table costs beyond its name and contents: its map slot, key and value. +const ENTRY_BYTES: u64 = 256; +/// How deep `assets/` may nest. Vite writes it flat. +const ASSET_DEPTH: usize = 4; + +pub struct Viewer { + index: Bytes, + /// By their path under `assets/`, with `/` between names. + assets: HashMap, +} + +struct Asset { + bytes: Bytes, + content_type: &'static str, +} + +impl Viewer { + /// Read the viewer's files from `directory`, or say why they cannot be served. + pub fn load(directory: &Path) -> Result { + let mut total = 0u64; + let mut count = 0usize; + // `index.html` and the `assets` directory are entries like any other. + admit(&mut count, &mut total, "index.html".len())?; + let index = read_file(&directory.join("index.html"), &mut total)?; + let mut assets = HashMap::new(); + let root = directory.join("assets"); + match std::fs::symlink_metadata(&root) { + Ok(metadata) if metadata.is_dir() => { + admit(&mut count, &mut total, "assets".len())?; + read_assets(&root, "", 0, &mut assets, &mut total, &mut count)? + } + Ok(_) => return Err(format!("{} is not a directory", root.display())), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("cannot read {}: {error}", root.display())), + } + Ok(Viewer { + index: Bytes::from(index), + assets, + }) + } + + /// `index.html`, for the viewer's own routes; the browser's router takes it from there. + pub fn index(&self) -> Response { + file(self.index.clone(), "text/html; charset=utf-8", "no-cache") + } + + /// A file under `assets/`, or 404. Vite names every one by a hash of its content, so a name + /// always means the same bytes and the browser may keep it. + pub fn asset(&self, name: &str) -> Response { + match self.assets.get(name) { + Some(asset) => file( + asset.bytes.clone(), + asset.content_type, + "public, max-age=31536000, immutable", + ), + None => ( + StatusCode::NOT_FOUND, + axum::Json(serde_json::json!({ "error": "Not found" })), + ) + .into_response(), + } + } +} + +fn file(bytes: Bytes, content_type: &'static str, cache: &'static str) -> Response { + let mut response = Body::from(bytes).into_response(); + let headers = response.headers_mut(); + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(content_type)); + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static(cache)); + headers.insert( + header::X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ); + response +} + +/// The open flags that refuse a symbolic link as the final component, and keep a FIFO or a device +/// from blocking the open; the handle's own metadata then says what was opened. +#[cfg(unix)] +fn open_regular(path: &Path) -> std::io::Result { + use std::os::unix::fs::OpenOptionsExt; + #[cfg(any(target_os = "linux", target_os = "android"))] + const NO_FOLLOW: i32 = 0o400_000; + #[cfg(any(target_os = "linux", target_os = "android"))] + const NON_BLOCK: i32 = 0o4000; + #[cfg(target_vendor = "apple")] + const NO_FOLLOW: i32 = 0x0100; + #[cfg(target_vendor = "apple")] + const NON_BLOCK: i32 = 0x0004; + std::fs::OpenOptions::new() + .read(true) + .custom_flags(NO_FOLLOW | NON_BLOCK) + .open(path) +} + +/// On Windows the link itself is opened (`FILE_FLAG_OPEN_REPARSE_POINT`), and its metadata says so. +#[cfg(windows)] +fn open_regular(path: &Path) -> std::io::Result { + use std::os::windows::fs::OpenOptionsExt; + const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000; + std::fs::OpenOptions::new() + .read(true) + .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) + .open(path) +} + +/// Counts one more entry against `VIEWER_ENTRIES`, and its stored name of `name_bytes` and +/// `ENTRY_BYTES` against `VIEWER_BYTES`, before either is allocated. +fn admit(count: &mut usize, total: &mut u64, name_bytes: usize) -> Result<(), String> { + *count += 1; + if *count > VIEWER_ENTRIES { + return Err(format!( + "the viewer's directory holds more than {VIEWER_ENTRIES} files and directories" + )); + } + charge(total, ENTRY_BYTES + name_bytes as u64) +} + +/// Counts `bytes` against `VIEWER_BYTES`, before they are allocated. +fn charge(total: &mut u64, bytes: u64) -> Result<(), String> { + *total = total.saturating_add(bytes); + if *total > VIEWER_BYTES { + return Err(format!( + "the viewer's files hold more than {VIEWER_BYTES} bytes, names and entries counted" + )); + } + Ok(()) +} + +/// One regular file, opened without following a link, read only up to what is left of +/// `VIEWER_BYTES`: a file that grows while it is read still cannot take more. +fn read_file(path: &Path, total: &mut u64) -> Result, String> { + use std::io::Read; + let unreadable = |error: std::io::Error| format!("cannot read {}: {error}", path.display()); + let file = open_regular(path).map_err(|error| { + // ELOOP on Linux and macOS: the final component is a symbolic link. + if error.raw_os_error() == Some(40) || error.raw_os_error() == Some(62) { + format!("{} is a symbolic link", path.display()) + } else { + unreadable(error) + } + })?; + let kind = file.metadata().map_err(unreadable)?.file_type(); + if kind.is_symlink() { + return Err(format!("{} is a symbolic link", path.display())); + } + if !kind.is_file() { + return Err(format!("{} is not a regular file", path.display())); + } + let left = VIEWER_BYTES.saturating_sub(*total); + let mut bytes = Vec::new(); + file.take(left + 1) + .read_to_end(&mut bytes) + .map_err(unreadable)?; + charge(total, bytes.len() as u64)?; + Ok(bytes) +} + +fn read_assets( + directory: &Path, + prefix: &str, + depth: usize, + assets: &mut HashMap, + total: &mut u64, + count: &mut usize, +) -> Result<(), String> { + if depth > ASSET_DEPTH { + return Err(format!("{} nests too deep", directory.display())); + } + let entries = std::fs::read_dir(directory) + .map_err(|error| format!("cannot read {}: {error}", directory.display()))?; + for entry in entries { + let entry = + entry.map_err(|error| format!("cannot read {}: {error}", directory.display()))?; + let path = entry.path(); + let name = entry + .file_name() + .into_string() + .map_err(|_| format!("{} is not named in UTF-8", path.display()))?; + // The key the table stores for it: its whole path under `assets/`. + admit(count, total, prefix.len() + name.len())?; + let metadata = std::fs::symlink_metadata(&path) + .map_err(|error| format!("cannot read {}: {error}", path.display()))?; + let key = format!("{prefix}{name}"); + if metadata.is_dir() { + read_assets(&path, &format!("{key}/"), depth + 1, assets, total, count)?; + } else if metadata.is_file() { + let bytes = read_file(&path, total)?; + assets.insert( + key, + Asset { + bytes: Bytes::from(bytes), + content_type: content_type(&name), + }, + ); + } else { + return Err(format!( + "{} is not a regular file or directory", + path.display() + )); + } + } + Ok(()) +} + +fn content_type(name: &str) -> &'static str { + let extension = name.rsplit_once('.').map_or("", |(_, extension)| extension); + match extension.to_ascii_lowercase().as_str() { + "js" | "mjs" => "text/javascript; charset=utf-8", + "css" => "text/css; charset=utf-8", + "html" => "text/html; charset=utf-8", + "json" | "map" => "application/json", + "svg" => "image/svg+xml", + "png" => "image/png", + "ico" => "image/x-icon", + "woff2" => "font/woff2", + "woff" => "font/woff", + "txt" => "text/plain; charset=utf-8", + _ => "application/octet-stream", + } +} diff --git a/packages/telemetry/collector/tests/process.rs b/packages/telemetry/collector/tests/process.rs index 4b08ad1275..d07a8d883f 100644 --- a/packages/telemetry/collector/tests/process.rs +++ b/packages/telemetry/collector/tests/process.rs @@ -5324,3 +5324,453 @@ fn an_export_that_writes_too_much_to_disk_is_refused_and_leaves_the_store_as_it_ assert_eq!(after[2], before[2] + 10); assert_eq!((after[4], after[5]), (before[4], before[5])); } + +/// A viewer build in `directory`: `index.html` and two hashed assets, one of them nested. +fn viewer_build(directory: &Path) { + std::fs::create_dir_all(directory.join("assets/fonts")).expect("assets"); + std::fs::write( + directory.join("index.html"), + "viewer", + ) + .expect("index"); + std::fs::write(directory.join("assets/index-abc123.js"), "console.log(1)").expect("js"); + std::fs::write(directory.join("assets/index-def456.css"), "body{}").expect("css"); + std::fs::write(directory.join("assets/fonts/inter-0a1b.woff2"), [0u8, 1, 2]).expect("font"); +} + +/// One request on its own connection, read raw: the status, the head lowercased and the body. +fn raw_request(address: &Address, raw: &str) -> (u16, String, Vec) { + let Address::Tcp(port) = address else { + panic!("TCP only") + }; + let mut stream = TcpStream::connect(("127.0.0.1", *port)).expect("connect"); + stream.set_read_timeout(Some(IO_LIMIT)).expect("timeout"); + stream.write_all(raw.as_bytes()).expect("send"); + let mut response = Vec::new(); + stream.read_to_end(&mut response).expect("receive"); + let (status, head, body) = split_response(&response); + (status, head, body.to_vec()) +} + +fn raw_get(address: &Address, method: &str, target: &str) -> (u16, String, Vec) { + raw_request( + address, + &format!( + "{method} {target} HTTP/1.1\r\nHost: collector\r\nConnection: close\r\nContent-Length: 0\r\n\r\n" + ), + ) +} + +#[test] +fn the_viewer_is_served_from_what_was_read_at_start_and_nothing_outside_it() { + let scratch = Scratch::new("viewer-routes"); + let viewer = scratch.0.join("viewer"); + viewer_build(&viewer); + // Files beside and above the viewer that no request may reach. + std::fs::write(scratch.0.join("secret.txt"), "outside").expect("secret"); + // A hard link is a regular file: the directory is trusted, and package stores install files + // this way, so it is served. + std::fs::write(scratch.0.join("shared.js"), "hard-linked").expect("shared"); + std::fs::hard_link( + scratch.0.join("shared.js"), + viewer.join("assets/shared-0000.js"), + ) + .expect("hard link"); + // Four directories deep is the deepest served. + std::fs::create_dir_all(viewer.join("assets/1/2/3/4")).expect("deep"); + std::fs::write(viewer.join("assets/1/2/3/4/deep-0000.js"), "deep").expect("deep file"); + let collector = Collector::start( + &scratch.data(), + &["--viewer", viewer.to_str().expect("utf-8")], + ); + let address = collector.address.clone(); + for route in [ + "/", + "/traces", + "/logs", + "/trace/0123456789abcdef0123456789abcdef", + ] { + let (status, head, body) = raw_get(&address, "GET", route); + assert_eq!(status, 200, "{route}"); + assert!( + head.contains("content-type: text/html; charset=utf-8"), + "{route}: {head}" + ); + assert!(head.contains("cache-control: no-cache"), "{route}: {head}"); + assert!( + head.contains("x-content-type-options: nosniff"), + "{route}: {head}" + ); + assert_eq!(body, b"viewer", "{route}"); + } + for (asset, content_type, bytes) in [ + ( + "/assets/index-abc123.js", + "text/javascript; charset=utf-8", + &b"console.log(1)"[..], + ), + ( + "/assets/index-def456.css", + "text/css; charset=utf-8", + b"body{}", + ), + ("/assets/fonts/inter-0a1b.woff2", "font/woff2", &[0, 1, 2]), + ( + "/assets/shared-0000.js", + "text/javascript; charset=utf-8", + b"hard-linked", + ), + ( + "/assets/1/2/3/4/deep-0000.js", + "text/javascript; charset=utf-8", + b"deep", + ), + ] { + let (status, head, body) = raw_get(&address, "GET", asset); + assert_eq!(status, 200, "{asset}"); + assert!( + head.contains(&format!("content-type: {content_type}")), + "{asset}: {head}" + ); + assert!( + head.contains("cache-control: public, max-age=31536000, immutable"), + "{asset}: {head}" + ); + assert!( + head.contains("x-content-type-options: nosniff"), + "{asset}: {head}" + ); + assert_eq!(body, bytes, "{asset}"); + } + // HEAD answers the head alone; every other method is refused. + let (status, head, body) = raw_get(&address, "HEAD", "/assets/index-abc123.js"); + assert_eq!(status, 200); + assert!(head.contains("content-length: 14"), "{head}"); + assert!(body.is_empty()); + for method in ["POST", "PUT", "DELETE", "PATCH", "OPTIONS"] { + for route in ["/", "/assets/index-abc123.js"] { + let (status, _, _) = raw_get(&address, method, route); + assert_eq!(status, 405, "{method} {route}"); + } + } + // Nothing a request names becomes a path: none of these reaches a file, in the viewer's + // directory or out of it, and none lists a directory. + for target in [ + "/assets/../index.html", + "/assets/..%2Findex.html", + "/assets/%2e%2e/index.html", + "/assets/..%2F..%2Fsecret.txt", + "/assets/%2E%2E%2F%2E%2E%2Fsecret.txt", + "/assets/..%5C..%5Csecret.txt", + "/assets//etc/passwd", + "/assets/%2Fetc%2Fpasswd", + "/assets/fonts/../../index.html", + "/assets/fonts", + "/assets/fonts/", + "/assets/", + "/assets", + "/index.html", + "/secret.txt", + "/viewer/index.html", + "/trace/", + "/trace/a/b", + ] { + let (status, _, body) = raw_get(&address, "GET", target); + assert_eq!(status, 404, "{target}: {}", String::from_utf8_lossy(&body)); + assert!(!body.windows(7).any(|w| w == b"outside"), "{target}"); + assert!(!body.windows(8).any(|w| w == b"doctype "), "{target}"); + } + // Files are read once, at start: a file added, changed or linked in since is not served. + std::fs::write(viewer.join("assets/late-0000.js"), "late").expect("late"); + std::os::unix::fs::symlink( + scratch.0.join("secret.txt"), + viewer.join("assets/link-0000.txt"), + ) + .expect("link"); + std::fs::write(viewer.join("index.html"), "changed").expect("change"); + assert_eq!(raw_get(&address, "GET", "/assets/late-0000.js").0, 404); + assert_eq!(raw_get(&address, "GET", "/assets/link-0000.txt").0, 404); + assert_eq!( + raw_get(&address, "GET", "/").2, + b"viewer" + ); + // The API is unchanged beside it. + assert_eq!(collector.get("/api/health").0, 200); + let (status, _, _) = collector.stop(); + assert!(status.success()); +} + +#[test] +fn without_a_viewer_its_routes_are_not_found() { + let scratch = Scratch::new("viewer-absent"); + let collector = Collector::start(&scratch.data(), &[]); + for route in ["/", "/traces", "/logs", "/trace/abc", "/assets/index.js"] { + assert_eq!(raw_get(&collector.address, "GET", route).0, 404, "{route}"); + } + let (status, _, _) = collector.stop(); + assert!(status.success()); +} + +#[test] +fn a_viewer_that_cannot_be_served_ends_the_start_before_the_store_is_touched() { + let scratch = Scratch::new("viewer-refused"); + std::fs::write(scratch.0.join("outside.js"), "outside").expect("outside"); + type Make = Box; + let cases: Vec<(&str, Make, &str)> = vec![ + ("missing", Box::new(|_: &Path| {}), "index.html"), + ( + "no index", + Box::new(|dir: &Path| { + viewer_build(dir); + std::fs::remove_file(dir.join("index.html")).expect("rm"); + }), + "index.html", + ), + ( + "index a directory", + Box::new(|dir: &Path| { + viewer_build(dir); + std::fs::remove_file(dir.join("index.html")).expect("rm"); + std::fs::create_dir(dir.join("index.html")).expect("dir"); + }), + "not a regular file", + ), + ( + "index a link", + Box::new(|dir: &Path| { + viewer_build(dir); + std::fs::remove_file(dir.join("index.html")).expect("rm"); + std::os::unix::fs::symlink( + dir.join("assets/index-abc123.js"), + dir.join("index.html"), + ) + .expect("link"); + }), + // Refused by the open itself, which does not follow a final symbolic link. + "is a symbolic link", + ), + ( + "an asset links out", + Box::new(|dir: &Path| { + viewer_build(dir); + std::os::unix::fs::symlink( + dir.parent().expect("parent").join("outside.js"), + dir.join("assets/out-0000.js"), + ) + .expect("link"); + }), + "not a regular file or directory", + ), + ( + "assets a link to a directory", + Box::new(|dir: &Path| { + std::fs::create_dir_all(dir).expect("dir"); + std::fs::write(dir.join("index.html"), "x").expect("index"); + std::os::unix::fs::symlink(dir.parent().expect("parent"), dir.join("assets")) + .expect("link"); + }), + "is not a directory", + ), + ( + "a nested link to a directory", + Box::new(|dir: &Path| { + viewer_build(dir); + std::os::unix::fs::symlink(dir.parent().expect("parent"), dir.join("assets/up")) + .expect("link"); + }), + "not a regular file or directory", + ), + ( + // The open does not block on it, and the opened file's own kind refuses it. + "index a FIFO", + Box::new(|dir: &Path| { + viewer_build(dir); + std::fs::remove_file(dir.join("index.html")).expect("rm"); + let made = Command::new("mkfifo") + .arg(dir.join("index.html")) + .status() + .expect("mkfifo"); + assert!(made.success()); + }), + "not a regular file", + ), + ( + "five deep", + Box::new(|dir: &Path| { + viewer_build(dir); + std::fs::create_dir_all(dir.join("assets/1/2/3/4/5")).expect("deep"); + std::fs::write(dir.join("assets/1/2/3/4/5/deep.js"), "x").expect("deep file"); + }), + "nests too deep", + ), + ( + "too many entries", + Box::new(|dir: &Path| { + viewer_build(dir); + for index in 0..executor_collector::viewer::VIEWER_ENTRIES { + std::fs::write(dir.join(format!("assets/empty-{index:05}.js")), "") + .expect("empty"); + } + }), + "more than 4096 files and directories", + ), + ( + "a FIFO", + Box::new(|dir: &Path| { + viewer_build(dir); + let made = Command::new("mkfifo") + .arg(dir.join("assets/pipe-0000.js")) + .status() + .expect("mkfifo"); + assert!(made.success()); + }), + "not a regular file or directory", + ), + ( + "over the size bound", + Box::new(|dir: &Path| { + viewer_build(dir); + // Sparse: it takes no disk. + std::fs::File::create(dir.join("assets/huge-0000.js")) + .expect("create") + .set_len(executor_collector::viewer::VIEWER_BYTES) + .expect("set_len"); + }), + "more than", + ), + ]; + for (name, make, reason) in cases { + let viewer = scratch.0.join(name.replace(' ', "-")); + make(&viewer); + let data = scratch.0.join(format!("data-{}", name.replace(' ', "-"))); + let (status, stderr) = + match Collector::try_start(&data, &["--viewer", viewer.to_str().expect("utf-8")]) { + Ok(_) => panic!("{name}: the collector started"), + Err(failed) => failed, + }; + assert_eq!(status.code(), Some(1), "{name}: {stderr}"); + assert!( + stderr.contains("cannot serve the viewer"), + "{name}: {stderr}" + ); + assert!(stderr.contains(reason), "{name}: {stderr}"); + assert!(!data.exists(), "{name}: the data directory was created"); + } + // A file far past the bound is read no further than the bound. The collector starts in about + // 16 MiB of address space; under a 160 MiB limit a 200 MiB file read whole fails to allocate and + // aborts, where the bound refuses it. 200 MiB stays under the 256 MiB file-size limit this suite + // runs under on shared hosts (`ulimit -f 262144`); the file is sparse and takes no disk. + let viewer = scratch.0.join("vast"); + viewer_build(&viewer); + std::fs::File::create(viewer.join("assets/vast-0000.js")) + .expect("create") + .set_len(200 << 20) + .expect("set_len"); + let data = scratch.0.join("data-vast"); + let mut command = Command::new("bash"); + command.args([ + "-c", + "ulimit -v 163840; exec \"$0\" \"$@\"", + BINARY, + "serve", + "--data", + ]); + command + .arg(&data) + .args(["--listen", "127.0.0.1:0", "--viewer"]) + .arg(&viewer); + let (status, stderr) = match Collector::spawn(&mut command) { + Ok(_) => panic!("a 200 MiB viewer started"), + Err(failed) => failed, + }; + assert_eq!(status.code(), Some(1), "{stderr}"); + assert!(stderr.contains("more than"), "{stderr}"); + assert!(!data.exists()); + // An empty value is a usage error. + let (status, _) = match Collector::try_start(&scratch.data(), &["--viewer", ""]) { + Ok(_) => panic!("an empty --viewer started"), + Err(failed) => failed, + }; + assert_eq!(status.code(), Some(2)); +} + +/// Starts the collector on `viewer`; whether it started, and its log when it did not. +fn viewer_starts(scratch: &Scratch, name: &str, viewer: &Path) -> Result<(), String> { + let data = scratch.0.join(format!("data-{name}")); + let started = Collector::try_start(&data, &["--viewer", viewer.to_str().expect("utf-8")]); + let _ = std::fs::remove_dir_all(&data); + match started { + Ok(collector) => { + let (status, _, _) = collector.stop(); + assert!(status.success(), "{name}"); + Ok(()) + } + Err((status, stderr)) => { + assert_eq!(status.code(), Some(1), "{name}: {stderr}"); + Err(stderr) + } + } +} + +#[test] +fn the_viewer_bound_counts_every_entry_its_stored_path_and_its_contents() { + use executor_collector::viewer::{VIEWER_BYTES, VIEWER_ENTRIES}; + let scratch = Scratch::new("viewer-bound"); + // Entries: `index.html` and `assets` count like the files under it. + let entries = |name: &str, files: usize| { + let viewer = scratch.0.join(name); + std::fs::create_dir_all(viewer.join("assets")).expect("assets"); + std::fs::write(viewer.join("index.html"), "x").expect("index"); + for index in 0..files { + std::fs::write(viewer.join(format!("assets/f{index:05}.js")), "").expect("file"); + } + viewer + }; + let at = entries("entries-at", VIEWER_ENTRIES - 2); + assert_eq!(viewer_starts(&scratch, "entries-at", &at), Ok(())); + let over = entries("entries-over", VIEWER_ENTRIES - 1); + let refused = viewer_starts(&scratch, "entries-over", &over).expect_err("one entry too many"); + assert!( + refused.contains("more than 4096 files and directories"), + "{refused}" + ); + + // `index.html` alone: its name and its entry count with its contents. + let index = |name: &str, bytes: u64| { + let viewer = scratch.0.join(name); + std::fs::create_dir_all(&viewer).expect("viewer"); + std::fs::File::create(viewer.join("index.html")) + .expect("index") + .set_len(bytes) + .expect("set_len"); + viewer + }; + let fits = VIEWER_BYTES - (256 + "index.html".len() as u64); + let at = index("index-at", fits); + assert_eq!(viewer_starts(&scratch, "index-at", &at), Ok(())); + let over = index("index-over", fits + 1); + let refused = viewer_starts(&scratch, "index-over", &over).expect_err("one byte too many"); + assert!(refused.contains("more than 16777216 bytes"), "{refused}"); + + // An asset two directories down is stored under its whole path, which is charged, not only its + // own name: index (256 + 10 + 1), assets (256 + 6), the directory (256 + 255) and the file + // (256 + 255 + 1 + 255) and its contents. + let directory = "d".repeat(255); + let file = "f".repeat(255); + let deep = |name: &str, bytes: u64| { + let viewer = scratch.0.join(name); + std::fs::create_dir_all(viewer.join("assets").join(&directory)).expect("dirs"); + std::fs::write(viewer.join("index.html"), "x").expect("index"); + std::fs::File::create(viewer.join("assets").join(&directory).join(&file)) + .expect("file") + .set_len(bytes) + .expect("set_len"); + viewer + }; + let charged = (256 + 10 + 1) + (256 + 6) + (256 + 255) + (256 + 255 + 1 + 255); + let at = deep("deep-at", VIEWER_BYTES - charged); + assert_eq!(viewer_starts(&scratch, "deep-at", &at), Ok(())); + let over = deep("deep-over", VIEWER_BYTES - charged + 1); + let refused = viewer_starts(&scratch, "deep-over", &over).expect_err("one byte too many"); + assert!(refused.contains("more than 16777216 bytes"), "{refused}"); +} diff --git a/packages/telemetry/motel/motel.capnp b/packages/telemetry/motel/motel.capnp deleted file mode 100644 index 68a42ed197..0000000000 --- a/packages/telemetry/motel/motel.capnp +++ /dev/null @@ -1,15 +0,0 @@ -# Serves the bundled Motel collector. Hosts supply the motel-data and motel-assets -# directories with --directory-path and the motel socket address with --socket-addr. -using Workerd = import "/workerd/workerd.capnp"; -const config :Workerd.Config = ( - services=[ - (name="motel",worker=( - compatibilityDate="2026-09-01",modules=[(name="motel.mjs",esModule=embed "motel.mjs")], - bindings=[(name="STORE",durableObjectNamespace="MotelCollector"),(name="ASSETS",service="motel-assets"),(name="MOTEL_OTEL_RETENTION_HOURS",text="168"),(name="MOTEL_OTEL_MAX_DB_SIZE_MB",text="1024")], - durableObjectNamespaces=[(className="MotelCollector",uniqueKey="motel",enableSql=true)],durableObjectStorage=(localDisk="motel-data") - )), - (name="motel-assets",disk=(path="web/dist")), - (name="motel-data",disk=(path="data",writable=true,allowDotfiles=true)) - ], - sockets=[(name="motel",address="127.0.0.1:0",http=(),service="motel")] -); diff --git a/packages/telemetry/motel/source.json b/packages/telemetry/motel/source.json deleted file mode 100644 index 4327aa0224..0000000000 --- a/packages/telemetry/motel/source.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "repository": "https://github.com/UsefulSoftwareCo/motel.git", - "revision": "b1783f4d344fd2e0fa584ef00b3e71838a8a1e03" -} diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json index 84590e173a..60b431aa27 100644 --- a/packages/telemetry/package.json +++ b/packages/telemetry/package.json @@ -19,17 +19,12 @@ "./schema-issue": "./src/schema-issue.ts", "./transport": "./src/transport.ts" }, - "scripts": { - "build:motel": "bun scripts/build-motel.ts" - }, "dependencies": { "@bufbuild/protobuf": "2.11.0", "@executor-js/utils": "workspace:*", - "effect": "4.1.4", - "workerd": "1.20260918.1" + "effect": "4.1.4" }, "devDependencies": { - "@effect/platform-bun": "4.1.4", "@effect/platform-node": "4.1.4" } } diff --git a/packages/telemetry/scripts/build-collector.ts b/packages/telemetry/scripts/build-collector.ts index 6e3c5b9478..c7150d7068 100644 --- a/packages/telemetry/scripts/build-collector.ts +++ b/packages/telemetry/scripts/build-collector.ts @@ -6,25 +6,74 @@ * artifact's identity. * * node packages/telemetry/scripts/build-collector.ts [--target ] --key - * node packages/telemetry/scripts/build-collector.ts [--target ] ... + * node packages/telemetry/scripts/build-collector.ts [--target ] [--verbose] ... * * `` is one of the release targets (`linux-x64`, `linux-arm64`, `darwin-arm64`, `darwin-x64`, * `win32-x64`) and defaults to the host's. The steps run in the order given: * - * - `build`: installs the pinned toolchain and the target's standard library with rustup, builds, - * and writes `dist/collector/` (the executable, `NOTICE`, `licenses/`, `cargo-packages.txt` and - * `identity`). It does nothing when `identity` already holds this tree's key. + * - `build`: builds with the pinned toolchain and writes `dist/collector/` (the executable, + * `NOTICE`, `licenses/`, `cargo-packages.txt` and `identity`). It does nothing when `identity` + * already holds this tree's key. Outside CI it first looks in the shared store (below) for a + * build of the same key and copies it. Cargo's intermediate files go to the shared cache, never + * into the workspace. * - `notices`: rewrites the licence files and package list in `dist/collector/`. * - `check`: the platform checks on the built executable: architecture, size, the newest glibc * symbol and the shared libraries on Linux, the minimum macOS version and the linked libraries on * macOS, the C runtime imports on Windows, and no build machine paths. + * `--executable ` makes `check` and `smoke` read that file instead of the build, such as the + * copy a package carries; `--packaged` makes `check` require it to be the build's own bytes (its + * SHA-256 and size, recorded in `executable.sha256` when it was built) in place of the build machine + * path search, so it needs no Rust toolchain. * - `smoke`: starts the built executable on a temporary store and port, sends one span and one log, * reads the ingest counters, stops it and starts it again on the same store. It needs neither git * nor Rust, so CI also runs it in an `ubuntu:22.04` container, the oldest glibc supported. * - * `--key` prints the artifact's identity: a SHA-256 of the crate's files (tracked, or untracked and - * not ignored, so uncommitted edits count), this script, the target, and the C compiler and linker - * on this host. Caches store the build under it. + * `--key` prints the artifact's identity: a SHA-256 of the crate's files as the compiler sees them + * (everything in its directory, whatever git ignores, but its own `target/` and the editor and OS + * leftovers `.DS_Store`, `.#*`, `#*#`, `*~`, `.*.swp` and `.*.swo`; a link to nothing counts by its + * target's name), this script, the target, and the C compiler and linker on this host. Caches store + * the build under it. + * + * `--verbose` runs cargo with `-vv`, which prints every command it runs, and the compiler with + * `RUST_BACKTRACE=1`. Neither is part of the identity, and neither changes the executable's bytes. + * + * The tools see only the variables `toolEnv` keeps; the rest of the caller's environment is dropped. + * Kept are the variables that find the tools (`keptVariables`), plus two groups that do not change + * what is compiled: + * - `CARGO_BUILD_JOBS`, how many compiler processes run at once, so a host can bound a cold build. + * - The proxy and certificate variables (`fetchVariables`), passed to `cargo fetch` alone, so a host + * behind a proxy can download crates. They decide whether crates download, not which: + * `cargo fetch --locked` checks every crate against the lockfile's checksum, and the compile runs + * `--offline`. + * Dropped though they do not change the bytes either: + * - The jobserver variables, `MAKEFLAGS`, `MFLAGS` and `CARGO_MAKEFLAGS`: `CARGO_BUILD_JOBS` is the + * one supported way to bound the build. + * - Diagnostics and locale (`RUST_BACKTRACE`, `RUST_LOG`, `CARGO_LOG`, `CARGO_TERM_*`, `LANG`, + * `LC_*`): a variable the caller sets never reaches the compiler; `--verbose` asks for the output + * instead. + * Everything else that is dropped can change the bytes: compilers, linkers and their flags, wrappers, + * Cargo's profile and target settings, build scripts' inputs, and SDK selection (`keptVariables`). + * + * The pinned toolchain must already be installed: without it the script stops and prints the one + * command that installs it. `--install-toolchain` installs it and the target's standard library + * (alone, or before `--key` or the steps), for CI and release runners. + * + * The shared cache is `$EXECUTOR_COLLECTOR_CACHE` (an absolute path), or `executor-collector` in the + * platform's cache directory (`$XDG_CACHE_HOME` or `~/.cache`, `~/Library/Caches`, + * `%LOCALAPPDATA%`). It holds `target//`, cargo's output for one workspace's crate, + * kept while used and removed after `targetDays` days unused or when the directories pass + * `targetBytes` in all, least recently used first; and, outside CI, `store///`, finished + * builds by identity, at most `storeEntries` a target and none unused for `storeDays` days. A + * build is written beside the store (`.partial--`) and renamed into it, and handed + * out by copying, never by link: nothing in a workspace can write through to it. A partial copy + * whose writer has exited, or that is older than an hour, is removed, and partial copies count + * against the bound. An entry, and a workspace's own build, is used only when its executable has + * the SHA-256 and size recorded beside it when it was built; a damaged entry is removed and the + * build compiled again. Builds of one identity that start together compile once: the first takes + * `locks//.lock` in the cache and the others wait for its stored build (`takeLock`). + * A store that cannot be written is said loudly, in the build's output and in `e2e:prepare`'s + * summary, and noted in `locks//store-unwritable`, so builds that would wait compile at + * once; the note goes when a build is stored again, and is ignored after an hour (`storeBroken`). Nothing after a whole, checked build fails it. */ import { spawn, spawnSync, type ChildProcess, type SpawnSyncReturns } from "node:child_process"; import { createHash } from "node:crypto"; @@ -115,6 +164,42 @@ const root = path.resolve(path.dirname(script), "../../.."); const crate = path.join(root, "packages/telemetry/collector"); const output = path.join(root, "packages/telemetry/dist/collector"); +/** The shared cache: one directory per machine and user, the same rule on every host. */ +const cacheRoot = (): string => { + const set = process.env.EXECUTOR_COLLECTOR_CACHE; + if (set !== undefined && set !== "") { + if (!path.isAbsolute(set)) + return fail(`EXECUTOR_COLLECTOR_CACHE must be an absolute path, not ${JSON.stringify(set)}.`); + return path.resolve(set); + } + const xdg = process.env.XDG_CACHE_HOME; + const base = + process.platform === "darwin" + ? path.join(os.homedir(), "Library", "Caches") + : process.platform === "win32" + ? (process.env.LOCALAPPDATA ?? path.join(os.homedir(), "AppData", "Local")) + : xdg !== undefined && path.isAbsolute(xdg) + ? xdg + : path.join(os.homedir(), ".cache"); + return path.join(base, "executor-collector"); +}; +/** + * Cargo's output for this checkout's crate. The path remapping in `rustFlags` names the crate's + * directory, so two checkouts never share one: each would rebuild every crate the other built. + */ +const targetDirectory = (): string => + path.join(cacheRoot(), "target", createHash("sha256").update(crate).digest("hex").slice(0, 16)); +/** Cargo's output directories kept in all: about 130 MB each for one target. */ +const targetBytes = 3 * 1024 * 1024 * 1024; +const targetDays = 14; +/** Finished builds kept per target, and for how long unused. A build is about 7 MB. */ +const storeEntries = 32; +const storeDays = 30; +/** No cleanup removes what was used this recently: another build may be using it now. */ +const inUseMs = 60 * 60 * 1000; +/** CI never reads or writes the store, as `scripts/e2e-prepare.ts` keeps no cache there. */ +const storeEnabled = () => process.env.CI === undefined || process.env.CI === ""; + const executableName = (target: Target) => target.os === "win32" ? "executor-collector.exe" : "executor-collector"; @@ -128,7 +213,10 @@ const executableName = (target: Target) => * `RUSTFLAGS`, every `CARGO_*` setting but `CARGO_HOME`, a Visual Studio developer prompt's * `INCLUDE` and `LIB`, and any other variable a build script reads. What is kept only finds the tools (`PATH`, the homes, * the Windows system directories the MSVC lookup uses), and the identity records the versions of - * the tools this environment finds. Compared with names in upper case, as Windows does. + * the tools this environment finds. One more is kept, `CARGO_BUILD_JOBS`: it sets how many + * compiler processes run at once, so a host can hold a cold build to its share of the cores, and + * not what any of them produces (the release profile's codegen units are fixed), so it stays out of + * the identity. Compared with names in upper case, as Windows does. */ const keptVariables: ReadonlySet = new Set([ "PATH", @@ -139,6 +227,7 @@ const keptVariables: ReadonlySet = new Set([ "TEMP", "TMP", "CARGO_HOME", + "CARGO_BUILD_JOBS", "RUSTUP_HOME", "SYSTEMROOT", "SYSTEMDRIVE", @@ -159,6 +248,30 @@ const keptVariables: ReadonlySet = new Set([ "OS", ]); +/** + * Variables `cargo fetch` alone is given, so it can download through a proxy that needs them: they + * decide whether crates download, not which, since `--locked` checks each crate against the + * lockfile's checksum. The compile runs `--offline` and never sees them. Compared in upper case, and + * passed with the caller's own spelling (`https_proxy` as well as `HTTPS_PROXY`). + */ +const fetchVariables: ReadonlySet = new Set([ + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "CARGO_HTTP_PROXY", + "CARGO_HTTP_CAINFO", + "SSL_CERT_FILE", +]); +const fetchEnv = (): NodeJS.ProcessEnv => + toolEnv( + Object.fromEntries( + Object.entries(process.env).filter( + (entry): entry is [string, string] => + fetchVariables.has(entry[0].toUpperCase()) && entry[1] !== undefined, + ), + ), + ); + /** The pinned Rust release, from the crate's `rust-toolchain.toml`. */ const pinnedToolchain = (): string => fs @@ -175,7 +288,15 @@ const toolEnv = (extra: Record): NodeJS.ProcessEnv => { const env: NodeJS.ProcessEnv = {}; for (const [name, value] of Object.entries(process.env)) if (keptVariables.has(name.toUpperCase())) env[name] = value; - return { ...env, CARGO_HOME: cargoHome(), RUSTUP_TOOLCHAIN: pinnedToolchain(), ...extra }; + // RUSTUP_AUTO_INSTALL=0: a toolchain named in RUSTUP_TOOLCHAIN that is missing is an error, never + // a download; only `--install-toolchain` installs one (`installToolchain`). + return { + ...env, + CARGO_HOME: cargoHome(), + RUSTUP_TOOLCHAIN: pinnedToolchain(), + RUSTUP_AUTO_INSTALL: "0", + ...extra, + }; }; type Run = { @@ -244,17 +365,50 @@ const cargoHome = (): string => { return path.resolve(home); }; -const requireRustup = () => { +/** The one command that installs the pinned toolchain, and `target`'s library when it needs one. */ +const installCommand = (target: Target) => { + const toolchain = pinnedToolchain(); + const host = target.name === hostTarget(); + return `rustup toolchain install ${toolchain} --profile minimal${host ? "" : ` && rustup target add --toolchain ${toolchain} ${target.triple}`}`; +}; + +const requireRustup = (target: Target) => { const result = spawnSync("rustup", ["--version"], { encoding: "utf8", env: toolEnv({}) }); if (result.error !== undefined || result.status !== 0) fail( - "rustup is not installed. Install it from https://rustup.rs; it installs the toolchain this crate pins in rust-toolchain.toml.", + process.platform === "win32" + ? `rustup is not installed. Install it from https://rustup.rs, then run: ${installCommand(target)}` + : `rustup is not installed. Install it with the toolchain this crate pins: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain ${pinnedToolchain()}`, ); }; +/** + * Stops, printing the command that installs it, unless the pinned toolchain and `target`'s standard + * library are installed. Never installs anything. + */ +const requireToolchain = (target: Target) => { + requireRustup(target); + const toolchain = pinnedToolchain(); + const listed = (args: ReadonlyArray) => + capture("rustup", args, { cwd: crate, failure: "Cannot list rustup's toolchains" }) + .split("\n") + .map((line) => line.trim().split(/\s/u)[0] ?? ""); + const installed = + listed(["toolchain", "list"]).some((name) => name.startsWith(`${toolchain}-`)) && + listed(["target", "list", "--installed", "--toolchain", toolchain]).includes(target.triple); + if (!installed) + fail( + `The Rust toolchain this crate pins (${toolchain}, for ${target.triple}) is not installed. Install it with: ${installCommand(target)}`, + ); +}; + +/** The pinned toolchain, installed with `--install-toolchain` or required to be present already. */ +const toolchainReady = (target: Target, install: boolean) => + install ? installToolchain(target) : requireToolchain(target); + /** Installs the pinned toolchain and `target`'s standard library; both are no-ops once present. */ const installToolchain = (target: Target) => { - requireRustup(); + requireRustup(target); const toolchain = pinnedToolchain(); stream( "rustup", @@ -406,19 +560,55 @@ const regularFile = (file: string): true => { /** The `identity` of a build a Cargo config affected. A key is 64 hex digits, so none equals it. */ const uncacheable = "uncacheable: a Cargo config file was in effect"; -/** The crate's files as git sees them: tracked, or untracked and not ignored. */ -const crateFiles = (): ReadonlyArray => - capture( - "git", - ["ls-files", "-z", "-co", "--exclude-standard", "--", "packages/telemetry/collector"], - { env: process.env, failure: "Cannot list the crate's files" }, - ) - .split("\0") - .filter((file) => file !== "") - .sort(); +/** + * Names editors and operating systems leave beside files, which no compile reads: `.DS_Store`, + * Emacs's lock links `.#*` and autosaves `#*#`, backups `*~`, and vim's swap files `.*.swp` and + * `.*.swo`. Exact patterns, matched against a file's or link's own name and never a directory's; + * no other name is left out, so a near miss (`main.swp`, `x.rs~1`) still counts. + */ +const editorLeftover = (name: string) => + name === ".DS_Store" || + name.startsWith(".#") || + (name.length > 2 && name.startsWith("#") && name.endsWith("#")) || + name.endsWith("~") || + /^\..+\.sw[po]$/u.test(name); /** - * The artifact identity of `target` built from this tree on this host: the crate's files, this + * The crate's files as the compiler sees them: everything in its directory, whatever git ignores, + * but `target/`, where `cargo test` and `cargo build` run in the crate write their output, and + * `editorLeftover` names. Paths relative to the repository, with forward slashes, sorted. A link to + * a file counts as its target and its contents, and a link to nothing as its target alone; a link + * to a directory, or anything else that is not a file or a directory, is refused, since its + * contents could change without changing what is hashed here. + */ +const crateFiles = (): ReadonlyArray => { + const files: string[] = []; + const walk = (directory: string) => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const absolute = path.join(directory, entry.name); + if (directory === crate && entry.name === "target") continue; + if (entry.isDirectory()) walk(absolute); + else if (editorLeftover(entry.name)) continue; + else if (entry.isFile()) files.push(absolute); + else if (entry.isSymbolicLink() && linked(absolute) !== "other") files.push(absolute); + else + fail( + `${absolute} is neither a file, a directory nor a link to a file, so the build's identity cannot cover it; remove it from the crate.`, + ); + } + }; + walk(crate); + return files.map((file) => path.relative(root, file).split(path.sep).join("/")).sort(); +}; + +/** What a link in the crate points at: a file, nothing at all, or something else. */ +const linked = (link: string): "file" | "nothing" | "other" => { + const stat = fs.statSync(link, { throwIfNoEntry: false }); + return stat === undefined ? "nothing" : stat.isFile() ? "file" : "other"; +}; + +/** + * The artifact identity of `target` built from this tree on this host: the crate's files (`crateFiles`), this * script, the target and the tools `toolEnv` finds. `toolEnv` keeps the caller's other variables * out of the build. It is a cacheable identity only while no Cargo config file is in effect * (`cargoConfigFiles`); callers check that first. Needs the pinned toolchain installed. @@ -434,11 +624,12 @@ const identity = (target: Target): string => { }), ); for (const file of crateFiles()) { - // A path git lists can be gone from the working tree (deleted, not yet staged). const absolute = path.join(root, file); - hash.update( - `${file}\0${fs.existsSync(absolute) ? sha256(fs.readFileSync(absolute)) : "deleted"}\n`, - ); + const isLink = fs.lstatSync(absolute).isSymbolicLink(); + const link = isLink ? fs.readlinkSync(absolute) : ""; + const contents = + isLink && linked(absolute) === "nothing" ? "nothing" : sha256(fs.readFileSync(absolute)); + hash.update(`${file}\0${link}\0${contents}\n`); } return hash.digest("hex"); }; @@ -447,7 +638,13 @@ const identity = (target: Target): string => { // Build /** Paths on the build machine that must not appear in the executable. */ -const buildPaths = (): ReadonlyArray => [cargoHome(), sysroot(), crate, root]; +const buildPaths = (): ReadonlyArray => [ + cargoHome(), + sysroot(), + crate, + root, + targetDirectory(), +]; const rustFlags = (target: Target): string => [ @@ -468,49 +665,598 @@ const buildEnv = (target: Target) => ...(target.macosMinimum === undefined ? {} : { MACOSX_DEPLOYMENT_TARGET: target.macosMinimum }), }); -const build = (target: Target) => { - installToolchain(target); - const configs = cargoConfigFiles(); - if (configs.length > 0) - console.log( - `Cargo config in effect (${configs.join(", ")}): building without the up-to-date check, and writing an identity no cache matches.`, +/** Whether `error` says a path is gone: another build removed it since it was listed. */ +const vanished = (error: unknown) => + (error as NodeJS.ErrnoException).code === "ENOENT" || + (error as NodeJS.ErrnoException).code === "ENOTDIR"; + +/** + * Bytes in `directory`, its files counted once each. Other builds compile into the shared cache + * meanwhile, so a file or directory that vanishes during the walk counts as nothing. + */ +const directoryBytes = (directory: string): number => { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(directory, { withFileTypes: true }); + } catch (error) { + if (vanished(error)) return 0; + throw error; + } + let bytes = 0; + for (const entry of entries) { + const absolute = path.join(directory, entry.name); + if (entry.isDirectory()) bytes += directoryBytes(absolute); + else if (entry.isFile()) + try { + bytes += fs.statSync(absolute).size; + } catch (error) { + if (!vanished(error)) throw error; + } + } + return bytes; +}; + +/** When a cache entry was last used: the mtime of its `last-used` file, or of the entry. */ +const lastUsed = (entry: string): number => { + const marker = path.join(entry, "last-used"); + try { + return (fs.existsSync(marker) ? fs.statSync(marker) : fs.statSync(entry)).mtimeMs; + } catch { + // Removed by another process since it was listed. + return Number.NaN; + } +}; +const markUsed = (entry: string) => { + fs.mkdirSync(entry, { recursive: true }); + const marker = path.join(entry, "last-used"); + const now = new Date(); + if (fs.existsSync(marker)) fs.utimesSync(marker, now, now); + else fs.writeFileSync(marker, ""); +}; + +/** A copy in progress beside `to`: `.partial--`, so its writer can be told. */ +const partialPath = (to: string) => + `${to}.partial-${process.pid}-${createHash("sha256").update(`${Date.now()}${Math.random()}`).digest("hex").slice(0, 8)}`; +const partialName = /\.partial-(\d+)-[0-9a-f]+$/u; + +/** Whether a partial copy was abandoned: its writer has exited, or it is older than `inUseMs`. */ +const abandoned = (entry: string, now: number): boolean => { + const pid = Number(partialName.exec(entry)?.[1]); + return !running(pid) || !(now - lastUsed(entry) <= inUseMs); +}; + +/** + * Removes cache entries in `directory` unused for `days` days, and abandoned partial copies; then + * the least recently used entries until `within` holds, partial copies counted. It never removes + * one used within `inUseMs` or `keep`, and reads an entry's last use again just before removing + * it, so an entry another process has just started copying is kept. + */ +const prune = ( + directory: string, + days: number, + keep: string, + within: (entries: ReadonlyArray) => boolean, +) => { + if (!fs.existsSync(directory)) return; + const now = Date.now(); + const stale = (entry: string, ms: number) => { + const used = lastUsed(entry); + return Number.isNaN(used) || now - used > ms; + }; + const remove = (entry: string) => fs.rmSync(entry, { recursive: true, force: true }); + let entries = fs + .readdirSync(directory) + .map((name) => path.join(directory, name)) + .map((entry) => ({ entry, used: lastUsed(entry) })) + .sort((a, b) => a.used - b.used); + for (const { entry } of entries) { + if (entry === keep) continue; + if (partialName.test(entry) ? abandoned(entry, now) : stale(entry, days * 24 * 60 * 60 * 1000)) + remove(entry); + } + entries = entries.filter(({ entry }) => fs.existsSync(entry)); + while (!within(entries.map(({ entry }) => entry))) { + const oldest = entries.find( + ({ entry }) => entry !== keep && !partialName.test(entry) && stale(entry, inUseMs), ); - const key = configs.length === 0 ? identity(target) : undefined; - const stamp = path.join(output, "identity"); - const executable = path.join(output, executableName(target)); - if ( - key !== undefined && - fs.existsSync(executable) && - fs.existsSync(stamp) && - fs.readFileSync(stamp, "utf8").trim() === key - ) { - console.log(`executor-collector for ${target.name} is up to date (${key}).`); - return; + if (oldest === undefined) return; + remove(oldest.entry); + entries = entries.filter(({ entry }) => entry !== oldest.entry); + } +}; + +/** Copies `from` to `to`: written beside `to` and renamed over it, by copying, never linking. */ +const replaceWithCopy = (from: string, to: string) => { + const partial = partialPath(to); + try { + fs.cpSync(from, partial, { recursive: true, dereference: true, verbatimSymlinks: false }); + fs.rmSync(path.join(partial, "last-used"), { force: true }); + fs.rmSync(to, { recursive: true, force: true }); + fs.renameSync(partial, to); + } finally { + fs.rmSync(partial, { recursive: true, force: true }); + } +}; + +const storeEntry = (target: Target, key: string) => + path.join(cacheRoot(), "store", target.name, key); + +/** The executable's SHA-256 and size, written beside it when it is built. */ +const digestFile = "executable.sha256"; +const executableDigest = (file: string) => { + const bytes = fs.readFileSync(file); + return `${sha256(bytes)} ${bytes.length}`; +}; + +/** + * Whether `directory` holds a whole build of `key`: its identity, and an executable whose SHA-256 + * and size are those recorded when it was built. + */ +const intact = (directory: string, target: Target, key: string): boolean => { + const read = (name: string) => { + try { + return fs.readFileSync(path.join(directory, name), "utf8").trim(); + } catch { + return undefined; + } + }; + if (read("identity") !== key) return false; + const recorded = read(digestFile); + const executable = path.join(directory, executableName(target)); + return ( + recorded !== undefined && fs.existsSync(executable) && executableDigest(executable) === recorded + ); +}; + +/** Puts the build in `output` in the store under `key`, unless another process just did. */ +const storeBuild = (target: Target, key: string) => { + const entry = storeEntry(target, key); + if (!fs.existsSync(entry)) { + fs.mkdirSync(path.dirname(entry), { recursive: true }); + const partial = partialPath(entry); + try { + fs.cpSync(output, partial, { recursive: true, dereference: true }); + fs.renameSync(partial, entry); + } catch (error) { + // Another build stored the same key first, and its entry holds the same build; any other + // failure is the caller's. + if (!fs.existsSync(entry)) throw error; + } finally { + fs.rmSync(partial, { recursive: true, force: true }); + } } + markUsed(entry); +}; + +// --------------------------------------------------------------------------------------------- +// One build per identity + +/** + * Builds that start together on the same identity (several workspaces preparing after a merge) + * compile once: the first takes the identity's lock, and the others wait for its stored build and + * copy it. The lock is `locks//.lock` in the cache, holding its builder's process id, + * that process's start time and a token. It is stale once that process has exited, once the id + * belongs to a process started at another time (a reused id), or after `inUseMs` as a last resort; + * a stale lock is taken over. The lock only saves compiles: what is correct rests on the store's + * rename and the executable's digest, so two builders that both take a lock (two taking over one + * stale lock at once) still store and copy whole builds. + */ +const lockWaitMs = 20 * 60 * 1000; +const lockPollMs = 250; +/** How far a process's start time, as `ps` reads it to the second, may differ from the lock's. */ +const startToleranceMs = 5_000; +const startedAt = Math.round(Date.now() - process.uptime() * 1000); + +const lockFile = (target: Target, key: string) => + path.join(cacheRoot(), "locks", target.name, `${key}.lock`); + +type Holder = { readonly pid: number; readonly started: number; readonly token: string }; + +const readHolder = (file: string): Holder | undefined => { + try { + const value: unknown = JSON.parse(fs.readFileSync(file, "utf8")); + if ( + isRecord(value) && + typeof value.pid === "number" && + typeof value.started === "number" && + typeof value.token === "string" + ) + return { pid: value.pid, started: value.started, token: value.token }; + } catch {} + return undefined; +}; + +const running = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM: the process exists but belongs to someone else. + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +}; + +/** + * When process `pid` started, from its elapsed time as `ps -o etime=` gives it on Linux and macOS + * (`[[dd-]hh:]mm:ss`, no time zone involved); undefined where `ps` cannot say (Windows). + */ +const processStarted = (pid: number): number | undefined => { + if (process.platform === "win32") return undefined; + const result = spawnSync("ps", ["-o", "etime=", "-p", String(pid)], { encoding: "utf8" }); + const elapsed = /^(?:(?:(\d+)-)?(\d+):)?(\d+):(\d+)$/u.exec(result.stdout?.trim() ?? ""); + if (result.status !== 0 || elapsed === null) return undefined; + const [days, hours, minutes, seconds] = elapsed.slice(1).map((part) => Number(part ?? 0)); + return Date.now() - (((days! * 24 + hours!) * 60 + minutes!) * 60 + seconds!) * 1000; +}; + +/** Why the lock in `file` no longer holds, or undefined while its builder runs. */ +const staleness = (file: string, holder: Holder | undefined): string | undefined => { + if (holder === undefined) return "it does not name a builder"; + if (!running(holder.pid)) return `its builder, process ${holder.pid}, has exited`; + const age = Date.now() - lastUsed(file); + if (age > inUseMs) return `it was taken ${Math.round(age / 60_000)} minutes ago`; + const started = processStarted(holder.pid); + if (started !== undefined && Math.abs(started - holder.started) > startToleranceMs) + return `process ${holder.pid} is another process now, started after the lock was taken`; + return undefined; +}; + +type Lock = + | { readonly kind: "taken"; readonly release: () => void } + | { readonly kind: "held"; readonly file: string; readonly holder: Holder | undefined } + | { readonly kind: "unavailable"; readonly reason: string }; + +/** + * Takes the lock for `key`: written whole beside it and linked into place, so no reader ever sees + * a lock half written, and the link fails when another builder holds it. + */ +const takeLock = (target: Target, key: string): Lock => { + const file = lockFile(target, key); + const holder: Holder = { + pid: process.pid, + started: startedAt, + token: createHash("sha256").update(`${Date.now()}${Math.random()}`).digest("hex").slice(0, 16), + }; + const written = partialPath(file); + try { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(written, JSON.stringify(holder)); + try { + fs.linkSync(written, file); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + return { kind: "held", file, holder: readHolder(file) }; + } + } catch (error) { + return { kind: "unavailable", reason: String(error) }; + } finally { + fs.rmSync(written, { force: true }); + } + return { + kind: "taken", + release: () => { + // Only this builder's own lock: a builder that took over a stale one holds its own token. + if (readHolder(file)?.token === holder.token) fs.rmSync(file, { force: true }); + }, + }; +}; + +/** Removes a stale lock. Moved aside first, so a lock another builder has just taken survives. */ +const breakLock = (file: string, stale: Holder | undefined) => { + const aside = partialPath(file); + try { + fs.renameSync(file, aside); + } catch (error) { + if (vanished(error)) return; + throw error; + } + const moved = readHolder(aside); + if (moved !== undefined && moved.token !== stale?.token) { + // A builder took the lock between the look and the move: put it back. + try { + fs.linkSync(aside, file); + } catch {} + } + fs.rmSync(aside, { force: true }); +}; + +/** Removes the locks of builders that are gone, and lock files a killed builder left half made. */ +const pruneLocks = (target: Target) => { + const directory = path.dirname(lockFile(target, "x")); + if (!fs.existsSync(directory)) return; + const now = Date.now(); + for (const name of fs.readdirSync(directory)) { + const file = path.join(directory, name); + if (partialName.test(name)) { + if (abandoned(file, now)) fs.rmSync(file, { force: true }); + } else { + const holder = readHolder(file); + if (holder !== undefined && staleness(file, holder) !== undefined) breakLock(file, holder); + } + } +}; + +const sleep = (ms: number) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); + +/** + * Copies the stored build of `key` into the workspace and says so; false when there is none. A + * damaged entry is removed, not left for the next workspace. + */ +const copyFromStore = (target: Target, key: string): boolean => { + const entry = storeEntry(target, key); + if (!fs.existsSync(entry)) return false; + // Marked before it is copied, so no prune removes it meanwhile. A store this process cannot write + // is still read. + try { + markUsed(entry); + } catch (error) { + console.error(`Cannot mark ${entry} used (${String(error)}); copying it all the same.`); + } + if (!intact(entry, target, key)) { + fs.rmSync(entry, { recursive: true, force: true }); + console.log(`Removed the damaged stored build ${entry}; building instead.`); + return false; + } + const executable = path.join(output, executableName(target)); + replaceWithCopy(entry, output); + if (target.os !== "win32") fs.chmodSync(executable, 0o755); + if (!intact(output, target, key)) + fail(`The copy of ${entry} in ${output} does not match it; build again.`); + console.log(`executor-collector for ${target.name} copied from ${entry} (${key}).`); + return true; +}; + +/** Compiles `target` into `output`, its identity written last. */ +const compile = (target: Target, key: string | undefined) => { + const targetDir = targetDirectory(); + const executable = path.join(output, executableName(target)); + markUsed(targetDir); // Separate from the build, so an outage at crates.io is named as one. stream("cargo", ["fetch", "--locked", "--target", target.triple], { cwd: crate, + env: fetchEnv(), failure: "Cannot download the crate's dependencies from crates.io", }); - stream("cargo", ["build", "--release", "--locked", "--offline", "--target", target.triple], { - cwd: crate, - env: buildEnv(target), - failure: "cargo build failed", - }); + stream( + "cargo", + [ + "build", + ...(verbose ? ["-vv"] : []), + "--release", + "--locked", + "--offline", + "--target", + target.triple, + "--target-dir", + targetDir, + ], + { + cwd: crate, + env: verbose ? { ...buildEnv(target), RUST_BACKTRACE: "1" } : buildEnv(target), + failure: verbose + ? "cargo build failed" + : `cargo build failed. For every command cargo ran and the compiler's backtrace, run the build again with --verbose (node packages/telemetry/scripts/build-collector.ts --target ${target.name} --verbose build); a RUST_BACKTRACE or RUST_LOG you set does not reach the compiler`, + }, + ); // Replace the whole directory, and write the identity last, so an interrupted build never // leaves a directory that claims to be complete. fs.rmSync(output, { recursive: true, force: true }); fs.mkdirSync(output, { recursive: true }); fs.copyFileSync( - path.join(crate, "target", target.triple, "release", executableName(target)), + path.join(targetDir, target.triple, "release", executableName(target)), executable, ); if (target.os !== "win32") fs.chmodSync(executable, 0o755); + fs.writeFileSync(path.join(output, digestFile), `${executableDigest(executable)}\n`); notices(target); - fs.writeFileSync(stamp, `${key ?? uncacheable}\n`); + fs.writeFileSync(path.join(output, "identity"), `${key ?? uncacheable}\n`); console.log(`Built executor-collector for ${target.name} at ${output} (${key ?? uncacheable}).`); }; +/** + * Work after a build that is already whole and checked: storing it and pruning the caches. None of + * it can fail the build; a problem is logged. + */ +const afterBuild = (what: string, work: () => void) => { + try { + work(); + } catch (error) { + console.error(`${what} failed; the build stands: ${String(error)}`); + } +}; +/** + * A shared store this host cannot write, a read-only or full disk, is not hidden. Each build that + * finds it so prints one loud line, which `e2e:prepare` repeats in its summary (the file + * `EXECUTOR_PREPARE_NOTES` names), and leaves `locks//store-unwritable` beside the locks + * with the reason. A build that finds the identity's lock held and this note under an hour old + * compiles at once instead of waiting for a build that will not be stored. A builder that takes the + * lock first checks it can write the store (`probeStore`): when it cannot, it releases the lock + * before compiling, so no waiter queues behind its compile even when the note itself cannot be + * written. One case still queues: a store with room for the probe but not for a build, and no room + * for the note either; then waiters compile one after another, as before this note existed. + * + * The note goes away without anyone removing it by hand: the first build that stores removes it, + * and it is ignored after `inUseMs`. While it is fresh, a builder that takes the lock releases it at + * once and compiles, then still tries to store, so a repaired store takes that build and the note + * goes; a passing probe alone does not remove it, since a full disk can take the probe and not the + * build. Nothing here fails a build: the note, the line and the probe only log. + */ +const unwritableNote = (target: Target) => + path.join(cacheRoot(), "locks", target.name, "store-unwritable"); + +const prepareNote = (line: string) => { + const file = process.env.EXECUTOR_PREPARE_NOTES; + if (file === undefined || file === "") return; + try { + fs.appendFileSync(file, `${line}\n`); + } catch {} +}; + +const storeBroken = (target: Target, reason: string) => { + const store = path.join(cacheRoot(), "store", target.name); + const line = `The shared collector store ${store} cannot be written (${reason}). This build stands, but it is not stored: every workspace compiles its own until the store can be written again.`; + console.error(`\n*** ${line}\n`); + prepareNote(line); + const note = unwritableNote(target); + try { + fs.mkdirSync(path.dirname(note), { recursive: true }); + fs.writeFileSync(note, `${reason}\n`); + } catch (error) { + console.error( + `Cannot leave the note ${note} either (${String(error)}): builds of one identity that start together wait for one another until the store can be written.`, + ); + } +}; + +const storeRepaired = (target: Target) => { + try { + fs.rmSync(unwritableNote(target), { force: true }); + } catch {} +}; + +/** The reason in a note under an hour old, or undefined. */ +const storeUnwritable = (target: Target): string | undefined => { + try { + const note = unwritableNote(target); + if (Date.now() - fs.statSync(note).mtimeMs > inUseMs) return undefined; + return fs.readFileSync(note, "utf8").trim(); + } catch { + return undefined; + } +}; + +/** Why the store cannot be written, by writing and removing a small file in it; or undefined. */ +const probeStore = (target: Target): string | undefined => { + const directory = path.join(cacheRoot(), "store", target.name); + // Named as a partial copy, so pruning removes one a killed build left. + const probe = partialPath(path.join(directory, "probe")); + try { + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync(probe, ""); + return undefined; + } catch (error) { + return String(error); + } finally { + try { + fs.rmSync(probe, { force: true }); + } catch {} + } +}; + +const storeAfterBuild = (target: Target, key: string) => { + try { + storeBuild(target, key); + } catch (error) { + storeBroken(target, String(error)); + return; + } + storeRepaired(target); + const entry = storeEntry(target, key); + afterBuild(`Pruning ${path.dirname(entry)}`, () => + prune(path.dirname(entry), storeDays, entry, (entries) => entries.length <= storeEntries), + ); +}; +const pruneTargets = () => { + const targetDir = targetDirectory(); + afterBuild(`Pruning ${path.dirname(targetDir)}`, () => + prune(path.dirname(targetDir), targetDays, targetDir, (entries) => { + let bytes = 0; + for (const entry of entries) bytes += directoryBytes(entry); + return bytes <= targetBytes; + }), + ); +}; + +const build = (target: Target, install: boolean) => { + toolchainReady(target, install); + const configs = cargoConfigFiles(); + if (configs.length > 0) + console.log( + `Cargo config in effect (${configs.join(", ")}): building without the up-to-date check, and writing an identity no cache matches.`, + ); + const key = configs.length === 0 ? identity(target) : undefined; + if (key !== undefined && intact(output, target, key)) { + console.log(`executor-collector for ${target.name} is up to date (${key}).`); + return; + } + if (key !== undefined && storeEnabled()) { + const deadline = Date.now() + lockWaitMs; + let waitingFor: string | undefined; + for (;;) { + if (copyFromStore(target, key)) return; + const lock = takeLock(target, key); + if (lock.kind === "taken") { + let held = true; + try { + // Another builder may have stored it between the look and the lock. + if (copyFromStore(target, key)) return; + const probed = probeStore(target); + const noted = probed === undefined ? storeUnwritable(target) : undefined; + if (probed === undefined && noted === undefined) { + compile(target, key); + storeAfterBuild(target, key); + } else { + // Nothing is likely to be stored: let the others compile now, not after this build. + if (probed !== undefined) storeBroken(target, probed); + else + console.error( + `The shared store was found unwritable less than an hour ago (${noted}; noted in ${unwritableNote(target)}): building without holding the lock, then trying to store.`, + ); + lock.release(); + held = false; + compile(target, key); + // A store that has been repaired takes this build, which removes the note. + if (probed === undefined) storeAfterBuild(target, key); + } + } finally { + if (held) lock.release(); + } + pruneTargets(); + afterBuild("Removing stale locks", () => pruneLocks(target)); + return; + } + if (lock.kind === "unavailable") { + console.error( + `Cannot take the lock for ${key} (${lock.reason}); building without waiting for other builds.`, + ); + break; + } + // Released between the link and the read: look again. + if (lock.holder === undefined && !fs.existsSync(lock.file)) continue; + const stale = staleness(lock.file, lock.holder); + if (stale !== undefined) { + console.log(`Taking over the build of ${key}: the lock ${lock.file} is stale, ${stale}.`); + breakLock(lock.file, lock.holder); + continue; + } + const holder = `process ${lock.holder?.pid}`; + const unwritable = storeUnwritable(target); + if (unwritable !== undefined) { + console.error( + `The shared store cannot be written (${unwritable}; noted in ${unwritableNote(target)}): building here without waiting for ${holder}.`, + ); + break; + } + if (Date.now() > deadline) { + console.error( + `Waited ${lockWaitMs / 60_000} minutes for ${holder} to build ${key} (lock ${lock.file}); building here instead.`, + ); + break; + } + if (waitingFor !== holder) { + waitingFor = holder; + console.log( + `Waiting for ${holder}, which is building executor-collector for ${target.name} (${key}; lock ${lock.file}), for up to ${lockWaitMs / 60_000} minutes.`, + ); + } + sleep(lockPollMs); + } + } + compile(target, key); + if (key !== undefined && storeEnabled()) storeAfterBuild(target, key); + pruneTargets(); +}; + // --------------------------------------------------------------------------------------------- // Notices @@ -813,14 +1559,33 @@ const executableBytes = (binary: string): Buffer => { return bytes; }; -const check = (target: Target) => { - const binary = path.join(output, executableName(target)); +/** Where the steps read the executable: `--executable`, or the build in `dist/collector`. */ +let chosenExecutable: string | undefined; +/** `--verbose`: cargo's `-vv` and the compiler's backtraces, outside the identity. */ +let verbose = false; +const executablePath = (target: Target) => + chosenExecutable ?? path.join(output, executableName(target)); + +/** + * `packaged`: the executable is a packaged copy of a build this script already checked, read on a + * runner that may have no Rust toolchain. Its bytes must be the build's (`executable.sha256` in + * `dist/collector`), which stands in for the build machine path search the build's own check made. + */ +const check = (target: Target, packaged: boolean) => { + const binary = executablePath(target); if (!fs.existsSync(binary)) fail(`${binary} does not exist; run the build step first`); const bytes = executableBytes(binary); + if (packaged) { + const recorded = fs.existsSync(path.join(output, digestFile)) + ? fs.readFileSync(path.join(output, digestFile), "utf8").trim() + : fail(`${output} records no ${digestFile}; it is not a whole build`); + if (executableDigest(binary) !== recorded) + fail(`${binary} is not the build in ${output} (its SHA-256 and size differ)`); + } // A leaked path continues into a file name, so each directory is searched with a separator after // it, which keeps a short directory such as a container's `/w` from matching other text. Both // separators are tried, since a path can be written either way on Windows. - const leaked = [...buildPaths(), os.homedir()] + const leaked = (packaged ? [] : [...buildPaths(), os.homedir()]) .map((directory) => directory.replace(/[\\/]+$/u, "")) .filter((directory) => directory !== "") .flatMap((directory) => { @@ -1145,9 +1910,10 @@ const probes: ReadonlyArray<(collector: Collector) => Promise> = [ ]; const smoke = async (target: Target) => { + const chosen = executablePath(target); if (target.os !== process.platform) fail(`Smoke-test ${target.name} on ${target.os}; this host is ${process.platform}`); - const binary = path.join(output, executableName(target)); + const binary = chosen; if (!fs.existsSync(binary)) fail(`${binary} does not exist; run the build step first`); executableBytes(binary); if (target.os === "win32") { @@ -1183,7 +1949,13 @@ const smoke = async (target: Target) => { ); } finally { await Promise.all([...owned].map(reap)); - fs.rmSync(scratch, { recursive: true, force: true }); + // Windows Defender can hold the store file for a moment after the collector exits: retry. A + // clean-up that still fails is logged, never thrown, so it cannot replace the smoke's own result. + try { + fs.rmSync(scratch, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); + } catch (error) { + console.error(`Could not remove the smoke's directory ${scratch}: ${String(error)}`); + } } }; @@ -1200,15 +1972,23 @@ const hostTarget = () => `${process.platform}-${process.arch}`; const parseArguments = (args: ReadonlyArray) => { let targetName = hostTarget(); let key = false; + let install = false; + let packaged = false; const chosen: Step[] = []; for (let index = 0; index < args.length; index++) { const arg = args[index] ?? ""; if (arg === "--target") targetName = args[++index] ?? fail("--target needs a value"); else if (arg === "--key") key = true; - else if (isStep(arg)) chosen.push(arg); + else if (arg === "--install-toolchain") install = true; + else if (arg === "--packaged") packaged = true; + else if (arg === "--verbose") verbose = true; + else if (arg === "--executable") { + const file = args[++index] ?? fail("--executable needs a path"); + chosenExecutable = path.resolve(file); + } else if (isStep(arg)) chosen.push(arg); else fail( - `Unknown argument ${arg}. Steps: ${steps.join(", ")}; options: --target , --key`, + `Unknown argument ${arg}. Steps: ${steps.join(", ")}; options: --target , --key, --install-toolchain, --packaged, --executable , --verbose`, ); } const target = @@ -1217,14 +1997,15 @@ const parseArguments = (args: ReadonlyArray) => { `No collector target ${targetName}; the targets are ${targets.map((t) => t.name).join(", ")}`, ); if (key && chosen.length > 0) fail("--key prints the identity and runs no steps"); - if (!key && chosen.length === 0) fail(`Name a step (${steps.join(", ")}) or --key`); - return { target, key, steps: chosen }; + if (!key && !install && chosen.length === 0) + fail(`Name a step (${steps.join(", ")}), --key or --install-toolchain`); + return { target, key, install, packaged, steps: chosen }; }; const main = async () => { const options = parseArguments(process.argv.slice(2)); if (options.key) { - installToolchain(options.target); + toolchainReady(options.target, options.install); const configs = cargoConfigFiles(); if (configs.length > 0) fail( @@ -1233,16 +2014,20 @@ const main = async () => { console.log(identity(options.target)); return; } + if (!options.key && options.steps.length === 0) { + installToolchain(options.target); + return; + } for (const step of options.steps) { switch (step) { case "build": - build(options.target); + build(options.target, options.install); break; case "notices": notices(options.target); break; case "check": - check(options.target); + check(options.target, options.packaged); break; case "smoke": await smoke(options.target); diff --git a/packages/telemetry/scripts/build-motel.ts b/packages/telemetry/scripts/build-motel.ts deleted file mode 100644 index d710aced2d..0000000000 --- a/packages/telemetry/scripts/build-motel.ts +++ /dev/null @@ -1,79 +0,0 @@ -/** Build the pinned Motel fork outside the monorepo's Effect override. */ -import * as BunRuntime from "@effect/platform-bun/BunRuntime"; -import * as BunServices from "@effect/platform-bun/BunServices"; -import { Effect, FileSystem, Path, Schema } from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/process"; -import { realpath } from "node:fs"; -import { promisify } from "node:util"; - -const build = Effect.gen(function* () { - if (process.versions.bun !== "1.4.4") - return yield* Effect.die(new Error("Build the collector with Bun 1.4.4.")); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const processes = yield* ChildProcessSpawner.ChildProcessSpawner; - const root = yield* path.fromFileUrl(new URL("../", import.meta.url)); - const source = path.join(root, "motel"); - const output = path.join(root, "dist/motel"); - const pinned = yield* fs.readFileString(path.join(source, "source.json")).pipe( - Effect.flatMap( - Schema.decodeUnknownEffect( - Schema.fromJsonString( - Schema.Struct({ - repository: Schema.NonEmptyString, - revision: Schema.String.check(Schema.isPattern(/^[a-f0-9]{40}$/u)), - }), - ), - ), - ), - ); - // Windows TEMP can use an 8.3 alias. Bun resolves workspace members to long - // paths, so canonicalize their root too before it compares the frozen lock. - const scratch = yield* fs - .makeTempDirectoryScoped({ prefix: "executor-motel-build-" }) - .pipe( - Effect.flatMap((directory) => Effect.tryPromise(() => promisify(realpath.native)(directory))), - ); - const command = (binary: string, args: readonly string[]) => - processes - .exitCode( - ChildProcess.make(binary, args, { - cwd: scratch, - stdout: "inherit", - stderr: "inherit", - }), - ) - .pipe( - Effect.flatMap((code) => - code === 0 ? Effect.void : Effect.die(new Error(`Motel build command exited ${code}`)), - ), - ); - const bun = (args: readonly string[]) => command(process.execPath, args); - yield* command("git", ["init", "--quiet"]); - yield* command("git", ["fetch", "--depth=1", pinned.repository, pinned.revision]); - yield* command("git", ["checkout", "--quiet", "--detach", "FETCH_HEAD"]); - yield* bun(["install", "--frozen-lockfile"]); - yield* bun(["run", "web:build"]); - // Replace the whole bundle so no file from an earlier build survives. - yield* fs.remove(output, { recursive: true, force: true }); - yield* fs.makeDirectory(output, { recursive: true }); - yield* bun(["run", "workerd:build"]); - yield* fs.copy(path.join(scratch, "dist/workerd"), output, { overwrite: true }); - // Hosts serve the bundle from this config; they supply its data directory and port. - yield* fs.copyFile(path.join(source, "motel.capnp"), path.join(output, "motel.capnp")); - yield* fs.copy(path.join(scratch, "web/dist"), path.join(output, "web/dist"), { - overwrite: true, - }); - yield* fs.copy(path.join(scratch, "skills"), path.join(output, "skills"), { overwrite: true }); - yield* fs.copyFile(path.join(scratch, "LICENSE"), path.join(output, "MOTEL-LICENSE")); - yield* fs.writeFileString( - path.join(output, "build.json"), - JSON.stringify({ - ...pinned, - runtime: "workerd", - }), - ); - yield* Effect.logInfo(`Built Motel for workerd at ${output}`); -}); - -BunRuntime.runMain(Effect.scoped(build).pipe(Effect.provide(BunServices.layer))); diff --git a/packages/telemetry/src/collector.ts b/packages/telemetry/src/collector.ts index d2c7c98438..7116b9339a 100644 --- a/packages/telemetry/src/collector.ts +++ b/packages/telemetry/src/collector.ts @@ -1,60 +1,70 @@ -/** Effect owns the bundled Motel workerd process and restarts it after unexpected exits. */ -import { createRequire } from "node:module"; +/** Effect owns the bundled telemetry collector process and restarts it after unexpected exits. */ import { Deferred, Effect, FileSystem, Path, Schema, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; -/** workerd's control message once the collector socket accepts connections. */ +/** The collector's one line on stdout, once it accepts connections on its TCP port. */ const Listening = Schema.Struct({ - event: Schema.Literal("listen"), - socket: Schema.Literal("motel"), + event: Schema.Literal("listening"), port: Schema.Number.check( Schema.makeFilter((port) => Number.isInteger(port) && port > 0 && port < 65_536), ), }); -const WorkerdPackage = Schema.Struct({ default: Schema.String }); +/** + * The bounds a local host gives its collector: the ones it gave Motel, a week of telemetry in at + * most 1 GiB and exports of at most 16 MiB, and four exports at once. + */ +const bounds = [ + "--retention-hours=168", + "--max-store-mb=1024", + "--max-export-bytes=16777216", + "--max-pending=4", +]; + +/** Where a build of this package keeps the collector and the viewer it serves. */ +export interface CollectorFiles { + /** The `executor-collector` executable. */ + readonly executable: string; + /** The viewer's build, served at the collector's origin root. */ + readonly viewer: string; +} /** - * Serve the Motel bundle with the workerd executable the runtime already ships. The bundle's - * config names its data and asset directories; the host supplies their paths and the port. - * Port 0 binds any free port, which workerd reports on descriptor 3. + * Serve the collector on loopback. Port 0 binds any free port, which the collector reports in its + * listening line. */ -const serveMotel = (bundle: string, data: string, port: number) => +const serveCollector = (files: CollectorFiles, data: string, port: number) => Effect.gen(function* () { const path = yield* Path.Path; const processes = yield* ChildProcessSpawner.ChildProcessSpawner; - const workerd = yield* Effect.try(() => createRequire(import.meta.url)("workerd")).pipe( - Effect.flatMap(Schema.decodeUnknownEffect(WorkerdPackage)), - ); const child = yield* processes.spawn( ChildProcess.make( - workerd.default, + files.executable, [ "serve", - "--control-fd=3", - `--directory-path=motel-data=${data}`, - `--directory-path=motel-assets=${path.join(bundle, "web/dist")}`, - `--socket-addr=motel=127.0.0.1:${port}`, - path.join(bundle, "motel.capnp"), + `--data=${data}`, + `--listen=127.0.0.1:${port}`, + `--viewer=${files.viewer}`, + ...bounds, ], { - // A process's working directory cannot be removed or renamed on Windows. workerd can - // outlive the parent's reported exit while it terminates, so it never runs inside data. - cwd: bundle, + // A process's working directory cannot be removed or renamed on Windows. The collector + // can outlive the parent's reported exit while it terminates, so it never runs inside data. + cwd: path.dirname(files.executable), stdin: "ignore", stdout: "pipe", stderr: "pipe", - additionalFds: { fd3: { type: "output" } }, - // workerd does not watch its parent. It shares the parent's process group on POSIX and - // Node's kill-on-close job on Windows, so stopping the parent also stops workerd. + // The collector does not watch its parent. It shares the parent's process group on POSIX + // and Node's kill-on-close job on Windows, so stopping the parent also stops it. detached: false, + // It stops within 2.5 seconds of SIGTERM, keeping every export it acknowledged. killSignal: "SIGTERM", forceKillAfter: 3_000, }, ), ); const listening = yield* Deferred.make(); - yield* child.getOutputFd(3).pipe( + yield* child.stdout.pipe( Stream.decodeText, Stream.splitLines, Stream.runForEach((line) => @@ -69,13 +79,13 @@ const serveMotel = (bundle: string, data: string, port: number) => }); /** Start supervision immediately; only exports await the collector's first ready address. */ -export const startCollector = (directory: string, bundle: string) => +export const startCollector = (directory: string, files: CollectorFiles) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const initial = yield* Deferred.make(); const statusPath = path.join(directory, "collector.json"); - const data = path.join(directory, "motel"); + const data = path.join(directory, "collector"); yield* fs.makeDirectory(data, { recursive: true, mode: 0o700 }); let pid: number | undefined; // A restart keeps the first bound port, so exporters keep one address. @@ -98,20 +108,16 @@ export const startCollector = (directory: string, bundle: string) => yield* Effect.addFinalizer(() => status("stopped").pipe(Effect.orDie)); const run = Effect.scoped( Effect.gen(function* () { - const { child, listening } = yield* serveMotel(bundle, data, port); + const { child, listening } = yield* serveCollector(files, data, port); pid = child.pid; - yield* Effect.forEach( - [child.stdout, child.stderr], - (output) => - output.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => - Effect.logInfo(line).pipe(Effect.annotateLogs({ process: "motel" })), - ), - Effect.forkScoped, - ), - { discard: true }, + // Its own log: JSON lines on stderr. + yield* child.stderr.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runForEach((line) => + Effect.logInfo(line).pipe(Effect.annotateLogs({ process: "collector" })), + ), + Effect.forkScoped, ); yield* listening.pipe( Effect.tap((bound) => diff --git a/packages/telemetry/src/local.ts b/packages/telemetry/src/local.ts index 43905bfc46..959c617f88 100644 --- a/packages/telemetry/src/local.ts +++ b/packages/telemetry/src/local.ts @@ -1,5 +1,5 @@ /** Local composition owns persistent files and its optional bundled collector. */ -import { Config, Deferred, Effect, Layer, Logger, Path, Scope } from "effect"; +import { Deferred, Effect, Layer, Logger, Path, Scope } from "effect"; import { FetchHttpClient, HttpClient, HttpClientError, HttpClientRequest } from "effect/http"; import { CurrentTelemetryClient } from "./transport.ts"; import { telemetryConfig } from "./config.ts"; @@ -21,7 +21,7 @@ export interface LocalTelemetryOptions { readonly collectorScope?: Effect.Effect | undefined; } -/** Local hosts keep JSONL logs; absent explicit exporters, they own a loopback Motel workerd. */ +/** Local hosts keep JSONL logs; absent explicit exporters, they own a loopback collector. */ export const localTelemetry = ( directory: string, service: string, @@ -42,12 +42,18 @@ export const localTelemetry = ( const config = yield* telemetryConfig(service); if (config.traces !== undefined || config.logs !== undefined) return telemetryLayer(config, "process", logger); - const defaultBundle = yield* path.fromFileUrl(new URL("../dist/motel", import.meta.url)); - const bundle = yield* Config.String("EXECUTOR_MOTEL_BUNDLE").pipe( - Config.withDefault(defaultBundle), - ); + // The collector's build (`collector:build`) and the viewer's, beside this package's source. + const files = { + executable: yield* path.fromFileUrl( + new URL( + `../dist/collector/executor-collector${process.platform === "win32" ? ".exe" : ""}`, + import.meta.url, + ), + ), + viewer: yield* path.fromFileUrl(new URL("../dist/viewer", import.meta.url)), + }; const ready = yield* Deferred.make(); - const start = startCollector(diagnostics, bundle).pipe( + const start = startCollector(diagnostics, files).pipe( Effect.flatMap((address) => Deferred.completeWith(ready, address)), ); if (options.collectorScope === undefined) yield* start; @@ -62,7 +68,7 @@ export const localTelemetry = ( const client = yield* HttpClient.HttpClient.pipe(Effect.provide(FetchHttpClient.layer)); // This internal address never reaches the network. Resolve only telemetry // requests after readiness; native exporter buffers/timeouts own delivery. - const origin = "http://motel.internal"; + const origin = "http://collector.internal"; const transport = client.pipe( HttpClient.mapRequestEffect((request) => { if (new URL(request.url).origin !== origin) return Effect.succeed(request); diff --git a/packages/telemetry/viewer/README.md b/packages/telemetry/viewer/README.md index 78864b5419..dd451eb9f2 100644 --- a/packages/telemetry/viewer/README.md +++ b/packages/telemetry/viewer/README.md @@ -1,11 +1,11 @@ # Telemetry viewer Executor telemetry: the trace and log viewer for the bundled telemetry collector. Its source is Motel's `web/` -viewer at the revision `packages/telemetry/motel/source.json` pins, under Motel's MIT license +viewer in [`UsefulSoftwareCo/motel`](https://github.com/UsefulSoftwareCo/motel) at `b1783f4d344fd2e0fa584ef00b3e71838a8a1e03`, under Motel's MIT license (`LICENSE`). The AI calls page is removed, and `src/contracts/api.ts` declares only the routes the viewer reads, with the response shapes that Motel revision returns. -Nothing serves this build yet: Motel still serves its own viewer. +The collector serves this build when it is started with `--viewer ` (its README, The viewer). ## Build @@ -13,7 +13,7 @@ Nothing serves this build yet: Motel still serves its own viewer. bun run --cwd packages/telemetry/viewer build ``` -This writes `packages/telemetry/dist/collector/viewer/`: `index.html` and hashed files under +This writes `packages/telemetry/dist/viewer/`, beside the collector's own `dist/collector/`, which its build replaces whole: `index.html` and hashed files under `assets/`, and the notices for the code in them: - `runtime-packages.txt` lists, as `name@version`, every package the bundler put into the diff --git a/packages/telemetry/viewer/src/contracts/api.ts b/packages/telemetry/viewer/src/contracts/api.ts index accc221f5f..de5a071b11 100644 --- a/packages/telemetry/viewer/src/contracts/api.ts +++ b/packages/telemetry/viewer/src/contracts/api.ts @@ -1,6 +1,7 @@ /** * The collector routes the viewer reads, with the response shapes Motel returns at the revision - * pinned in `packages/telemetry/motel/source.json` (its `src/domain.ts` and `src/httpApi.ts`). + * b1783f4d344fd2e0fa584ef00b3e71838a8a1e03 of UsefulSoftwareCo/motel (its `src/domain.ts` and + * `src/httpApi.ts`). * The collector that replaces Motel returns the same shapes, so this schema serves both, with one * addition Motel never sends: `partial` on a trace answered in part (the collector's README, * "A trace answered in part"). diff --git a/packages/telemetry/viewer/vite.config.ts b/packages/telemetry/viewer/vite.config.ts index 2fd8b2d61a..a5b2097cff 100644 --- a/packages/telemetry/viewer/vite.config.ts +++ b/packages/telemetry/viewer/vite.config.ts @@ -12,7 +12,7 @@ export default defineConfig({ plugins: [notices.recordingStylesheetSources(tailwindcss()), react(), notices.write], server: collector === undefined ? {} : { proxy: { "/api": collector } }, build: { - outDir: "../dist/collector/viewer", + outDir: "../dist/viewer", emptyOutDir: true, }, }); diff --git a/packages/ui/src/contracts/app-accounts.ts b/packages/ui/src/contracts/app-accounts.ts index ed94f0becb..a147fbad5d 100644 --- a/packages/ui/src/contracts/app-accounts.ts +++ b/packages/ui/src/contracts/app-accounts.ts @@ -11,6 +11,18 @@ export interface AppAccountLink { readonly account: AccountId; } +/** + * An account linked to one of an app's app requirements, shared by everyone using the app. The + * caller may not own it, so its label and provider come with the link. + */ +export interface SharedAppLink { + readonly app: AppId; + readonly requirement: string; + readonly account: AccountId; + readonly provider: ProviderId; + readonly label: string; +} + /** The ids of the accounts linked to one app, in link order. */ export const linkedAccountIds = (links: readonly AppAccountLink[], app: AppId) => links.flatMap((link) => (link.app === app ? [link.account] : [])); diff --git a/packages/ui/src/contracts/dashboard.ts b/packages/ui/src/contracts/dashboard.ts index 83e946a80e..713acc49b3 100644 --- a/packages/ui/src/contracts/dashboard.ts +++ b/packages/ui/src/contracts/dashboard.ts @@ -18,6 +18,7 @@ import { missingRequirements, usedAccounts, type AppAccountLink, + type SharedAppLink, type RequirementAccounts, } from "./app-accounts.ts"; import type { @@ -61,6 +62,8 @@ export interface Inventory { readonly apps: readonly App[]; /** The caller's linked accounts for each listed app. */ readonly links: readonly AppAccountLink[]; + /** The app links of each listed app, oldest first. */ + readonly appLinks: readonly SharedAppLink[]; readonly accounts: readonly AccountSummary[]; } /** Any Effect Atom source, including a live stream or a one-shot HTTP query. */ diff --git a/packages/ui/src/implementation/dashboard/accounts.tsx b/packages/ui/src/implementation/dashboard/accounts.tsx index 46d84686cd..04eca16141 100644 --- a/packages/ui/src/implementation/dashboard/accounts.tsx +++ b/packages/ui/src/implementation/dashboard/accounts.tsx @@ -118,7 +118,9 @@ function AccountsList({ {accounts.map((account) => { const apps = data.apps.filter((app) => - data.links.some((link) => link.app === app.id && link.account === account.id), + [...data.links, ...data.appLinks].some( + (link) => link.app === app.id && link.account === account.id, + ), ); return (
{pending} {apps.map((app) => { - const ids = linkedAccountIds(data.links, app.id); - const selected = ids.flatMap((id) => { - const account = data.accounts.find((item) => item.id === id); + const own = (id: AccountId) => data.accounts.find((item) => item.id === id); + const userLinked = linkedAccountIds(data.links, app.id).flatMap((id) => { + const account = own(id); return account === undefined ? [] : [account]; }); - // The inventory names no requirements, so a user requirement counts any linked - // account of its provider; app links are not listed here. + const shared = data.appLinks.filter((link) => link.app === app.id); + // The inventory names no requirements for user links, so a user requirement counts + // any linked account of its provider. App links name theirs. const user = Object.entries(app.requirements.accounts).flatMap(([slot, requirement]) => requirement.scope === "user" - ? selected + ? userLinked .filter((account) => account.provider === requirement.provider) .map((account) => ({ requirement: slot, account })) : [], ); - const missing = missingRequirements(requirementAccounts(app, { user })).length > 0; - const needsSignIn = selected.some((account) => accountNeedsSignIn(account)); - const unavailable = selected.some((account) => account.signIn?.state === "unavailable"); + const appScope = shared.map((link) => ({ + requirement: link.requirement, + account: { id: link.account, provider: link.provider }, + })); + const missing = + missingRequirements(requirementAccounts(app, { user, app: appScope })).length > 0; + // Each linked account once, user links first. A shared account the caller does not + // own shows its link's label and no sign-in state. + const linked: { readonly id: AccountId; readonly label: string }[] = []; + for (const account of [ + ...userLinked, + ...shared.map((link) => own(link.account) ?? { id: link.account, label: link.label }), + ]) + if (!linked.some((item) => item.id === account.id)) linked.push(account); + const owned = linked.flatMap((account) => { + const found = own(account.id); + return found === undefined ? [] : [found]; + }); + const needsSignIn = owned.some((account) => accountNeedsSignIn(account)); + const unavailable = owned.some((account) => account.signIn?.state === "unavailable"); const provider = Object.values(app.requirements.accounts)[0]?.definition; return ( @@ -167,7 +186,7 @@ function AppsList({ Needs account - ) : selected.length === 0 ? ( + ) : linked.length === 0 ? ( {provider === undefined ? "No account required" : "No accounts linked"} @@ -184,9 +203,7 @@ function AppsList({ )} - {selected - .map((account) => account.label || "Unnamed account") - .join(", ")} + {linked.map((account) => account.label || "Unnamed account").join(", ")} )} diff --git a/packages/ui/src/implementation/dashboard/oauth-fields.tsx b/packages/ui/src/implementation/dashboard/oauth-fields.tsx index 3f7f4ad7c9..3561bf5fdf 100644 --- a/packages/ui/src/implementation/dashboard/oauth-fields.tsx +++ b/packages/ui/src/implementation/dashboard/oauth-fields.tsx @@ -2,7 +2,12 @@ import { useEffect, useState } from "react"; import { ArrowDown01Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { Cause, Exit, Option, Redacted } from "effect"; -import type { Account, OAuthClientId, OAuthConnectOptions } from "@executor-js/sdk"; +import { + OAuthSetupFailed, + type Account, + type OAuthClientId, + type OAuthConnectOptions, +} from "@executor-js/sdk"; import type { OAuthSubmission } from "../../contracts/credentials.ts"; import type { FailureProps, Query } from "../../contracts/dashboard.ts"; import type { ComponentType, ReactNode } from "react"; @@ -183,6 +188,15 @@ export function OAuthFields({ : acceptsSecret ? "client ID (and secret, if it has one)" : "client ID"; + // The instance's own client matches the service but may not ask for these scopes, so the person + // enters their own; that explanation takes the place of the line of help. + const refusal = + setup !== "unresolved" && setup.operatorScopeRefusal !== undefined + ? new OAuthSetupFailed({ + reason: "operator_scope_refused", + operatorScopes: setup.operatorScopeRefusal, + }) + : undefined; // Client entry's one line of help; a failure, which carries its own recovery, replaces it. const guidance = machine ? `Create an OAuth client in ${providerName}’s developer settings${setup.scopes.length > 0 ? " with the permissions under Advanced" : ""}, then enter its ${details}.` @@ -312,6 +326,8 @@ export function OAuthFields({ {prominent && clientSelect} {error ? ( + ) : manual && refusal !== undefined ? ( + ) : ( manual && (

{guidance}

diff --git a/patches/@cloudflare%2Fworker-bundler@0.2.4.patch b/patches/@cloudflare%2Fworker-bundler@0.2.4.patch index e192b4ab85..3c87cdbe40 100644 --- a/patches/@cloudflare%2Fworker-bundler@0.2.4.patch +++ b/patches/@cloudflare%2Fworker-bundler@0.2.4.patch @@ -21,7 +21,7 @@ index fa559109c179700e47dd94029abea7d76a00fb50..aa562a81a15e168b321e063ce668d85f list(prefix?: string): string[]; flush(): Promise; diff --git a/dist/index.d.ts b/dist/index.d.ts -index 53dd55f4cd0292a11050505c4824afe0cac15d33..f2ef7c5e31a0561eedcd59d11abe74fb43b68fd8 100644 +index 53dd55f4cd0292a11050505c4824afe0cac15d33..6ccabb03b2982011b5ad2635f2f302a87d8c164f 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -331,6 +331,8 @@ declare function handleAssetRequest( @@ -33,12 +33,20 @@ index 53dd55f4cd0292a11050505c4824afe0cac15d33..f2ef7c5e31a0561eedcd59d11abe74fb /** * Input files — keys are paths relative to project root, values are file contents. * Accepts a plain object (which is wrapped in an InMemoryFileSystem automatically) -@@ -499,6 +501,8 @@ interface InstallResult { +@@ -499,6 +501,16 @@ interface InstallResult { //#endregion //#region src/installer.d.ts interface InstallOptions { + /** Skip dependency traversal for a self-contained framework archive. Default: true. */ + transitive?: boolean; ++ /** ++ * Fetches every request, and every redirect hop, whose origin is not the ++ * registry's: tarball URL versions at any depth, and tarballs or redirects ++ * registry metadata sends elsewhere. Requests on the registry's origin use the ++ * global fetch. Reject with an error named `NetworkRefused` for a destination ++ * the host's network policy refuses. Default: the global fetch. ++ */ ++ tarballFetch?: (input: string, init?: RequestInit) => Promise; /** * Include devDependencies (default: false) */ @@ -117,26 +125,101 @@ index 9ddcbac086bbd60fece6571299c86f92d33412a6..2f2667088015a158ebd9f1d324137199 installWarnings.push(...installResult.warnings); } diff --git a/dist/installer-DDvaMqav.js b/dist/installer-DDvaMqav.js -index 5ed55da1726ab87d1c11932262f972cf39c3dd45..1f0d34b1860380058a4f43092355267ac36b4045 100644 +index 5ed55da1726ab87d1c11932262f972cf39c3dd45..c50d0cf085a7dffd44af6558f56c5a627bc8e2ec 100644 --- a/dist/installer-DDvaMqav.js +++ b/dist/installer-DDvaMqav.js -@@ -381,7 +381,7 @@ const NPM_REGISTRY = "https://registry.npmjs.org"; +@@ -49,21 +49,63 @@ function isTextFile(path) { + * Fetch with a timeout. + * Throws an error if the request takes longer than the specified timeout. + */ +-async function fetchWithTimeout(url, options = {}, timeoutMs = DEFAULT_TIMEOUT_MS) { ++async function fetchWithTimeout(url, options = {}, timeoutMs = DEFAULT_TIMEOUT_MS, send = fetch) { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + try { +- return await fetch(url, { ++ return await send(url, { + ...options, + signal: controller.signal + }); + } catch (error) { +- if (error instanceof Error && error.name === "AbortError") throw new Error(`Request to ${url} timed out after ${timeoutMs}ms (npm registry slow or unreachable from this Worker)`); +- throw error; ++ if (error instanceof Error && error.name === "AbortError") throw new InstallFailure(`the request timed out after ${timeoutMs}ms`); ++ throw new InstallFailure(networkFailure(error)); + } finally { + clearTimeout(timeoutId); + } + } ++/** ++* A failure the installer words itself. Its message names no URL a server ++* chose and repeats nothing a server or an archive sent; any other error is ++* reported as unexpected. ++*/ ++var InstallFailure = class extends Error {}; ++/** ++* Why a request failed to send. A fetch hook rejects with an error named ++* `NetworkRefused` for a destination its network policy refuses. workerd ++* reports DNS, TLS and address-policy failures alike as an internal error ++* whose reference names the cause in workerd's own log. ++*/ ++function networkFailure(error) { ++ if (error instanceof Error && error.name === "NetworkRefused") return "the network policy refused its address"; ++ const reference = error instanceof Error ? /^internal error; reference = ([0-9a-z]+)$/.exec(error.message)?.[1] : void 0; ++ if (reference !== void 0) return `the connection failed in a DNS lookup, a TLS handshake or the network's address policy (workerd error reference ${reference})`; ++ return "the connection failed"; ++} ++const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); ++const MAX_REDIRECTS = 10; ++/** ++* Fetch `url`, following each redirect here. Every hop on the registry's ++* origin uses the global fetch, the host's own network; every other hop, ++* where a redirect lands included, uses `network.elsewhere`. ++*/ ++async function fetchFrom(url, options, timeoutMs, network) { ++ let current = url; ++ for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { ++ const send = URL.parse(current)?.origin === network.registryOrigin ? fetch : network.elsewhere; ++ const response = await fetchWithTimeout(current, { ++ ...options, ++ redirect: "manual" ++ }, timeoutMs, send); ++ const location = REDIRECT_STATUSES.has(response.status) ? response.headers.get("location") : null; ++ if (location === null) return response; ++ response.body?.cancel().catch(() => {}); ++ const next = URL.parse(location, current); ++ if (next === null || next.protocol !== "https:" && next.protocol !== "http:") throw new InstallFailure("a redirect named an invalid URL"); ++ current = next.href; ++ } ++ throw new InstallFailure("it was redirected too many times"); ++} + //#endregion + //#region src/installer-python.ts + const PYODIDE_VERSION = "v0.27.5"; +@@ -381,7 +423,11 @@ const NPM_REGISTRY = "https://registry.npmjs.org"; * @returns Metadata about the installation */ async function installDependencies(fileSystem, options = {}) { - const { dev = false, registry = NPM_REGISTRY } = options; -+ const { dev = false, registry = NPM_REGISTRY, transitive = true } = options; ++ const { dev = false, registry = NPM_REGISTRY, transitive = true, tarballFetch = fetch } = options; ++ const network = { ++ registryOrigin: URL.parse(registry)?.origin, ++ elsewhere: tarballFetch ++ }; const result = { installed: [], warnings: [] -@@ -407,14 +407,16 @@ async function installDependencies(fileSystem, options = {}) { +@@ -407,14 +453,19 @@ async function installDependencies(fileSystem, options = {}) { if (Object.keys(depsToInstall).length === 0) return result; const installedPackages = /* @__PURE__ */ new Set(); const inProgress = /* @__PURE__ */ new Map(); - await Promise.all(Object.entries(depsToInstall).map(([name, versionRange]) => installPackage(name, versionRange, result, fileSystem, installedPackages, inProgress, registry))); + // Unpacked archives are the install's memory peak: unpack a few at a time. -+ const extract = limitConcurrency(TARBALL_EXTRACT_CONCURRENCY, (name, metadata) => fetchPackageFiles(name, metadata, { buildInputsOnly: true })); -+ await Promise.all(Object.entries(depsToInstall).map(([name, versionRange]) => installPackage(name, versionRange, result, fileSystem, installedPackages, inProgress, registry, transitive, extract))); ++ const extract = limitConcurrency(TARBALL_EXTRACT_CONCURRENCY, (tarballUrl) => fetchPackageFiles(void 0, { dist: { tarball: tarballUrl } }, { ++ buildInputsOnly: true, ++ network ++ })); ++ await Promise.all(Object.entries(depsToInstall).map(([name, versionRange]) => installPackage(name, versionRange, result, fileSystem, installedPackages, inProgress, registry, transitive, extract, network, void 0))); } else if (pyprojectTomlContent) return await installDependenciesPython(fileSystem, pyprojectTomlContent); return result; } @@ -144,44 +227,110 @@ index 5ed55da1726ab87d1c11932262f972cf39c3dd45..1f0d34b1860380058a4f43092355267a * Install a single package and its dependencies recursively. */ -async function installPackage(name, versionRange, result, fileSystem, installedPackages, inProgress, registry) { -+async function installPackage(name, versionRange, result, fileSystem, installedPackages, inProgress, registry, transitive, extract) { ++async function installPackage(name, versionRange, result, fileSystem, installedPackages, inProgress, registry, transitive, extract, network, root) { if (installedPackages.has(name)) return; if (fileSystem.read(`node_modules/${name}/package.json`) !== null) { installedPackages.add(name); -@@ -424,6 +426,19 @@ async function installPackage(name, versionRange, result, fileSystem, installedP +@@ -422,28 +473,43 @@ async function installPackage(name, versionRange, result, fileSystem, installedP + } + const existing = inProgress.get(name); if (existing) return existing; ++ const installDependency = (depName, depVersion) => installPackage(depName, depVersion, result, fileSystem, installedPackages, inProgress, registry, transitive, extract, network, root ?? name); const installPromise = (async () => { try { +- const metadata = await fetchPackageMetadata(name, registry); +- const version = resolveVersion(versionRange, metadata); +- if (!version) { +- result.warnings.push(`Could not resolve version for ${name}@${versionRange}`); + // npm permits HTTP tarballs, including pinned framework dependencies. + // Read their manifest from the archive; do not execute package scripts. + if (/^https?:\/\//.test(versionRange)) { -+ const packageFiles = await extract(name, { dist: { tarball: versionRange } }); -+ const manifest = JSON.parse(packageFiles["package.json"]); -+ if (manifest.name !== name || typeof manifest.version !== "string") -+ throw new Error("Tarball package identity does not match its dependency"); ++ const packageFiles = await extract(versionRange); ++ let manifest; ++ try { ++ manifest = JSON.parse(packageFiles["package.json"]); ++ } catch { ++ throw new InstallFailure("the tarball has no valid package.json"); ++ } ++ if (typeof manifest !== "object" || manifest === null || manifest.name !== name || typeof manifest.version !== "string") throw new InstallFailure("the tarball's package.json names another package"); ++ const deps = dependencyMap(manifest.dependencies, "the tarball's package.json lists invalid dependencies"); + installedPackages.add(name); + for (const [filePath, content] of Object.entries(packageFiles)) fileSystem.write(`node_modules/${name}/${filePath}`, content); + result.installed.push(`${name}@${manifest.version}`); -+ if (transitive) await Promise.all(Object.entries(manifest.dependencies ?? {}).map(([depName, depVersion]) => installPackage(depName, depVersion, result, fileSystem, installedPackages, inProgress, registry, transitive, extract))); -+ return; -+ } - const metadata = await fetchPackageMetadata(name, registry); - const version = resolveVersion(versionRange, metadata); - if (!version) { -@@ -437,10 +452,10 @@ async function installPackage(name, versionRange, result, fileSystem, installedP ++ if (transitive) await Promise.all(Object.entries(deps).map(([depName, depVersion]) => installDependency(depName, depVersion))); + return; } ++ const metadata = await fetchPackageMetadata(name, registry, network); ++ const version = resolveVersion(versionRange, metadata); ++ if (!version) throw new InstallFailure("no version in the registry matches its declared range"); + const versionMetadata = metadata.versions[version]; +- if (!versionMetadata) { +- result.warnings.push(`Version ${version} not found for ${name}`); +- return; +- } ++ if (typeof versionMetadata !== "object" || versionMetadata === null || typeof versionMetadata.dist?.tarball !== "string") throw new InstallFailure("the registry's metadata names no tarball for the version it resolved"); ++ const deps = dependencyMap(versionMetadata.dependencies, "the registry's metadata lists invalid dependencies"); installedPackages.add(name); result.installed.push(`${name}@${version}`); - const packageFiles = await fetchPackageFiles(name, versionMetadata); -+ const packageFiles = await extract(name, versionMetadata); ++ const packageFiles = await extract(versionMetadata.dist.tarball); for (const [filePath, content] of Object.entries(packageFiles)) fileSystem.write(`node_modules/${name}/${filePath}`, content); - const deps = versionMetadata.dependencies ?? {}; +- const deps = versionMetadata.dependencies ?? {}; - await Promise.all(Object.entries(deps).map(([depName, depVersion]) => installPackage(depName, depVersion, result, fileSystem, installedPackages, inProgress, registry))); -+ if (transitive) await Promise.all(Object.entries(deps).map(([depName, depVersion]) => installPackage(depName, depVersion, result, fileSystem, installedPackages, inProgress, registry, transitive, extract))); ++ if (transitive) await Promise.all(Object.entries(deps).map(([depName, depVersion]) => installDependency(depName, depVersion))); } catch (error) { - const message = error instanceof Error ? error.message : String(error); - result.warnings.push(`Failed to install ${name}: ${message}`); -@@ -475,82 +490,85 @@ function resolveVersion(range, metadata) { +- const message = error instanceof Error ? error.message : String(error); +- result.warnings.push(`Failed to install ${name}: ${message}`); ++ // A dependency's name and range came from a server or an archive: only ++ // the declared package is named. ++ const reason = error instanceof InstallFailure ? error.message : "it failed unexpectedly"; ++ result.warnings.push(root === void 0 ? `Failed to install ${name}: ${reason}` : `Failed to install a dependency of ${root}: ${reason}`); + } + })(); + inProgress.set(name, installPromise); +@@ -453,104 +519,128 @@ async function installPackage(name, versionRange, result, fileSystem, installedP + inProgress.delete(name); + } + } ++/** A manifest's dependencies: absent, or package names mapped to version strings. */ ++function dependencyMap(value, invalid) { ++ if (value === void 0) return {}; ++ if (typeof value !== "object" || value === null || Object.values(value).some((range) => typeof range !== "string")) throw new InstallFailure(invalid); ++ return value; ++} + /** + * Fetch package metadata from npm registry. + */ +-async function fetchPackageMetadata(name, registry) { ++async function fetchPackageMetadata(name, registry, network) { + const url = `${registry}/${name.startsWith("@") ? `@${encodeURIComponent(name.slice(1))}` : name}`; +- const response = await fetchWithTimeout(url, { headers: { Accept: "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8" } }); +- if (!response.ok) { +- const hint = response.status === 404 ? " (package not found — check the name in package.json or set the `registry` option if it lives on a private registry)" : ""; +- throw new Error(`Registry returned ${response.status} ${response.statusText} for "${name}" at ${url}${hint}`); ++ const response = await fetchFrom(url, { headers: { Accept: "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8" } }, DEFAULT_TIMEOUT_MS, network); ++ if (response.status === 404) throw new InstallFailure(`the registry at ${registry} has no such package (HTTP 404)`); ++ if (!response.ok) throw new InstallFailure(`the registry at ${registry} answered HTTP ${response.status}`); ++ let metadata; ++ try { ++ metadata = await response.json(); ++ } catch { ++ throw new InstallFailure("the registry's metadata is not valid JSON"); + } +- return await response.json(); ++ if (typeof metadata !== "object" || metadata === null || typeof metadata.versions !== "object" || metadata.versions === null) throw new InstallFailure("the registry's metadata has no versions"); ++ return metadata; + } + /** + * Resolve a semver range to a specific version. + */ + function resolveVersion(range, metadata) { +- if (range === "latest" || range === "*") return metadata["dist-tags"]["latest"]; ++ const tags = typeof metadata["dist-tags"] === "object" && metadata["dist-tags"] !== null ? metadata["dist-tags"] : {}; ++ if (range === "latest" || range === "*") return tags["latest"]; + if (metadata.versions[range]) return range; +- if (metadata["dist-tags"][range]) return metadata["dist-tags"][range]; ++ if (tags[range]) return tags[range]; const versions = Object.keys(metadata.versions); return semver.maxSatisfying(versions, range) ?? void 0; } @@ -206,18 +355,30 @@ index 5ed55da1726ab87d1c11932262f972cf39c3dd45..1f0d34b1860380058a4f43092355267a * Fetch and extract package files from npm tarball. +* +* `buildInputsOnly` keeps only files a bundle can read: no source maps, type -+* declarations or package documentation. ++* declarations or package documentation. `network` chooses each hop's fetch ++* (see `fetchFrom`); without it every hop uses the global fetch. */ -async function fetchPackageFiles(name, metadata) { +async function fetchPackageFiles(name, metadata, options = {}) { const tarballUrl = metadata.dist?.tarball; - if (!tarballUrl) throw new Error(`Registry metadata for ${name}@${metadata.version} is missing \`dist.tarball\` — the registry response is likely malformed or the version was unpublished.`); - const response = await fetchWithTimeout(tarballUrl, {}, DEFAULT_TIMEOUT_MS * 2); - if (!response.ok) throw new Error(`Failed to fetch tarball for ${name}@${metadata.version}: ${response.status} ${response.statusText} (${tarballUrl})`); +- if (!tarballUrl) throw new Error(`Registry metadata for ${name}@${metadata.version} is missing \`dist.tarball\` — the registry response is likely malformed or the version was unpublished.`); +- const response = await fetchWithTimeout(tarballUrl, {}, DEFAULT_TIMEOUT_MS * 2); +- if (!response.ok) throw new Error(`Failed to fetch tarball for ${name}@${metadata.version}: ${response.status} ${response.statusText} (${tarballUrl})`); - const buffer = await response.arrayBuffer(); - return extractTarball(new Uint8Array(buffer)); -+ if (response.body === null) throw new Error(`Tarball for ${name}@${metadata.version} has no body (${tarballUrl})`); -+ return extractTarball(response.body, options.buildInputsOnly === true ? isBuildInput : isTextFile); ++ if (typeof tarballUrl !== "string") throw new InstallFailure("the registry's metadata names no tarball"); ++ const response = await fetchFrom(tarballUrl, {}, DEFAULT_TIMEOUT_MS * 2, options.network ?? { ++ registryOrigin: void 0, ++ elsewhere: fetch ++ }); ++ if (!response.ok) throw new InstallFailure(`the tarball download answered HTTP ${response.status}`); ++ if (response.body === null) throw new InstallFailure("the tarball download has no body"); ++ try { ++ return await extractTarball(response.body, options.buildInputsOnly === true ? isBuildInput : isTextFile); ++ } catch (error) { ++ if (error instanceof InstallFailure) throw error; ++ throw new InstallFailure("the tarball is not a valid gzip tar archive"); ++ } +} +/** Files a bundler never reads: source maps, type declarations and package documentation. */ +function isBuildInput(path) { @@ -311,7 +472,7 @@ index 5ed55da1726ab87d1c11932262f972cf39c3dd45..1f0d34b1860380058a4f43092355267a + const wasm = isRegularFile && size > 0 && filePath.endsWith(".wasm"); + const text = isRegularFile && size > 0 && !wasm && keep(filePath); + if (wasm || text) { -+ if (!await queue.fill(size)) throw new Error(`Tarball ends inside ${filePath}`); ++ if (!await queue.fill(size)) throw new InstallFailure("the tarball ends inside one of its files"); + const content = queue.take(size); + if (wasm) textFiles.set(filePath, { data: content }); + else try { @@ -326,7 +487,7 @@ index 5ed55da1726ab87d1c11932262f972cf39c3dd45..1f0d34b1860380058a4f43092355267a } const archiveRoot = findSharedTarRootDirectory(regularFilePaths); const files = {}; -@@ -560,6 +578,70 @@ function parseTar(data) { +@@ -560,6 +650,70 @@ function parseTar(data) { } return files; } diff --git a/patches/@effect%2Fsql-pg@4.1.4.patch b/patches/@effect%2Fsql-pg@4.1.4.patch index 81604cb259..943253c7fc 100644 --- a/patches/@effect%2Fsql-pg@4.1.4.patch +++ b/patches/@effect%2Fsql-pg@4.1.4.patch @@ -160,7 +160,7 @@ index 25c6fbe0c019cc0c5cd8160a517c6162c86214ea..1fd839230545d312f7dd5e811ffb32e5 //# sourceMappingURL=PgConnection.d.ts.map \ No newline at end of file diff --git a/dist/PgConnection.js b/dist/PgConnection.js -index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d73623728 100644 +index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..fdea9a6c014d1de38b0f7c760d4524bd25a1d56c 100644 --- a/dist/PgConnection.js +++ b/dist/PgConnection.js @@ -2,8 +2,11 @@ import * as Cause from "effect/Cause"; @@ -561,7 +561,7 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d if (!this.pipelineIsIdle()) return; this.consumer = undefined; this.session.parser.readField = undefined; -@@ -323,6 +614,7 @@ class PgConnectionImpl { +@@ -323,11 +614,16 @@ class PgConnectionImpl { this.pipelineInFlight.length = 0; this.pipelineHead = 0; } @@ -569,7 +569,25 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d }; /** Writes everything queued since the last flush as one batch. */ flushPipeline = () => { -@@ -352,7 +644,10 @@ class PgConnectionImpl { + this.pipelineFlushScheduled = false; + if (this.deadWith !== undefined) return; ++ // Nothing is written behind a batch's `COMMIT` until it has answered: through Hyperdrive, a ++ // statement written while a `BEGIN` … `COMMIT` on the connection is still unanswered is never ++ // answered. What waits stays queued; the flush after the answer's chunk (`onBatchEnd`) sends it. ++ if (this.transactionOnWire()) return; + let capacity = maxPipelineDepth - this.pipelineDepth(); + if (capacity <= 0) return; + const wasEmpty = this.pipelineDepth() === 0; +@@ -344,6 +640,8 @@ class PgConnectionImpl { + batch.push(entry); + this.pipelineInFlight.push(entry); + capacity--; ++ // A batch's `COMMIT` ends the write; what was queued after it waits for its answer. ++ if (entry.closesTransaction === true) break; + } + this.pipelinePending.splice(0, index); + if (batch.length === 0) { +@@ -352,25 +650,112 @@ class PgConnectionImpl { } if (wasEmpty) this.session.parser.readField = batch[0].machine.readField; const frame = batch.length === 1 ? batch[0].plan.frame : concat(batch.map(entry => entry.plan.frame)); @@ -580,7 +598,18 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d this.session.socket.write(frame); } catch (cause) { this.fatal(connectionQueryError(cause, "PgConnection: Failed to write query batch")); -@@ -364,13 +659,90 @@ class PgConnectionImpl { + } + }; ++ /** Whether a batch's `COMMIT` is written and not yet answered. */ ++ transactionOnWire = () => { ++ for (let index = this.pipelineHead; index < this.pipelineInFlight.length; index++) { ++ if (this.pipelineInFlight[index].closesTransaction === true) return true; ++ } ++ return false; ++ }; + schedulePipelineFlush = () => { + if (this.pipelineFlushScheduled || this.pipelineDepth() >= maxPipelineDepth) return; + this.pipelineFlushScheduled = true; // A microtask is what lets fibers submitted in the same tick share a write. queueMicrotask(this.flushPipeline); }; @@ -672,7 +701,7 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d let plan; try { plan = this.encodeQuery(sql, params, cache); -@@ -387,21 +759,23 @@ class PgConnectionImpl { +@@ -387,21 +772,23 @@ class PgConnectionImpl { abandoned: false }; this.pipelinePending.push(entry); @@ -701,7 +730,7 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d /** * Runs one cycle. A reused statement the backend has since dropped, or whose * plan no longer matches its columns, is parsed again on a second attempt; -@@ -461,6 +835,82 @@ class PgConnectionImpl { +@@ -461,6 +848,84 @@ class PgConnectionImpl { const cache = prepare ? this.prepared : undefined; return this.multiplex ? this.pipelineCycle(sql, params, wantRows, cache) : this.owner.withPermit(this.cycleOwned(sql, params, wantRows, cache)); }; @@ -776,7 +805,9 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d + return entries; + } + for (const item of inside) enqueue(item.sql, item.params, item.wantRows, item.deferred, wrap); -+ if (wrap) enqueue("COMMIT", emptyParams, false, Deferred.makeUnsafe(), false); ++ if (wrap && enqueue("COMMIT", emptyParams, false, Deferred.makeUnsafe(), false)) { ++ entries[entries.length - 1].closesTransaction = true; ++ } + this.consumer = this.pipelineConsumer; + this.schedulePipelineFlush(); + return entries; @@ -784,7 +815,7 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d query = (sql, params, prepare = true) => Effect.map(this.run(sql, params ?? emptyParams, true, prepare), takeResult); queryValues = (sql, params, prepare = true) => Effect.map(this.run(sql, params ?? emptyParams, false, prepare), takeValues); stream = (sql, params) => streamRows(this, this.pin, sql, params ?? emptyParams); -@@ -1132,8 +1582,15 @@ const drainAborted = (conn, timeoutMessage, abort) => { +@@ -1132,8 +1597,15 @@ const drainAborted = (conn, timeoutMessage, abort) => { const cancelAndDrain = Effect.andThen(conn.cancel, Deferred.await(drained)); return Effect.andThen(Effect.raceFirst(Deferred.await(drained), grace), Effect.suspend(() => Deferred.isDoneUnsafe(drained) ? Effect.void : cancelAndDrain)); }; @@ -802,7 +833,7 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d if (conn.deadWith !== undefined) { resume(Effect.fail(conn.deadWith)); return; -@@ -1142,12 +1599,32 @@ const runQuery = (conn, plan, wantRows) => Effect.callback(resume => { +@@ -1142,12 +1614,32 @@ const runQuery = (conn, plan, wantRows) => Effect.callback(resume => { conn.consumer = undefined; resume(effect); }); @@ -837,7 +868,7 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d } catch (cause) { conn.fatal(connectionQueryError(cause, "PgConnection: Failed to write query")); } -@@ -1525,33 +2002,43 @@ const sendCancelRequest = (config, session) => Effect.callback(resume => { +@@ -1525,33 +2017,43 @@ const sendCancelRequest = (config, session) => Effect.callback(resume => { if (config.stream !== undefined) begin();else socket.once("connect", begin); return Effect.sync(finish); }); @@ -887,7 +918,7 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d const onError = cause => failConnect(cause, "PgConnection: Failed to connect"); const onClose = () => failConnect(new Error("Connection closed unexpectedly"), "PgConnection: Connection closed during startup"); const password = () => { -@@ -1649,7 +2136,7 @@ const connect = (config, resolvedPassword) => Effect.callback(resume => { +@@ -1649,7 +2151,7 @@ const connect = (config, resolvedPassword) => Effect.callback(resume => { secretKey = message.secret; return; case "ErrorResponse": @@ -896,7 +927,7 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d case "ReadyForQuery": if (scram !== undefined) { return failAuth(new Error("The server completed authentication without proving its identity"), "PgConnection: SCRAM exchange did not complete"); -@@ -1708,7 +2195,7 @@ const connect = (config, resolvedPassword) => Effect.callback(resume => { +@@ -1708,7 +2210,7 @@ const connect = (config, resolvedPassword) => Effect.callback(resume => { if (messages.length !== 1 || message._tag !== "ErrorResponse") { return failConnect(new Error("Expected one ErrorResponse after SSLRequest"), "PgConnection: Invalid SSLRequest response"); } @@ -905,7 +936,7 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d } socket.off("data", onSslResponse); if (chunk.length !== 1) { -@@ -1759,12 +2246,10 @@ const connect = (config, resolvedPassword) => Effect.callback(resume => { +@@ -1759,12 +2261,10 @@ const connect = (config, resolvedPassword) => Effect.callback(resume => { }) }); } catch (cause) { @@ -922,7 +953,7 @@ index 791d1cf5b0014993f13fda641d3c7a0be8e4dfda..5f2bf15aecb93d0d731c207d0033250d }))); return; } -@@ -1773,7 +2258,10 @@ const connect = (config, resolvedPassword) => Effect.callback(resume => { +@@ -1773,7 +2273,10 @@ const connect = (config, resolvedPassword) => Effect.callback(resume => { if (config.stream !== undefined) { begin(); } else { @@ -1055,7 +1086,7 @@ index 1b8881f1717666f7434fe6ac6245671523ec0619..6c4c1fb8181192068b63b658e60ac6b0 //# sourceMappingURL=PgPool.d.ts.map \ No newline at end of file diff --git a/dist/PgPool.js b/dist/PgPool.js -index 4b9c73361f97db293299bbfa379712fe63d6aafb..f7e8483422dbec5c0f4a1954400fecfd51040da0 100644 +index 4b9c73361f97db293299bbfa379712fe63d6aafb..5b5f5226537a26a976a0f6b795d270af89a64e9e 100644 --- a/dist/PgPool.js +++ b/dist/PgPool.js @@ -8,6 +8,8 @@ import * as Clock from "effect/Clock"; @@ -1417,7 +1448,7 @@ index 0892ccb889bf3a03b13490e4c5680a1b6989e9b6..501ca57b1e4f2c1bf9f5079ae37e9567 /** * Creates a scoped PostgreSQL client backed by one connection. diff --git a/src/PgConnection.ts b/src/PgConnection.ts -index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b10c0dd52 100644 +index c0740bc46059cf1e21c2783a04cc48000fc7906e..cee1cbae48ba029d04dfc2f35c216b1de04083e7 100644 --- a/src/PgConnection.ts +++ b/src/PgConnection.ts @@ -14,9 +14,11 @@ import * as Cause from "effect/Cause" @@ -1859,12 +1890,14 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b interface Session { readonly socket: Duplex -@@ -363,8 +706,20 @@ interface PipelineEntry { +@@ -363,8 +706,22 @@ interface PipelineEntry { readonly deferred: Deferred.Deferred readonly machine: QueryMachine abandoned: boolean + /** Which socket write sent it (a {@link pipelined} batch's span counts them); unset until sent. */ + write?: number | undefined ++ /** A {@link pipelined} batch's `COMMIT`: nothing is written behind it until it answers. */ ++ closesTransaction?: boolean | undefined } +/** @@ -1880,7 +1913,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b const abortDrainTimeoutMillis = 5000 const abortDrainGraceMillis = 10 /** PostgreSQL `query_canceled` SQLSTATE. */ -@@ -399,6 +754,7 @@ class PgConnectionImpl implements PgConnection { +@@ -399,6 +756,7 @@ class PgConnectionImpl implements PgConnection { cancelPending = false /** Reported by the latest `ReadyForQuery`. */ transactionStatus: PgProtocol.TransactionStatus = "I" @@ -1888,7 +1921,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b readonly channels = new Map>>() readonly retireHooks = new Set<() => void>() /** Queued but not yet written; drained into `pipelineInFlight` on flush. */ -@@ -407,7 +763,20 @@ class PgConnectionImpl implements PgConnection { +@@ -407,7 +765,20 @@ class PgConnectionImpl implements PgConnection { readonly pipelineInFlight: Array = [] pipelineHead = 0 pipelineFlushScheduled = false @@ -1909,7 +1942,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b readonly [internalsKey]: ConnectionInternals constructor(config: Config, resolved: ResolvedConfig, session: Session, registry: PgTypes.Registry | undefined) { -@@ -422,7 +791,25 @@ class PgConnectionImpl implements PgConnection { +@@ -422,7 +793,25 @@ class PgConnectionImpl implements PgConnection { this[internalsKey] = { base: this, deadError: () => this.deadWith, @@ -1936,7 +1969,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b } session.parser.onFieldError = this.onFieldError session.socket.on("data", this.onData) -@@ -431,6 +818,7 @@ class PgConnectionImpl implements PgConnection { +@@ -431,6 +820,7 @@ class PgConnectionImpl implements PgConnection { } private readonly onData = (chunk: Uint8Array): void => { @@ -1944,7 +1977,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b try { this.session.parser.pushEach(chunk, this.dispatch) } catch (cause) { -@@ -527,6 +915,7 @@ class PgConnectionImpl implements PgConnection { +@@ -527,6 +917,7 @@ class PgConnectionImpl implements PgConnection { } private retire(): void { @@ -1952,7 +1985,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b if (this.closed) return for (const hook of this.retireHooks) hook() } -@@ -618,7 +1007,22 @@ class PgConnectionImpl implements PgConnection { +@@ -618,7 +1009,22 @@ class PgConnectionImpl implements PgConnection { private readonly pipelineIsIdle = (): boolean => this.pipelineDepth() === 0 && this.pipelinePending.length === 0 @@ -1975,7 +2008,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b if (!this.pipelineIsIdle()) return this.consumer = undefined this.session.parser.readField = undefined -@@ -654,6 +1058,7 @@ class PgConnectionImpl implements PgConnection { +@@ -654,12 +1060,17 @@ class PgConnectionImpl implements PgConnection { this.pipelineInFlight.length = 0 this.pipelineHead = 0 } @@ -1983,7 +2016,26 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b } /** Writes everything queued since the last flush as one batch. */ -@@ -686,7 +1091,10 @@ class PgConnectionImpl implements PgConnection { + private readonly flushPipeline = (): void => { + this.pipelineFlushScheduled = false + if (this.deadWith !== undefined) return ++ // Nothing is written behind a batch's `COMMIT` until it has answered: through Hyperdrive, a ++ // statement written while a `BEGIN` … `COMMIT` on the connection is still unanswered is never ++ // answered. What waits stays queued; the flush after the answer's chunk (`onBatchEnd`) sends it. ++ if (this.transactionOnWire()) return + let capacity = maxPipelineDepth - this.pipelineDepth() + if (capacity <= 0) return + const wasEmpty = this.pipelineDepth() === 0 +@@ -676,6 +1087,8 @@ class PgConnectionImpl implements PgConnection { + batch.push(entry) + this.pipelineInFlight.push(entry) + capacity-- ++ // A batch's `COMMIT` ends the write; what was queued after it waits for its answer. ++ if (entry.closesTransaction === true) break + } + this.pipelinePending.splice(0, index) + if (batch.length === 0) { +@@ -686,13 +1099,24 @@ class PgConnectionImpl implements PgConnection { const frame = batch.length === 1 ? batch[0].plan.frame : concat(batch.map((entry) => entry.plan.frame)) @@ -1994,7 +2046,21 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b this.session.socket.write(frame) } catch (cause) { this.fatal(connectionQueryError(cause, "PgConnection: Failed to write query batch")) -@@ -700,6 +1108,128 @@ class PgConnectionImpl implements PgConnection { + } + } + ++ /** Whether a batch's `COMMIT` is written and not yet answered. */ ++ private readonly transactionOnWire = (): boolean => { ++ for (let index = this.pipelineHead; index < this.pipelineInFlight.length; index++) { ++ if (this.pipelineInFlight[index].closesTransaction === true) return true ++ } ++ return false ++ } ++ + private readonly schedulePipelineFlush = (): void => { + if (this.pipelineFlushScheduled || this.pipelineDepth() >= maxPipelineDepth) return + this.pipelineFlushScheduled = true +@@ -700,6 +1124,128 @@ class PgConnectionImpl implements PgConnection { queueMicrotask(this.flushPipeline) } @@ -2123,7 +2189,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b /** * One cycle on a multiplexed connection. The `owner` permit is only taken to * queue the statement, so a pin still orders against submissions without -@@ -710,10 +1240,24 @@ class PgConnectionImpl implements PgConnection { +@@ -710,10 +1256,24 @@ class PgConnectionImpl implements PgConnection { params: ReadonlyArray, wantRows: boolean, cache: PreparedCache | undefined @@ -2148,7 +2214,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b let plan: Plan try { plan = this.encodeQuery(sql, params, cache) -@@ -725,20 +1269,17 @@ class PgConnectionImpl implements PgConnection { +@@ -725,20 +1285,17 @@ class PgConnectionImpl implements PgConnection { const machine = new QueryMachine(this, plan, wantRows, (result) => this.finishPipelineEntry(entry, result)) entry = { plan, deferred, machine, abandoned: false } this.pipelinePending.push(entry) @@ -2172,7 +2238,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b } ) -@@ -751,7 +1292,11 @@ class PgConnectionImpl implements PgConnection { +@@ -751,7 +1308,11 @@ class PgConnectionImpl implements PgConnection { params: ReadonlyArray, wantRows: boolean, cache: PreparedCache | undefined @@ -2185,7 +2251,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b /** * Runs one cycle. A reused statement the backend has since dropped, or whose -@@ -846,6 +1391,107 @@ class PgConnectionImpl implements PgConnection { +@@ -846,6 +1407,109 @@ class PgConnectionImpl implements PgConnection { : this.owner.withPermit(this.cycleOwned(sql, params, wantRows, cache)) } @@ -2284,7 +2350,9 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b + return entries + } + for (const item of inside) enqueue(item.sql, item.params, item.wantRows, item.deferred, wrap) -+ if (wrap) enqueue("COMMIT", emptyParams, false, Deferred.makeUnsafe(), false) ++ if (wrap && enqueue("COMMIT", emptyParams, false, Deferred.makeUnsafe(), false)) { ++ entries[entries.length - 1].closesTransaction = true ++ } + this.consumer = this.pipelineConsumer + this.schedulePipelineFlush() + return entries @@ -2293,7 +2361,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b readonly query = (sql: string, params?: ReadonlyArray, prepare = true): Effect.Effect => Effect.map(this.run(sql, params ?? emptyParams, true, prepare), takeResult) -@@ -1648,11 +2294,26 @@ const drainAborted = ( +@@ -1648,11 +2312,26 @@ const drainAborted = ( ) } @@ -2321,7 +2389,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b ): Effect.Effect => Effect.callback((resume) => { if (conn.deadWith !== undefined) { -@@ -1663,12 +2324,32 @@ const runQuery = ( +@@ -1663,12 +2342,32 @@ const runQuery = ( conn.consumer = undefined resume(effect) }) @@ -2356,7 +2424,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b } catch (cause) { conn.fatal(connectionQueryError(cause, "PgConnection: Failed to write query")) } -@@ -2109,10 +2790,34 @@ const sendCancelRequest = (config: ResolvedConfig, session: Session): Effect.Eff +@@ -2109,10 +2808,34 @@ const sendCancelRequest = (config: ResolvedConfig, session: Session): Effect.Eff return Effect.sync(finish) }) @@ -2391,7 +2459,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b let encrypted = false let parser: PgProtocol.Parser | undefined let sslErrorParser: PgProtocol.Parser | undefined -@@ -2120,16 +2825,21 @@ const connect = (config: ResolvedConfig, resolvedPassword: string | undefined): +@@ -2120,16 +2843,21 @@ const connect = (config: ResolvedConfig, resolvedPassword: string | undefined): let processId = 0 let secretKey = 0 @@ -2417,7 +2485,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b const onError = (cause: Error) => failConnect(cause, "PgConnection: Failed to connect") const onClose = () => -@@ -2222,7 +2932,7 @@ const connect = (config: ResolvedConfig, resolvedPassword: string | undefined): +@@ -2222,7 +2950,7 @@ const connect = (config: ResolvedConfig, resolvedPassword: string | undefined): secretKey = message.secret return case "ErrorResponse": @@ -2426,7 +2494,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b case "ReadyForQuery": if (scram !== undefined) { return failAuth( -@@ -2283,7 +2993,7 @@ const connect = (config: ResolvedConfig, resolvedPassword: string | undefined): +@@ -2283,7 +3011,7 @@ const connect = (config: ResolvedConfig, resolvedPassword: string | undefined): "PgConnection: Invalid SSLRequest response" ) } @@ -2435,7 +2503,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b } socket.off("data", onSslResponse) if (chunk.length !== 1) { -@@ -2337,10 +3047,9 @@ const connect = (config: ResolvedConfig, resolvedPassword: string | undefined): +@@ -2337,10 +3065,9 @@ const connect = (config: ResolvedConfig, resolvedPassword: string | undefined): ...(config.keepAlive !== undefined && { keepAlive: true, keepAliveInitialDelay: config.keepAlive }) }) } catch (cause) { @@ -2449,7 +2517,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b )) return } -@@ -2349,7 +3058,10 @@ const connect = (config: ResolvedConfig, resolvedPassword: string | undefined): +@@ -2349,7 +3076,10 @@ const connect = (config: ResolvedConfig, resolvedPassword: string | undefined): if (config.stream !== undefined) { begin() } else { @@ -2462,7 +2530,7 @@ index c0740bc46059cf1e21c2783a04cc48000fc7906e..ebf5cb4fcd939bf48f469d4db4f0046b return Effect.sync(() => { diff --git a/src/PgPool.ts b/src/PgPool.ts -index 1fb1de888e1bfbc17e051949716952d284c110e3..07794884e7166a9a6a0fe9b888dbcf54ede1f1ed 100644 +index 1fb1de888e1bfbc17e051949716952d284c110e3..96eef71a283f99a1ab50ecc8f570fcaa522cbd0e 100644 --- a/src/PgPool.ts +++ b/src/PgPool.ts @@ -8,6 +8,8 @@ import * as Clock from "effect/Clock" diff --git a/patches/effect-sql-pg-tracing.md b/patches/effect-sql-pg-tracing.md index 1c31310606..c7804efc29 100644 --- a/patches/effect-sql-pg-tracing.md +++ b/patches/effect-sql-pg-tracing.md @@ -56,6 +56,12 @@ caller per connection, `PgPool.held`, settling abandoned and unawaited statement `notes/database-pool.md` describes them. Like everything else here they are in both `src/` and `dist/`, so a port from either copy keeps them; only `dist/` is loaded. +Nothing is written on a connection while a `pipelined` batch's `COMMIT` on it is unanswered: a +leaf's statement issued meanwhile waits in the connection's queue and goes out with the flush that +follows the answer, and a batch's `COMMIT` ends the socket write it is in. Through Hyperdrive a +statement written behind an unanswered `COMMIT` was never answered (`notes/database-pool.md`, +"Batches"). + ## Connection retries The patch also adds a `connectRetries` option (default `0`) to `PgConnection` diff --git a/scripts/check-conflict-markers.ts b/scripts/check-conflict-markers.ts new file mode 100644 index 0000000000..69c9c75a0a --- /dev/null +++ b/scripts/check-conflict-markers.ts @@ -0,0 +1,45 @@ +/** + * Fails when a tracked text file holds a line Git writes for a merge conflict: `<<<<<<<`, `|||||||` + * or `>>>>>>>` alone or followed by a space, or `=======` alone. Run from the repository's root; it + * names each as `file:line`. Binary files are skipped (`git grep -I`). + * + * No tracked file holds such a line on purpose today. A file that must (a fixture, notes on + * merging) goes in `allowed` with its reason. The cases (e2e/tests/conflict-markers.spec.ts) build + * their markers at run time, so they need no entry. + */ +import { spawnSync } from "node:child_process"; + +/** Tracked files allowed to hold conflict markers, each with why. None today. */ +const allowed: ReadonlyMap = new Map([]); + +const marker = /^(?:(?:<{7}|\|{7}|>{7})(?: .*)?|={7})\r?$/u; + +// `-z` separates file, line number and text with NUL, so no file name is misread. +const grep = spawnSync("git", ["grep", "-I", "-n", "-z", "-E", "^(<{7}|[|]{7}|>{7}|={7})"], { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, +}); +// git grep exits 1 when nothing matches, and above 1 when it fails. +if (grep.error || grep.status === null || grep.status > 1) { + console.error(`git grep failed: ${grep.error?.message ?? grep.stderr}`); + process.exit(2); +} + +const found = grep.stdout + .split("\n") + .filter((line) => line !== "") + .map((line) => { + const [file = "", number = "", ...text] = line.split("\0"); + return { file, number, text: text.join("\0") }; + }) + .filter(({ file, text }) => marker.test(text) && !allowed.has(file)); + +if (found.length) { + console.error("Leftover merge conflict markers. Resolve each conflict and remove its markers:"); + for (const { file, number, text } of found) console.error(` ${file}:${number}: ${text}`); + console.error( + "A file that must hold such a line goes in `allowed` in scripts/check-conflict-markers.ts.", + ); + process.exit(1); +} +console.log("No conflict markers in tracked files."); diff --git a/scripts/ci/worker-baseline.ts b/scripts/ci/worker-baseline.ts new file mode 100644 index 0000000000..3387c2d314 --- /dev/null +++ b/scripts/ci/worker-baseline.ts @@ -0,0 +1,956 @@ +/** + * What a pull request's Worker sizes are compared with: the base of the bottom of its stack, at the + * commit the pull request is built on. + * + * node scripts/ci/worker-baseline.ts resolve [--main] writes branch, commit, id, label, note + * node scripts/ci/worker-baseline.ts fetch writes found and build + * node scripts/ci/worker-baseline.ts fallback main's baseline and its sizes + * node scripts/ci/worker-baseline.ts above checks the layer above a `skip` + * node scripts/ci/worker-baseline.ts merged the pull requests a push merges + * node scripts/ci/worker-baseline.ts fetch-build the previous push's built Workers + * + * The rule: nothing reaches main without having been compared with main. A pull request whose stack + * ends on main (its base is main, or its base is the head of an open pull request whose stack ends on + * main) is compared with main's sizes at the commit it applies to, as before. One whose stack ends on + * another branch, such as a long-lived feature branch, is compared with that branch at its merge base + * with the pull request, so it is charged for its own growth; the branch is charged for its total + * once, against main, by its own pull request into main. The stack is read from GitHub the way the + * E2E selection reads it: follow the base branch while it is the head of an open pull request in this + * repository. A branch that cannot be read (deleted, or no merge base) falls back to main. + * + * `resolve` reads GITHUB_REPOSITORY, DEFAULT_BRANCH, BASE_REF and BASE_SHA (the pull request's base, + * or for a push the commit before it) and HEAD_SHA, and writes to GITHUB_OUTPUT: `branch`, `commit`, + * `id` (`@`, which the size gate's record holds, so a run compared with one baseline + * is never reported for another) and `note` when it fell back to main. `--main` resolves main's + * baseline whatever the stack ends on: the fallback when the branch's sizes cannot be had. + * + * `fetch` reads BASELINE_BRANCH and BASELINE_COMMIT and downloads the baseline's sizes into the + * directory: for main, the `worker-uploads` artifact of main's CI run at that commit; for another + * branch, the `worker-uploads-at-` artifact a pull request run recorded when it built that + * commit's Workers. It writes `found` and, for another branch with nothing recorded, `build=true`: + * the job then builds that commit's Workers itself, once per commit. Only sizes from a source this + * repository's own code wrote are used: main's from main's push runs, a branch's from CI runs of + * this repository's own branches, and only when the sizes name the commit they were built from. + * + * `above` checks a lower stack layer whose e2e block says `skip`: such a layer runs nothing, and + * passes on the strength of the layer above, so that layer's run must have been computed with this + * layer in its chain. It passes when an open pull request in this repository whose base is this + * branch has a CI run on its current head whose record (the select job's `ci-tested-…` artifact) + * says it was built on this layer's head, and compared its Worker sizes with the branch this + * layer's stack ends on. The E2E selection always compares with main, whatever the chain, so the + * base commit covers it. Otherwise it waits a few minutes, for a layer above whose run has not + * recorded yet, and then fails, naming the pull request above and the commands that run it again. + * + * GitHub's API sometimes fails for a moment: each read is tried again a few times, and a stack that + * still cannot be read is compared with main, with a notice. + */ +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { + Cause, + Clock, + Config, + Console, + Duration, + Effect, + FileSystem, + Option, + Schedule, + Schema, + Stream, +} from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import { + type Lower, + type Observation, + type RecordFields, + type RunSeen, + type UpperSeen, + type Verdict, + decisionLine, + enforce, + fails, + guarded, + summarySentence, + judge, + next, + parseRecord, +} from "../../e2e/ci-skip-check.ts"; + +class BaselineFailed extends Schema.TaggedError()("BaselineFailed", { + message: Schema.String, +}) {} + +/** Runs `gh` once, for at most two minutes, and returns its exit code and what it printed. */ +const ghRaw = (args: ReadonlyArray, limit: Duration.Duration = Duration.minutes(2)) => + Effect.gen(function* () { + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* processes.spawn(ChildProcess.make("gh", args)); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + child.stdout.pipe(Stream.decodeText(), Stream.mkString), + child.stderr.pipe(Stream.decodeText(), Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + return { stdout, stderr, exitCode: Number(exitCode) }; + }).pipe( + Effect.scoped, + // A stalled connection is no answer; it is not one of GitHub's brief failures, so not retried. + Effect.timeout(limit), + Effect.mapError((error) => + error._tag === "TimeoutError" + ? new BaselineFailed({ + message: `gh ${args.join(" ")}: no answer within ${Duration.toSeconds(limit)} s`, + }) + : new BaselineFailed({ message: `gh ${args.join(" ")}: ${String(error)}` }), + ), + ); + +/** Runs `gh` once and returns what it prints, or fails with what it said. */ +const ghOnce = (args: ReadonlyArray, limit?: Duration.Duration) => + Effect.gen(function* () { + const { stdout, stderr, exitCode } = yield* ghRaw(args, limit); + if (exitCode !== 0) + return yield* new BaselineFailed({ message: `gh ${args.join(" ")}: ${stderr.trim()}` }); + return stdout.trim(); + }); + +/** What `gh` says when GitHub failed for a moment, rather than answered. */ +const transient = + /HTTP (5\d\d|429)|secondary rate limit|timed? ?out|timeout|connection (reset|refused)|ECONNRESET|unexpected EOF|TLS handshake|temporar/i; + +/** How many times a read that failed for a moment is tried again: after one, two and four seconds. */ +const retries = 3; + +/** Tries `read` again after a failure that was GitHub's for a moment. */ +const retried = (read: Effect.Effect) => + Effect.gen(function* () { + // The first wait, in seconds; the spec shortens it. + const first = yield* Config.Number("WORKER_BASELINE_RETRY_SECONDS").pipe( + Config.withDefault(1), + Effect.orDie, + ); + return yield* read.pipe( + Effect.retry({ + while: (failure) => transient.test(failure.message), + schedule: Schedule.exponential(Duration.millis(first * 1000)), + times: retries, + }), + ); + }); + +/** Runs `gh`, again after a failure that was GitHub's for a moment, and returns what it prints. */ +const gh = (args: ReadonlyArray, limit?: Duration.Duration) => retried(ghOnce(args, limit)); + +/** + * The REST reads the skip check repeats while it waits, made conditional on what each last + * returned: GitHub answers 304 for one that has not changed, which does not count against the + * token's rate limit. `requests` and `unchanged` count them for the step's log. + */ +const conditional = { + etags: new Map(), + requests: 0, + unchanged: 0, +}; +const ghConditional = (path: string, limit: Duration.Duration) => + retried( + Effect.gen(function* () { + const cached = conditional.etags.get(path); + const { stdout, stderr, exitCode } = yield* ghRaw( + [ + "api", + "-i", + ...(cached === undefined ? [] : ["-H", `If-None-Match: ${cached.etag}`]), + path, + ], + limit, + ); + const split = stdout.search(/\r?\n\r?\n/); + const head = split === -1 ? stdout : stdout.slice(0, split); + const body = split === -1 ? "" : stdout.slice(split).trim(); + // gh exits 1 on a 304, with the status line on stdout. + if (cached !== undefined && /^HTTP\/[\d.]+ 304/.test(head)) { + conditional.unchanged += 1; + return cached.body; + } + conditional.requests += 1; + if (exitCode !== 0) + return yield* new BaselineFailed({ message: `gh api ${path}: ${stderr.trim()}` }); + const etag = head.match(/^etag: *(.+?)\s*$/im)?.[1]; + if (etag !== undefined) conditional.etags.set(path, { etag, body }); + return body; + }), + ); + +const OpenPullRequests = Schema.fromJsonString( + Schema.Array(Schema.Struct({ baseRefName: Schema.String, isCrossRepository: Schema.Boolean })), +); + +/** + * The branches a stack can end on, starting from `base`: follow each open pull request in this + * repository whose head is the branch to its base, until a branch that is the head of none. + */ +const bottoms = (repository: string, defaultBranch: string, base: string) => + Effect.gen(function* () { + const ends = new Set(); + const seen = new Set(); + let frontier = [base]; + while (frontier.length > 0) { + const next: Array = []; + for (const branch of frontier) { + if (seen.has(branch)) continue; + seen.add(branch); + if (branch === defaultBranch) { + ends.add(branch); + continue; + } + const open = (yield* Schema.decodeUnknownEffect(OpenPullRequests)( + yield* gh([ + "pr", + "list", + "--repo", + repository, + "--head", + branch, + "--state", + "open", + "--json", + "baseRefName,isCrossRepository", + ]), + )).filter((pull) => !pull.isCrossRepository); + if (open.length === 0) ends.add(branch); + else next.push(...open.map((pull) => pull.baseRefName)); + } + frontier = next; + } + return ends; + }); + +/** The commit `head` and `branch` last shared, as GitHub's comparison names it. */ +const mergeBase = (repository: string, branch: string, head: string) => + gh([ + "api", + `repos/${repository}/compare/${encodeURIComponent(branch)}...${head}`, + "--jq", + ".merge_base_commit.sha", + ]); + +const write = (lines: Record) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const output = yield* Config.String("GITHUB_OUTPUT").pipe(Config.option); + const text = Object.entries(lines) + .map(([key, value]) => `${key}=${value}\n`) + .join(""); + yield* Console.log(text.trim()); + if (Option.isSome(output)) yield* fs.writeFileString(output.value, text, { flag: "a" }); + }); + +const resolve = (forceMain: boolean) => + Effect.gen(function* () { + const repository = yield* Config.String("GITHUB_REPOSITORY"); + const defaultBranch = yield* Config.String("DEFAULT_BRANCH"); + const baseRef = yield* Config.String("BASE_REF").pipe(Config.withDefault("")); + const baseSha = yield* Config.String("BASE_SHA"); + const head = yield* Config.String("HEAD_SHA").pipe(Config.withDefault("")); + /** Main's baseline: the base commit when the base is main or this is a push, else the merge base. */ + const main = (note: string) => + baseRef === "" || baseRef === defaultBranch + ? Effect.succeed({ branch: defaultBranch, commit: baseSha, note }) + : mergeBase(repository, defaultBranch, head).pipe( + Effect.map((commit) => ({ branch: defaultBranch, commit, note })), + // No commit: no sizes are fetched, so no change is shown; the budgets still hold. + Effect.catchTag("BaselineFailed", (failure) => + Effect.succeed({ + branch: defaultBranch, + commit: "", + note: `${note === "" ? "" : `${note}; `}the commit this applies to on ${defaultBranch} could not be read (${failure.message.split("\n")[0]}), so no change is shown`, + }), + ), + ); + const chosen = yield* Effect.gen(function* () { + if (forceMain || baseRef === "" || baseRef === defaultBranch) return yield* main(""); + const read = yield* bottoms(repository, defaultBranch, baseRef).pipe(Effect.option); + if (Option.isNone(read)) + return yield* main( + `the stack below ${baseRef} could not be read, so this compares with ${defaultBranch}`, + ); + const ends = read.value; + // A stack that can end on main is compared with main. + if (ends.has(defaultBranch) || ends.size !== 1) return yield* main(""); + const [branch] = [...ends]; + return yield* mergeBase(repository, branch!, head).pipe( + Effect.map((commit) => ({ branch: branch!, commit, note: "" })), + Effect.catchTag("BaselineFailed", (failure) => + main( + `${branch} could not be read (${failure.message.split("\n")[0]}), so this compares with ${defaultBranch}`, + ), + ), + ); + }); + if (chosen.commit !== "" && !/^[0-9a-f]{40}$/.test(chosen.commit)) + return yield* new BaselineFailed({ + message: `No baseline commit for ${chosen.branch}: ${JSON.stringify(chosen.commit)}`, + }); + if (chosen.note !== "") yield* Console.log(`::notice title=Worker sizes::${chosen.note}.`); + return chosen; + }); + +/** The outputs that name a baseline. */ +const named = (chosen: { branch: string; commit: string; note: string }) => ({ + branch: chosen.branch, + commit: chosen.commit, + id: `${chosen.branch}@${chosen.commit === "" ? "unread" : chosen.commit}`, + label: chosen.commit === "" ? chosen.branch : `${chosen.branch} at ${chosen.commit.slice(0, 9)}`, + note: chosen.note, +}); + +/** How many release commits `fetch` steps back over to find main's sizes: four betas a day. */ +const releaseSteps = 5; + +const CommitInfo = Schema.fromJsonString( + Schema.Struct({ + message: Schema.String, + author: Schema.NullOr(Schema.String), + parents: Schema.Array(Schema.String), + files: Schema.Array( + Schema.Struct({ name: Schema.String, patch: Schema.NullOr(Schema.String) }), + ), + }), +); + +/** A commit's message, author, parents and changed files with their diffs, or none when unread. */ +const commitInfo = (repository: string, sha: string) => + gh([ + "api", + `repos/${repository}/commits/${sha}`, + "--jq", + "{message: .commit.message, author: .author.login, parents: [.parents[].sha], files: [.files[] | {name: .filename, patch}]}", + ]).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(CommitInfo)), + Effect.option, + Effect.map(Option.getOrUndefined), + ); + +/** The lines a diff removes and adds, each with the lines above it in the file. */ +const changedLines = (patch: string) => { + const changed: Array<{ line: string; above: ReadonlyArray }> = []; + const kept: Array = []; + for (const line of patch.split("\n")) { + if (line.startsWith("@@")) kept.length = 0; + else if (line.startsWith("-") || line.startsWith("+")) changed.push({ line, above: [...kept] }); + else kept.push(line.slice(1)); + } + return changed; +}; + +/** A diff that changes one `"version"` line from one value to another and nothing else. */ +const versionLine = /^[-+]\s*"version": "[^"]+",?$/; +const onlyVersion = (patch: string | null, inside?: string) => { + if (patch === null) return false; + const changed = changedLines(patch); + return ( + changed.length === 2 && + changed[0]!.line.startsWith("-") && + changed[1]!.line.startsWith("+") && + changed.every(({ line }) => versionLine.test(line)) && + // In the lockfile, the version line of the CLI's own workspace entry, under its name. + (inside === undefined || changed[0]!.above.slice(-2).join("\n").trim().startsWith(inside)) + ); +}; + +/** + * A commit the beta workflow makes to release the CLI: by github-actions[bot], titled `Release … + * [skip ci]` (so it gets no CI run), and changing only the CLI's changelog, its version, and the + * lockfile's record of that version, none of which any Cloud Worker reads. A lockfile diff that + * changes anything else, such as a dependency, could change what a Worker bundles, so a commit + * with one is not a release commit here. + */ +const releaseFiles: Record boolean> = { + "apps/cli/CHANGELOG.md": () => true, + "apps/cli/package.json": (patch) => onlyVersion(patch), + "bun.lock": (patch) => onlyVersion(patch, '"apps/cli": {'), +}; +const releaseCommit = (info: typeof CommitInfo.Type) => + info.author === "github-actions[bot]" && + /^Release \S+ \[skip ci\]/.test(info.message) && + info.files.length > 0 && + info.files.every( + (file) => Object.hasOwn(releaseFiles, file.name) && releaseFiles[file.name]!(file.patch), + ); + +const Artifacts = Schema.fromJsonString( + Schema.Array(Schema.Struct({ expired: Schema.Boolean, run: Schema.NullOr(Schema.Number) })), +); + +/** The workflow, event and head repository of a run, to tell whose code wrote its artifacts. */ +const RunSource = Schema.fromJsonString( + Schema.Struct({ + path: Schema.String, + event: Schema.String, + repository: Schema.NullOr(Schema.String), + }), +); + +/** The file a recorded baseline holds next to its sizes: the commit they were built from. */ +const builtFrom = "baseline-commit.txt"; + +/** + * An artifact of main's own CI run at `commit` (a push run, which runs only main's code), in + * `directory`. A release commit gets no run (`[skip ci]`), so it is that of the nearest earlier + * main commit that has one: a release commit changes no Worker (`releaseCommit`). Only release + * commits are stepped over, at most `releaseSteps`. With `builtFrom`, the artifact must name the + * commit it was built from in that file, and it is used only when that is the commit found. + */ +const fromMain = ( + directory: string, + commit: string, + name: string, + title: string, + what: string, + builtFrom?: string, +) => + Effect.gen(function* () { + const repository = yield* Config.String("GITHUB_REPOSITORY"); + const defaultBranch = yield* Config.String("DEFAULT_BRANCH"); + const fs = yield* FileSystem.FileSystem; + const runAt = (sha: string) => + gh([ + "run", + "list", + "--repo", + repository, + "--workflow", + "ci.yml", + "--branch", + defaultBranch, + // Main's own runs, which run only main's code: a pull request's run can name main too. + "--event", + "push", + "--commit", + sha, + "--json", + "databaseId", + "--jq", + ".[0].databaseId // empty", + ]).pipe(Effect.catchTag("BaselineFailed", () => Effect.succeed(""))); + const download = (run: string) => + gh(["run", "download", run, "--repo", repository, "--name", name, "--dir", directory]).pipe( + Effect.as(true), + Effect.catchTag("BaselineFailed", () => Effect.succeed(false)), + ); + /** Whether what was downloaded names `sha` as the commit it was built from. */ + const fromCommit = (sha: string) => + Effect.gen(function* () { + if (builtFrom === undefined) return true; + const path = `${directory}/${builtFrom}`; + const named = (yield* fs.exists(path)) ? (yield* fs.readFileString(path)).trim() : ""; + if (named === sha) return true; + yield* fs.remove(directory, { recursive: true, force: true }); + yield* Console.log( + `::notice title=${title}::The ${name} artifact of ${sha}'s run names ${named === "" ? "no commit" : named} as the one it was built from, so it is not used.`, + ); + return false; + }).pipe(Effect.catch(() => Effect.succeed(false))); + const stepped: Array = []; + let at = commit; + for (;;) { + const run = yield* runAt(at); + if (run !== "" && (yield* download(run))) { + if (!(yield* fromCommit(at))) return { found: false, commit, stepped }; + if (stepped.length > 0) + yield* Console.log( + `::notice title=${title}::${defaultBranch}'s ${what} for ${commit} are those of ${at}, the nearest earlier commit with ${what}: ${stepped.length === 1 ? `${stepped[0]} is a release commit, which changes` : `${stepped.join(", ")} are release commits, which change`} no Worker.`, + ); + return { found: true, commit: at, stepped }; + } + const info = stepped.length < releaseSteps ? yield* commitInfo(repository, at) : undefined; + if (info === undefined || !releaseCommit(info) || info.parents.length !== 1) break; + stepped.push(at); + at = info.parents[0]!; + } + return { found: false, commit, stepped }; + }); + +const fetch = (directory: string, branch: string, commit: string) => + Effect.gen(function* () { + const repository = yield* Config.String("GITHUB_REPOSITORY"); + const defaultBranch = yield* Config.String("DEFAULT_BRANCH"); + const fs = yield* FileSystem.FileSystem; + const download = (run: string, name: string) => + gh(["run", "download", run, "--repo", repository, "--name", name, "--dir", directory]).pipe( + Effect.as(true), + Effect.catchTag("BaselineFailed", () => Effect.succeed(false)), + ); + if (branch === defaultBranch) { + const found = yield* fromMain(directory, commit, "worker-uploads", "Worker sizes", "sizes"); + if (!found.found) + yield* Console.log( + `::notice title=Worker sizes::${defaultBranch}'s sizes for ${commit} are not available yet${found.stepped.length > 0 ? `, nor for the release commits before it (${found.stepped.join(", ")})` : ""}, so no change is shown.`, + ); + return { found: found.found, build: false, commit: found.commit }; + } + // A branch's sizes, as a CI run of this repository's own branches recorded them: a fork's run + // runs the fork's code, which could write any sizes under any name. + const name = `worker-uploads-at-${commit}`; + const ownRun = (run: number) => + gh([ + "api", + `repos/${repository}/actions/runs/${run}`, + "--jq", + "{path, event, repository: .head_repository.full_name}", + ]).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(RunSource)), + Effect.map( + (source) => + source.path === ".github/workflows/ci.yml" && + source.event === "pull_request" && + source.repository === repository, + ), + Effect.catch(() => Effect.succeed(false)), + ); + // The sizes name the commit they were built from; any others are not this commit's. + const fromCommit = Effect.gen(function* () { + const path = `${directory}/${builtFrom}`; + const built = (yield* fs.exists(path)) ? (yield* fs.readFileString(path)).trim() : ""; + if (built === commit) return true; + yield* fs.remove(directory, { recursive: true, force: true }); + return false; + }).pipe(Effect.catch(() => Effect.succeed(false))); + const recorded = yield* gh([ + "api", + `repos/${repository}/actions/artifacts?name=${name}&per_page=20`, + "--jq", + "[.artifacts[] | {expired, run: .workflow_run.id}]", + ]).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Artifacts)), + Effect.catch(() => Effect.succeed([])), + ); + for (const artifact of recorded) + if ( + !artifact.expired && + artifact.run !== null && + (yield* ownRun(artifact.run)) && + (yield* download(String(artifact.run), name)) && + (yield* fromCommit) + ) + return { found: true, build: false, commit }; + return { found: false, build: true, commit }; + }); + +const Pulls = Schema.fromJsonString( + Schema.Array( + Schema.Struct({ + number: Schema.Number, + head: Schema.Struct({ + sha: Schema.String, + repo: Schema.NullOr(Schema.Struct({ full_name: Schema.String })), + }), + }), + ), +); + +const HeadRuns = Schema.fromJsonString( + Schema.Struct({ + workflow_runs: Schema.Array( + Schema.Struct({ + id: Schema.Number, + path: Schema.String, + status: Schema.String, + conclusion: Schema.NullOr(Schema.String), + html_url: Schema.String, + head_repository: Schema.NullOr(Schema.Struct({ full_name: Schema.String })), + }), + ), + }), +); + +const RunArtifacts = Schema.fromJsonString( + Schema.Struct({ + artifacts: Schema.Array(Schema.Struct({ id: Schema.Number, name: Schema.String })), + }), +); + +const RunJobs = Schema.fromJsonString( + Schema.Struct({ + jobs: Schema.Array( + Schema.Struct({ + name: Schema.String, + status: Schema.String, + steps: Schema.optional( + Schema.Array(Schema.Struct({ name: Schema.String, status: Schema.String })), + ), + }), + ), + }), +); + +/** The select job's step that runs this check, by its name in checks.yml. */ +const checkStep = "Check that the layer above was computed on this layer"; + +/** + * Whether `lower.head` is an ancestor of `tested`, the merge commit an upper run tested: GitHub's + * compare of the two, which says `ahead` or `identical` exactly when it is (the equivalent of `git + * merge-base --is-ancestor`, without the history a shallow checkout lacks). Asked once per tested + * commit and remembered; an answer that could not be read is undefined, so the check keeps waiting. + */ +const containsLower = ( + lower: Lower, + tested: string, + limit: Duration.Duration, + contained: Map, +) => + Effect.gen(function* () { + const known = contained.get(tested); + if (known !== undefined) return known; + const status = (yield* gh( + [ + "api", + `repos/${lower.repository}/compare/${lower.head}...${tested}?per_page=1`, + "--jq", + ".status", + ], + limit, + )).trim(); + if (!["ahead", "identical", "behind", "diverged"].includes(status)) + return yield* new BaselineFailed({ message: `the compare answered ${status}` }); + const answer = status === "ahead" || status === "identical"; + contained.set(tested, answer); + return answer; + }).pipe(Effect.catchCause(() => Effect.succeed(undefined))); + +/** + * One look at the pull requests above `lower`: this repository's open pull requests based on its + * branch, and their CI runs on their current heads. A record is downloaded once per artifact. + */ +const look = ( + lower: Lower, + scratch: string, + records: Map, + limits: { readonly read: Duration.Duration; readonly look: Duration.Duration }, + contained: Map, +) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const repository = lower.repository; + const read = (schema: S, path: string) => + Effect.flatMap(ghConditional(path, limits.read), (body) => + Schema.decodeUnknownEffect(schema)(body).pipe( + Effect.mapError( + (error) => new BaselineFailed({ message: `${path}: ${String(error).split("\n")[0]}` }), + ), + ), + ); + // A fork's pull request, and a fork's run on the same commit, run the fork's code, which could + // write any record: only this repository's count. + const pulls = (yield* read( + Pulls, + `repos/${repository}/pulls?base=${encodeURIComponent(lower.ref)}&state=open&per_page=100`, + )).filter((pull) => pull.head.repo?.full_name === repository); + const uppers = yield* Effect.forEach(pulls, (pull) => + Effect.gen(function* () { + const { workflow_runs: all } = yield* read( + HeadRuns, + `repos/${repository}/actions/runs?event=pull_request&head_sha=${pull.head.sha}&per_page=50`, + ); + // CI's runs, not another workflow's; a cancelled run may have recorded and then stopped. + const runs = all + .filter( + (run) => + run.path === ".github/workflows/ci.yml" && + run.head_repository?.full_name === repository && + run.conclusion !== "cancelled", + ) + .toSorted((a, b) => b.id - a.id); + const seen: Array = []; + // Every run on the head is read, since any one with a fitting record passes; the ten newest + // cover a stack's duplicates and re-runs. + for (const run of runs.slice(0, 10)) { + const { artifacts } = yield* read( + RunArtifacts, + `repos/${repository}/actions/runs/${run.id}/artifacts?per_page=100`, + ); + // Its own record: a re-run that chose differently uploads a second one, and the newest + // is what the run's latest attempt chose. + const newest = artifacts + .filter((artifact) => artifact.name.startsWith(`ci-tested-pr${pull.number}-`)) + .toSorted((a, b) => b.id - a.id)[0]; + if (newest !== undefined && !records.has(newest.id)) { + const directory = `${scratch}/${newest.id}`; + yield* fs.remove(directory, { recursive: true, force: true }); + yield* gh( + [ + "run", + "download", + String(run.id), + "--repo", + repository, + "--name", + newest.name, + "--dir", + directory, + ], + limits.read, + ); + records.set(newest.id, parseRecord(yield* fs.readFileString(`${directory}/record`))); + } + const record = newest === undefined ? undefined : records.get(newest.id); + if (record !== undefined) { + seen.push({ + id: run.id, + url: run.html_url, + record, + selectDone: true, + waitingAbove: false, + contains: + record.tested === undefined || record.base === lower.head + ? undefined + : yield* containsLower(lower, record.tested, limits.read, contained), + }); + continue; + } + if (run.status === "completed") { + seen.push({ id: run.id, url: run.html_url, selectDone: true, waitingAbove: false }); + continue; + } + const select = (yield* read( + RunJobs, + `repos/${repository}/actions/runs/${run.id}/jobs?per_page=100`, + )).jobs.find((job) => job.name === "checks / select"); + seen.push({ + id: run.id, + url: run.html_url, + selectDone: select?.status === "completed", + waitingAbove: + select?.steps?.some( + (step) => step.name === checkStep && step.status === "in_progress", + ) ?? false, + }); + } + return { pull: pull.number, head: pull.head.sha, runs: seen } satisfies UpperSeen; + }), + ); + return { _tag: "Read", uppers } as Observation; + }).pipe( + // However many reads a look makes, it ends in time for the next. + Effect.timeoutOrElse({ + duration: limits.look, + orElse: () => + Effect.fail( + new BaselineFailed({ + message: `the look got no answer within ${Duration.toSeconds(limits.look)} s`, + }), + ), + }), + // Not reading is not an answer: the check looks again until its deadline. + Effect.catchCause((cause) => + Effect.succeed({ + _tag: "Unread", + why: String(Cause.squash(cause)).split("\n")[0]!, + } as Observation), + ), + ); + +/** + * The check of the layer above a skip layer (e2e/ci-skip-check.ts): it looks, waits within the + * bounds, logs one decision line and a summary sentence, and fails only when enforcing. + */ +const decide = Effect.gen(function* () { + const lower: Lower = { + repository: yield* Config.String("GITHUB_REPOSITORY"), + ref: yield* Config.String("HEAD_REF"), + head: yield* Config.String("HEAD_SHA"), + bottom: yield* Config.String("BASELINE_BRANCH"), + runId: yield* Config.String("GITHUB_RUN_ID").pipe(Config.withDefault("")), + }; + const scratch = `${yield* Config.String("RUNNER_TEMP").pipe(Config.withDefault("/tmp"))}/ci-above`; + // Seconds of the clock per second of the rule. The spec runs the waits faster, and then counts + // only the waits, since its stand-in reads take longer than the waits it shortens. + const scale = yield* Config.Number("SKIP_ABOVE_TIME_SCALE").pipe(Config.withDefault(1)); + // Each read, download included, gets 20 seconds and each look a minute: a stalled connection + // is no answer, and cannot hold the step past its deadline. + const limits = { + read: Duration.millis(Math.max(1000, 20_000 * scale)), + look: Duration.millis(Math.max(2000, 60_000 * scale)), + }; + const records = new Map(); + const contained = new Map(); + const start = yield* Clock.currentTimeMillis; + let previous: Verdict | undefined; + let waited = 0; + for (let polls = 0; ; polls += 1) { + const verdict = judge(yield* look(lower, scratch, records, limits, contained), lower, previous); + if (verdict._tag === "Wait" && verdict.why !== (previous?._tag === "Wait" ? previous.why : "")) + yield* Console.log(`Waiting: ${verdict.why}.`); + previous = verdict; + const elapsed = scale === 1 ? ((yield* Clock.currentTimeMillis) - start) / 1000 : waited; + const step = next(verdict, lower, elapsed, polls); + if (step._tag === "Done") return step.decision; + waited += step.after; + yield* Effect.sleep(Duration.millis(step.after * 1000 * scale)); + } +}); + +/** + * The check of the layer above a skip layer (e2e/ci-skip-check.ts): it looks, waits within the + * bounds, logs one decision line and a summary sentence, and fails only when enforcing. While + * observing nothing in it can fail the step, a defect included: that is logged as `error`. + */ +const above = Effect.gen(function* () { + const outcome = yield* guarded(decide, enforce); + yield* Console.log(decisionLine(outcome)); + const report = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const summary = yield* Config.String("GITHUB_STEP_SUMMARY").pipe(Config.option); + if (Option.isSome(summary)) + yield* fs.writeFileString(summary.value, `${summarySentence(outcome)}\n`, { flag: "a" }); + if (outcome._tag === "Decided" && outcome.decision.oldFormat) + yield* Console.log(`::notice title=Skip check::${outcome.decision.message}`); + yield* write({ decision: outcome._tag === "Error" ? "error" : outcome.decision.decision }); + const limit = yield* gh( + [ + "api", + "rate_limit", + "--jq", + '.resources | "core \\(.core.used) used of \\(.core.limit), \\(.core.remaining) left; graphql \\(.graphql.used) used of \\(.graphql.limit)"', + ], + Duration.seconds(20), + ).pipe(Effect.catch(() => Effect.succeed("unread"))); + yield* Console.log( + `GitHub API: this check made ${conditional.requests} counted requests and ${conditional.unchanged} unchanged (free) ones; the token's rate limit: ${limit}.`, + ); + }); + yield* enforce + ? report + : report.pipe( + Effect.catchCause((cause) => + Console.log( + `The skip check's report could not be written (${String(Cause.squash(cause)).split("\n")[0]}); it passes while observing.`, + ), + ), + ); + if (fails(outcome, enforce)) + return yield* new BaselineFailed({ + message: outcome._tag === "Error" ? outcome.why : outcome.decision.message, + }); +}); + +const MergedPulls = Schema.fromJsonString(Schema.Array(Schema.Number)); + +/** + * The descriptions of the pull requests a push to main merges, for the size gate on main + * (worker-sizes.ts `--merged`): the merged pull requests of each commit between BEFORE_SHA and + * HEAD_SHA. Writes them to `file` as JSON and `pulls` (#1, #2) to the step's outputs. If GitHub + * cannot be read it writes no file and says so: not knowing the allowances is not growth, so the + * gate then only reports. A commit no pull request merged allows nothing, with a notice. + */ +const merged = (file: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const repository = yield* Config.String("GITHUB_REPOSITORY"); + const before = yield* Config.String("BEFORE_SHA"); + const head = yield* Config.String("HEAD_SHA"); + const read = Effect.gen(function* () { + const commits = yield* Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Array(Schema.String)), + )( + yield* gh([ + "api", + `repos/${repository}/compare/${before}...${head}`, + "--jq", + "[.commits[].sha]", + ]), + ); + const numbers = new Set(); + const alone: Array = []; + for (const commit of commits) { + const pulls = yield* Schema.decodeUnknownEffect(MergedPulls)( + yield* gh([ + "api", + `repos/${repository}/commits/${commit}/pulls`, + "--jq", + "[.[] | select(.merged_at != null) | .number]", + ]), + ); + if (pulls.length === 0) alone.push(commit.slice(0, 9)); + for (const pull of pulls) numbers.add(pull); + } + const bodies = yield* Effect.forEach( + [...numbers].toSorted((a, b) => a - b), + (number) => + gh(["api", `repos/${repository}/pulls/${number}`, "--jq", '.body // ""']).pipe( + Effect.map((body) => ({ number, body })), + ), + ); + return { bodies, alone }; + }); + const found = yield* read.pipe(Effect.option); + if (Option.isNone(found)) { + yield* Console.log( + `::notice title=Worker sizes::The pull requests this push merges could not be read, so its growth is reported and not checked against their allowances.`, + ); + return yield* write({ read: "false" }); + } + const { bodies, alone } = found.value; + if (alone.length > 0) + yield* Console.log( + `::notice title=Worker sizes::${alone.join(", ")} ${alone.length === 1 ? "is a commit" : "are commits"} no pull request merged, so ${alone.length === 1 ? "it allows" : "they allow"} no growth.`, + ); + yield* fs.writeFileString(file, JSON.stringify(bodies)); + return yield* write({ + read: "true", + file, + pulls: bodies.map((pull) => `#${pull.number}`).join(", "), + }); + }); + +NodeRuntime.runMain( + Effect.gen(function* () { + const [command, ...args] = process.argv.slice(2); + if (command === "resolve") return yield* write(named(yield* resolve(args.includes("--main")))); + if (command === "fetch" && args[0] !== undefined) { + const branch = yield* Config.String("BASELINE_BRANCH"); + const result = yield* fetch(args[0], branch, yield* Config.String("BASELINE_COMMIT")); + return yield* write({ + found: String(result.found), + build: String(result.build), + label: `${branch} at ${result.commit.slice(0, 9)}`, + }); + } + // The branch's sizes could not be had: compare with main's, never skip the comparison. + if (command === "fallback" && args[0] !== undefined) { + const chosen = yield* resolve(true); + const result = yield* fetch(args[0], chosen.branch, chosen.commit); + return yield* write({ + ...named(chosen), + found: String(result.found), + label: `${chosen.branch} at ${result.commit.slice(0, 9)}`, + }); + } + if (command === "above") return yield* above; + if (command === "merged" && args[0] !== undefined) return yield* merged(args[0]); + // The previous push's built Workers, for the start-up record (worker-startup-record.ts). + if (command === "fetch-build" && args[0] !== undefined) { + const commit = yield* Config.String("BASE_SHA"); + const found = yield* fromMain( + args[0], + commit, + "worker-build", + "Worker start-up", + "built Workers", + "built-from.txt", + ); + if (!found.found) + yield* Console.log( + `::notice title=Worker start-up::No build of ${commit}${found.stepped.length > 0 ? ` or the release commits before it` : ""} is recorded, so this push's start-up is recorded alone.`, + ); + return yield* write({ found: String(found.found), commit: found.found ? found.commit : "" }); + } + return yield* new BaselineFailed({ + message: + "Usage: node scripts/ci/worker-baseline.ts resolve [--main] | fetch | fallback | above | merged | fetch-build ", + }); + }).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/scripts/ci/worker-startup-record.ts b/scripts/ci/worker-startup-record.ts new file mode 100644 index 0000000000..555fd1e543 --- /dev/null +++ b/scripts/ci/worker-startup-record.ts @@ -0,0 +1,119 @@ +/** + * The start-up record of one push to main, from what apps/hosted/cloud/scripts/worker-startup.ts + * printed for it, and a table for the job summary. It records and gates nothing else. + * + * node scripts/ci/worker-startup-record.ts + * + * Reads HEAD_SHA (the pushed commit), BEFORE_SHA (the commit before the push) and BASE_COMMIT + * (the commit whose build was measured against it, empty when there was none: the head was then + * measured against itself, and only its own numbers are kept). Writes the record: + * + * { commit, before, base, workerd, runs, machine, skipped, + * workers: { : { staticBytes, chunks, compile, evaluate } } } + * + * where `staticBytes` and `chunks` are `{ base, head }`, and `compile` and `evaluate` are + * `{ base: { median, min }, head: { median, min }, difference }` in milliseconds, `difference` + * being the head's median less the base's. Without a base, every `base` and `difference` is null. + */ +import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; + +interface Summary { + readonly median: number; + readonly min: number; +} +interface Measured { + readonly staticModules: { readonly base: number; readonly head: number }; + readonly staticBytes: { readonly base: number; readonly head: number }; + readonly compile: { readonly base: Summary; readonly head: Summary }; + readonly evaluate: { readonly base: Summary; readonly head: Summary }; +} +interface Harness { + readonly workerd: string; + readonly runs: number; + readonly machine: unknown; + readonly workers: Readonly>; + readonly skipped: Readonly>; +} + +const [input, output] = process.argv.slice(2); +if (input === undefined || output === undefined) { + console.error("Usage: node scripts/ci/worker-startup-record.ts "); + process.exit(2); +} +const commit = process.env["HEAD_SHA"] ?? ""; +const before = process.env["BEFORE_SHA"] ?? ""; +const base = process.env["BASE_COMMIT"] ?? ""; +if (!/^[0-9a-f]{40}$/.test(commit) || (base !== "" && !/^[0-9a-f]{40}$/.test(base))) { + console.error("HEAD_SHA must be a commit, and BASE_COMMIT a commit or empty."); + process.exit(2); +} +const harness = JSON.parse(readFileSync(input, "utf8")) as Harness; +const alone = base === ""; +const pick = ({ median, min }: Summary) => ({ median, min }); +const timing = (times: { readonly base: Summary; readonly head: Summary }) => ({ + base: alone ? null : pick(times.base), + head: pick(times.head), + difference: alone ? null : times.head.median - times.base.median, +}); +const workers = Object.fromEntries( + Object.entries(harness.workers).map(([worker, measured]) => [ + worker, + { + staticBytes: { + base: alone ? null : measured.staticBytes.base, + head: measured.staticBytes.head, + }, + chunks: { + base: alone ? null : measured.staticModules.base, + head: measured.staticModules.head, + }, + compile: timing(measured.compile), + evaluate: timing(measured.evaluate), + }, + ]), +); +if (Object.keys(workers).length === 0) { + console.error("The harness measured no Worker."); + process.exit(1); +} +const record = { + commit, + before: before === "" ? null : before, + base: alone ? null : base, + workerd: harness.workerd, + runs: harness.runs, + machine: harness.machine, + skipped: harness.skipped, + workers, +}; +mkdirSync(dirname(output), { recursive: true }); +writeFileSync(output, `${JSON.stringify(record, null, 2)}\n`); + +const signed = (value: number | null) => + value === null ? "" : ` (${value > 0 ? "+" : ""}${value})`; +const cell = (times: ReturnType) => + times.base === null + ? `${times.head.median}` + : `${times.base.median} → ${times.head.median}${signed(times.difference)}`; +const table = [ + "## Worker start-up", + "", + alone + ? `${commit.slice(0, 9)} alone: no build of the commit before it was recorded. Median of ${harness.runs} runs in workerd ${harness.workerd}, milliseconds.` + : `${base.slice(0, 9)} → ${commit.slice(0, 9)}, median of ${harness.runs} runs each, alternating, in workerd ${harness.workerd}; milliseconds. It gates nothing.`, + "", + "| Worker | Static bytes | Chunks | Compile | Evaluate |", + "|---|---:|---:|---:|---:|", + ...Object.entries(workers).map( + ([worker, measured]) => + `| ${worker} | ${measured.staticBytes.head.toLocaleString("en-US")} | ${measured.chunks.head} | ${cell(measured.compile)} | ${cell(measured.evaluate)} |`, + ), + ...Object.entries(harness.skipped).map( + ([worker, why]) => `| ${worker} | skipped: ${why} | | | |`, + ), + "", +].join("\n"); +console.log(table); +const summary = process.env["GITHUB_STEP_SUMMARY"]; +if (summary !== undefined && summary !== "") appendFileSync(summary, `${table}\n`); diff --git a/scripts/e2e-prepare.ts b/scripts/e2e-prepare.ts index d2b99d4355..77e722d811 100644 --- a/scripts/e2e-prepare.ts +++ b/scripts/e2e-prepare.ts @@ -1,14 +1,22 @@ /** - * Builds what the e2e servers load: the staged `apps` package and its tarball, Motel, the workerd + * Builds what the e2e servers load: the staged `apps` package and its tarball, the telemetry + * collector and its viewer, the workerd * host bundle, the two dashboards and the pinned npm archives the suite's registry serves. Each step's outputs are cached under a hash of its inputs, * so a rift that has not changed a step's inputs restores its outputs instead of building them. * * A step's inputs are the files git sees (tracked, or untracked and not ignored) in its * directories and in the workspace packages they depend on, the root files every build reads, the - * Node and Bun versions and the environment variables the builds read. + * Node and Bun versions and the environment variables the builds read. A package's files leave out + * the workspace packages inside it (each is an input only of the steps that depend on it) and + * `notPackageSources`. * * The cache lives in `$EXECUTOR_E2E_CACHE`, or `~/.cache/executor-e2e-prepare`. CI and * `--no-cache` build every step without reading or writing it. + * + * `--skip ` neither runs a step nor reads or changes its outputs: something else provides them. + * CI skips `collector:build` and then fetches the run's one collector build into + * `packages/telemetry/dist/collector` (checks.yml), so its jobs need no Rust toolchain; a job whose + * fetch fails stops before any scenario starts. */ import { execFileSync, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; @@ -26,6 +34,12 @@ type Step = { /** Steps whose outputs this step reads. Their keys are part of this step's key. */ readonly after: ReadonlyArray; readonly outputs: ReadonlyArray; + /** + * `false`: always run the command, never this cache. The collector's build script keeps its own + * identity (toolchain, C compiler, linker, Cargo config) and its own store, and checks the + * pinned toolchain; a cache keyed on less would skip both. + */ + readonly cached?: false; }; const steps: ReadonlyArray = [ @@ -39,17 +53,25 @@ const steps: ReadonlyArray = [ outputs: ["packages/apps/dist"], }, { - // The Motel fork is pinned by revision in motel/source.json and has its own lockfile. - name: "telemetry:build", - command: ["bun", "run", "telemetry:build"], + // The telemetry collector. Its build script runs every time: it checks the pinned toolchain, + // computes the build's whole identity and copies an unchanged build from its own store in a + // fraction of a second (packages/telemetry/collector/README.md, Release builds). + name: "collector:build", + command: ["bun", "run", "collector:build"], packages: [], - files: [ - "packages/telemetry/package.json", - "packages/telemetry/motel", - "packages/telemetry/scripts", - ], + files: ["packages/telemetry/collector", "packages/telemetry/scripts/build-collector.ts"], + after: [], + outputs: ["packages/telemetry/dist/collector"], + cached: false, + }, + { + // The trace viewer the collector serves. + name: "viewer:build", + command: ["bun", "run", "viewer:build"], + packages: ["packages/telemetry/viewer"], + files: [], after: [], - outputs: ["packages/telemetry/dist/motel"], + outputs: ["packages/telemetry/dist/viewer"], }, { name: "e2e:runtime", @@ -96,6 +118,16 @@ const steps: ReadonlyArray = [ }, ]; +/** + * Files inside a workspace package that are not its sources: no build of its dependents reads + * them. The collector's crate and build script sit in `@executor-js/telemetry`, but only + * `collector:build` reads them, and it names them itself. + */ +const notPackageSources = [ + "packages/telemetry/collector", + "packages/telemetry/scripts/build-collector.ts", +]; + /** * Root files the builds read: the manifest and lockfile, dependency patches, the base tsconfig, * and `.gitignore`, which Tailwind reads to choose the files it scans for class names. @@ -110,6 +142,23 @@ process.env.EXECUTOR_DASHBOARD_SOURCEMAP = "hidden"; const keep = 20; const useCache = !process.env.CI && !process.argv.includes("--no-cache"); +const skipped = new Set(); +for (let index = 2; index < process.argv.length; index++) { + const arg = process.argv[index]; + if (arg === "--no-cache") continue; + if (arg !== "--skip") { + console.error(`Unknown argument ${arg}. Options: --no-cache, --skip .`); + process.exit(2); + } + const name = process.argv[++index] ?? ""; + if (!steps.some((step) => step.name === name)) { + console.error( + `--skip names no step: ${JSON.stringify(name)}. Steps: ${steps.map((step) => step.name).join(", ")}.`, + ); + process.exit(2); + } + skipped.add(name); +} const cacheRoot = process.env.EXECUTOR_E2E_CACHE ?? path.join(os.homedir(), ".cache", "executor-e2e-prepare"); @@ -172,20 +221,40 @@ const hashFile = (file: string) => { return hash; }; +/** The files git sees under `paths`: tracked, or untracked and not ignored. */ +const gitFiles = (paths: ReadonlyArray): ReadonlyArray => + paths.length === 0 + ? [] + : execFileSync("git", ["ls-files", "-z", "-co", "--exclude-standard", "--", ...paths], { + encoding: "utf8", + maxBuffer: 256 * 1024 * 1024, + }) + .split("\0") + .filter(Boolean); + +const within = (file: string, directory: string) => + file === directory || file.startsWith(`${directory}/`); + +/** A package's own files: without the workspace packages inside it and `notPackageSources`. */ +const packageFiles = (found: string) => { + const directory = found.split(path.sep).join("/"); + const left = [ + ...notPackageSources, + ...[...workspaces.values()] + .map((other) => other.split(path.sep).join("/")) + .filter((other) => other !== directory && within(other, directory)), + ]; + return gitFiles([directory]).filter((file) => !left.some((out) => within(file, out))); +}; + const keys = new Map(); const keyOf = (step: Step) => { - const paths = [...rootFiles, ...step.files, ...closure(step.packages)]; - const files = execFileSync( - "git", - ["ls-files", "-z", "-co", "--exclude-standard", "--", ...paths], - { - encoding: "utf8", - maxBuffer: 256 * 1024 * 1024, - }, - ) - .split("\0") - .filter(Boolean) - .sort(); + const files = [ + ...new Set([ + ...gitFiles([...rootFiles, ...step.files]), + ...closure(step.packages).flatMap(packageFiles), + ]), + ].sort(); const hash = createHash("sha256"); hash.update( JSON.stringify({ @@ -243,8 +312,12 @@ const store = (step: Step, entry: string) => { const started = performance.now(); const summary: Array = []; for (const step of steps) { + if (skipped.has(step.name)) { + summary.push(`${step.name}: skipped, provided elsewhere`); + continue; + } const stepStarted = performance.now(); - const key = useCache ? keyOf(step) : undefined; + const key = useCache && step.cached !== false ? keyOf(step) : undefined; if (key !== undefined) keys.set(step.name, key); const entry = key === undefined ? undefined : path.join(cacheRoot, step.name, key); if (entry !== undefined && fs.existsSync(entry)) { @@ -255,13 +328,31 @@ for (const step of steps) { summary.push(`${step.name}: cached`); } else { const [command, ...args] = step.command; - const run = spawnSync(command!, args, { stdio: "inherit" }); + // A step can leave lines for the summary in this file: the collector's build says so when the + // shared store it stores builds in cannot be written. + const notes = path.join( + os.tmpdir(), + `executor-prepare-notes-${process.pid}-${step.name.replace(/\W/gu, "-")}`, + ); + fs.rmSync(notes, { force: true }); + const run = spawnSync(command!, args, { + stdio: "inherit", + env: { ...process.env, EXECUTOR_PREPARE_NOTES: notes }, + }); + const noted = fs.existsSync(notes) + ? fs.readFileSync(notes, "utf8").split("\n").filter(Boolean) + : []; + fs.rmSync(notes, { force: true }); if (run.status !== 0) { console.error(`\n${step.name} failed (${run.signal ?? `exit ${run.status}`}).`); process.exit(run.status ?? 1); } if (entry !== undefined) store(step, entry); - summary.push(`${step.name}: built`); + summary.push( + `${step.name}: built in ${((performance.now() - stepStarted) / 1000).toFixed(1)}s`, + ); + for (const line of new Set(noted)) summary.push(` ${step.name}: ${line}`); + continue; } summary[summary.length - 1] += ` in ${((performance.now() - stepStarted) / 1000).toFixed(1)}s`; } diff --git a/scripts/releases/README.md b/scripts/releases/README.md index d008e8c421..d2b813171c 100644 --- a/scripts/releases/README.md +++ b/scripts/releases/README.md @@ -20,7 +20,7 @@ no `executor service`, so run `executor serve` there instead. Executor is one executable, built with Bun, with the files it runs or reads beside it: the dashboard, the app framework and compiler references, workerd, -Git, the native credential-store binding and the Motel collector. It needs no +Git, the native credential-store binding and the telemetry collector with its viewer. It needs no source checkout, Node, Bun or 1Password on the user machine. The `executor` npm package is a launcher. The first time a version starts, it downloads that release's archive for the platform, for macOS (Apple Silicon and Intel), Linux diff --git a/scripts/releases/apps-self-host.ts b/scripts/releases/apps-self-host.ts index d3006c7f53..b0ea204908 100644 --- a/scripts/releases/apps-self-host.ts +++ b/scripts/releases/apps-self-host.ts @@ -7,8 +7,8 @@ */ import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { Config, Effect, FileSystem, Path, Schema } from "effect"; -import { FetchHttpClient, HttpClient } from "effect/http"; +import { Config, Effect, FileSystem, Path, Result, Schema } from "effect"; +import { FetchHttpClient, HttpClient, Url } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import apps from "../../packages/apps/package.json" with { type: "json" }; import { @@ -32,6 +32,44 @@ const args = process.argv.slice(2); const publish = args.includes("--publish"); const packageIndex = args.indexOf("--package"); const packageDirectory = packageIndex < 0 ? staged : args[packageIndex + 1]; +const redirectLimit = 5; + +/** The origin of an absolute URL, or `undefined` when the text is not one. */ +const originOf = (url: string) => { + const parsed = Url.fromString(url); + return Result.isSuccess(parsed) ? parsed.success.origin : undefined; +}; + +/** + * GET the SDK tarball without reaching an origin the builder refuses. `fetch` follows redirects + * by default, so a same-origin URL could land anywhere; each hop is requested with manual + * redirects and refused unless it stays on `origin`, the builder registry's own. + */ +const downloadFromRegistry = (url: string, origin: string) => + Effect.gen(function* () { + const http = yield* HttpClient.HttpClient; + let target = url; + for (let hop = 0; ; hop += 1) { + const response = yield* http + .get(target) + .pipe(Effect.provideService(FetchHttpClient.RequestInit, { redirect: "manual" })); + const location = response.headers.location; + if (response.status < 300 || response.status >= 400 || location === undefined) + return response; + yield* response.text; + const parsed = Url.fromString(location, target); + const next = Result.isSuccess(parsed) ? parsed.success : undefined; + if (next === undefined || next.origin !== origin) + return yield* new AppsReleaseMismatch({ + message: `Builder SDK tarball on ${origin} redirects to ${next?.origin ?? location}, outside the builder registry's origin. The builder refuses that tarball, so deployment is refused.`, + }); + if (hop >= redirectLimit) + return yield* new AppsReleaseMismatch({ + message: `Builder SDK tarball redirects more than ${redirectLimit} times. Deployment refused.`, + }); + target = next.href; + } + }); NodeRuntime.runMain( Effect.gen(function* () { @@ -124,7 +162,13 @@ NodeRuntime.runMain( return yield* new AppsReleaseMismatch({ message: "Builder registry returned another SDK version.", }); - const archive = yield* http.get(published.dist.tarball); + const registryOrigin = originOf(registry); + const tarballOrigin = originOf(published.dist.tarball); + if (registryOrigin === undefined || tarballOrigin !== registryOrigin) + return yield* new AppsReleaseMismatch({ + message: `Builder registry metadata at ${registryOrigin ?? registry} serves its SDK tarball at ${published.dist.tarball} (${tarballOrigin ?? "not an absolute URL"}), outside the builder registry's origin. The builder refuses that tarball, so deployment is refused.`, + }); + const archive = yield* downloadFromRegistry(published.dist.tarball, registryOrigin); if (archive.status !== 200) { yield* archive.text; return yield* new AppsReleaseMismatch({ diff --git a/scripts/releases/channel-guard.ts b/scripts/releases/channel-guard.ts new file mode 100644 index 0000000000..c6deb20205 --- /dev/null +++ b/scripts/releases/channel-guard.ts @@ -0,0 +1,341 @@ +/** + * Refuse to move a release channel backwards. Each step of the publish job that moves a channel + * runs this first, for its own surface: + * + * node scripts/releases/channel-guard.ts image # IMAGE:CHANNEL on GHCR + * node scripts/releases/channel-guard.ts npm # npm's CHANNEL dist-tag for executor + * node scripts/releases/channel-guard.ts homebrew # the cloned tap's formula for CHANNEL + * node scripts/releases/channel-guard.ts desktop-feed # each executor-v2-CHANNEL*.yml + * node scripts/releases/channel-guard.ts preflight # all four, before anything moves + * + A channel that names a version newer than VERSION (`2.0.0-beta.10` is newer than `2.0.0-beta.9`; + * `2.0.0` is newer than its betas) stops it, exit 1, saying which channel names what, before + * anything changes: that is what re-running an old publication after a newer one would find. The + * same version, an older one or nothing yet lets it go on. + * + * A moving step's own check, in the middle of a publication, reads patiently (public-state.ts: + * about two minutes) and stops when it still cannot read the channel or cannot order what it + * names. A stop is loud and re-running the failed jobs finishes it; a backward move is silent. + * + * `preflight` is the publish job's first step after setup, so a stale run stops before anything + * moves and says why at once: + * - It refuses when any surface it could read names a newer version. A newer release that was + * half published shows on the image first, so npm alone is not enough. + * - npm decides whether to go on at all: it is read patiently, and a publication stops when npm + * cannot be read or names what cannot be ordered, since its npm step would fail later, after the + * release is public and the image moved. + * - The image, the formula the tap publishes (TAP; the tap is not cloned yet) and the feed are + * read quickly. One that cannot be read or names what cannot be ordered is a `::warning` naming + * the surface and what was seen, and the publication goes on: its own step checks it again. + * + * - Before anything else: a version npm already holds is refused unless this is the resume of the + * same publication, which finds a published release for RELEASE_TAG (see the code for how the + * draft step and the launcher check make that exact). The message names the version, when npm + * published it and whether it is deprecated, and says the release needs another number. + * + * node scripts/releases/channel-guard.ts launcher # after release:wrapper + * + * `launcher` refuses when npm already holds VERSION with an archive other than this run's launcher, + * before the upload, the image and the release's publication; npm holding nothing, or this very + * launcher (a resumed publication), lets the job go on. + * + * A first publication under a new image name is a warning in the preflight and a stop at the image + * step: see public-state.ts for why and what to do. + * + * VERSION's own channel must be CHANNEL: a beta never moves the stable channel, whose tag, formula + * and feed files a beta's steps never name, and a stable release never moves only the beta one. + * + * Reads VERSION, CHANNEL, IMAGE and PUBLIC_REPOSITORY, as the publish job sets them, and changes + * nothing anywhere (public-state.ts). + */ +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { createHash } from "node:crypto"; +import { Config, Console, Effect, FileSystem, Path, Schema } from "effect"; +import { FetchHttpClient } from "effect/http"; +import { releaseChannel, ReleaseVersion } from "@executor-js/utils/release-version"; +import { + type Endpoints, + endpoints, + feedVersions, + formulaName, + formulaVersion, + imageVersion, + newerThan, + npmArchive, + npmPublishedAt, + npmTags, + githubRelease, + patiently, + publishedFormulaVersion, +} from "./public-state.ts"; + +class ChannelGuardFailed extends Schema.TaggedError()("ChannelGuardFailed", { + message: Schema.String, +}) {} + +const surfaces = ["image", "npm", "homebrew", "desktop-feed", "preflight", "launcher"] as const; +type Surface = (typeof surfaces)[number]; +const isSurface = (value: string | undefined): value is Surface => + surfaces.some((surface) => surface === value); + +const program = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const [surface, tap] = process.argv.slice(2); + if (!isSurface(surface)) + return yield* new ChannelGuardFailed({ + message: `Usage: channel-guard.ts <${surfaces.join("|")}> [tap directory | launcher archive]`, + }); + const version = yield* Config.String("VERSION"); + if (!Schema.is(ReleaseVersion)(version)) + return yield* new ChannelGuardFailed({ message: `${version} is not a release version.` }); + const channel = yield* Config.String("CHANNEL"); + if (channel !== releaseChannel(version)) + return yield* new ChannelGuardFailed({ + message: `${version} publishes to ${releaseChannel(version)}, not ${channel}: a beta never moves the stable channel. Nothing was changed.`, + }); + const at = yield* endpoints; + const patient = patiently(at); + + type Seen = { readonly where: string; readonly current: string | undefined }; + const image = (from: Endpoints) => + Effect.gen(function* () { + const name = yield* Config.String("IMAGE"); + return [{ where: `${name}:${channel}`, current: yield* imageVersion(from, name, channel) }]; + }); + const npm = (from: Endpoints) => + npmTags(from, "executor").pipe( + Effect.map((tags) => [{ where: `npm's executor@${channel}`, current: tags?.[channel] }]), + ); + const feed = (from: Endpoints) => + Effect.gen(function* () { + const repository = yield* Config.String("PUBLIC_REPOSITORY"); + return (yield* feedVersions(from, repository, channel)).map((file) => ({ + where: `the desktop feed's ${file.name}`, + current: file.version, + })); + }); + const clonedFormula = (directory: string) => + Effect.gen(function* () { + // A missing tap is a missing input, not a channel that names nothing yet: stop. + const tapExists = yield* fs.stat(directory).pipe( + Effect.map((info) => info.type === "Directory"), + Effect.orElseSucceed(() => false), + ); + if (!tapExists) + return yield* new ChannelGuardFailed({ + message: `The tap directory ${directory} does not exist. Nothing was changed; clone the tap there first.`, + }); + const file = path.join(directory, "Formula", `${formulaName(channel)}.rb`); + const current = (yield* fs.exists(file)) + ? (formulaVersion(yield* fs.readFileString(file)) ?? "no version") + : undefined; + return [{ where: `the ${formulaName(channel)} formula`, current }]; + }); + const publishedFormula = (from: Endpoints) => + Effect.gen(function* () { + const name = yield* Config.String("TAP").pipe(Config.withDefault("")); + if (name === "") return []; + return [ + { + where: `${name}'s ${formulaName(channel)} formula`, + current: yield* publishedFormulaVersion(from, name, channel), + }, + ]; + }); + + const newerMessage = (seen: ReadonlyArray) => + `${seen.map(({ where, current }) => `${where} names ${current}`).join("; ")}: newer than ${version}. Nothing was changed: a newer release already moved this channel, and ${version} must not move it back. Do not re-run this publication.`; + const unorderedMessage = ({ where, current }: Seen) => + `${where} names ${current}, which cannot be ordered against ${version}, so moving it cannot be shown to go forward.`; + const forward = ({ where, current }: Seen) => + Console.log( + current === undefined + ? `${where} names nothing yet; ${version} may move it.` + : `${where} names ${current}; ${version} may move it.`, + ); + + /** What npm holds as `executor@VERSION`, read patiently; a read that fails stops the step. */ + const held = npmArchive(patient, "executor", version).pipe( + Effect.catchTag("PublicReadFailed", (error) => + Effect.fail( + new ChannelGuardFailed({ + message: `Cannot read whether npm holds executor@${version} (${error.message}). Nothing was changed; re-run the failed jobs once npm answers.`, + }), + ), + ), + ); + const deprecation = (archive: { readonly deprecated: string | undefined }) => + archive.deprecated === undefined ? "not deprecated" : `deprecated: "${archive.deprecated}"`; + + if (surface === "launcher") { + // After release:wrapper, before the upload, the image and the release's publication: npm must + // not already hold this version with another archive. On a version npm does not hold this is + // one read; on a resumed publication npm holds this very launcher. + if (tap === undefined) + return yield* new ChannelGuardFailed({ message: "launcher needs the launcher archive" }); + const bytes = yield* fs.readFile(tap); + const built = `sha512-${createHash("sha512").update(bytes).digest("base64")}`; + const archive = yield* held; + if (archive === undefined) + return yield* Console.log( + `npm does not hold executor@${version} yet; this launcher may be published.`, + ); + if (archive.integrity === built) + return yield* Console.log( + `npm already serves this launcher as executor@${version}, byte for byte: a resumed publication.`, + ); + // Two causes, told apart by the release: a published release for this tag that targets this + // run's export commit means this release was published from this source, so the likely cause + // is a re-run of the whole workflow, whose rebuilt archives differ from the ones it published; + // anything else means npm's archive is not this source's, and the version cannot be used. + const repository = yield* Config.String("PUBLIC_REPOSITORY"); + const tag = yield* Config.String("RELEASE_TAG"); + const token = yield* Config.String("GH_TOKEN").pipe(Config.withDefault("")); + const exportFile = yield* Config.String("PUBLIC_EXPORT_COMMIT_FILE").pipe( + Config.withDefault(""), + ); + const exported = + exportFile !== "" && (yield* fs.exists(exportFile)) + ? (yield* fs.readFileString(exportFile)).trim() + : undefined; + const release = yield* githubRelease(patient, repository, tag, token).pipe( + Effect.orElseSucceed(() => undefined), + ); + const ours = + release !== undefined && + !release.draft && + exported !== undefined && + release.target === exported; + const remedy = ours + ? `The release ${tag} is published and targets this run's export commit (${exported}), so the likely cause is a re-run of the whole workflow, whose rebuilt archives differ from the ones that release published: re-run the failed jobs of the original run instead, not the whole workflow.` + : `${repository} has no published release ${tag} from this run's export commit, so npm's archive is not this source's, and npm never accepts a version twice: this release needs another version number.`; + return yield* new ChannelGuardFailed({ + message: `npm serves a different archive as executor@${version} (${archive.integrity}, ${deprecation(archive)}), not the launcher this run built (${built}). ${remedy} Nothing public was changed: the upload, the image and the release's publication come after this step.`, + }); + } + + if (surface !== "preflight") { + // In the middle of a publication: read patiently, and stop when it cannot read or order what + // the channel names. A stop is loud and a re-run finishes it; a backward move is silent. + const named: ReadonlyArray = yield* Effect.gen(function* () { + switch (surface) { + case "image": + return yield* image(patient); + case "npm": + return yield* npm(patient); + case "desktop-feed": + return yield* feed(patient); + case "homebrew": + if (tap === undefined) + return yield* new ChannelGuardFailed({ message: "homebrew needs the tap directory" }); + return yield* clonedFormula(tap); + } + }).pipe( + Effect.catchTag("PublicReadFailed", (error) => + Effect.fail( + new ChannelGuardFailed({ + message: `Cannot read what ${surface}'s ${channel} channel names after about two minutes of asking (${error.message}). Nothing was changed; re-run the failed jobs once it answers.`, + }), + ), + ), + ); + for (const seen of named) { + if (seen.current !== undefined) { + const newer = newerThan(seen.current, version); + if (newer === undefined) + return yield* new ChannelGuardFailed({ + message: `${unorderedMessage(seen)} Nothing was changed; inspect it.`, + }); + if (newer) return yield* new ChannelGuardFailed({ message: newerMessage([seen]) }); + } + yield* forward(seen); + } + return; + } + + // Before anything moves, and before anything else is read: a version npm already holds is only + // published again as the resume of this same publication. A publication makes its release + // public before npm gets the launcher, so a resume finds a published release for RELEASE_TAG; + // the draft step then checks that release targets this run's export commit, and the launcher + // step that npm's archive is this build's. Without a published release, npm's archive is not + // this pipeline's, and npm never accepts a version twice. + const stray = yield* held; + if (stray !== undefined) { + const repository = yield* Config.String("PUBLIC_REPOSITORY"); + const tag = yield* Config.String("RELEASE_TAG"); + const token = yield* Config.String("GH_TOKEN").pipe(Config.withDefault("")); + const release = yield* githubRelease(patient, repository, tag, token).pipe( + Effect.catchTag("PublicReadFailed", (error) => + Effect.fail( + new ChannelGuardFailed({ + message: `npm already serves executor@${version}, and the release ${tag} cannot be read (${error.message}), so this cannot tell a resumed publication from a version npm holds from elsewhere. Nothing was changed; re-run the failed jobs once GitHub answers.`, + }), + ), + ), + ); + if (release === undefined || release.draft) { + const published = yield* npmPublishedAt(at, "executor", version).pipe( + Effect.orElseSucceed(() => undefined), + ); + return yield* new ChannelGuardFailed({ + message: `npm already serves executor@${version} (published ${published ?? "at a time npm does not say"}, ${deprecation(stray)}), and this is not a resumed publication of it: ${repository} has no published release ${tag}, which a publication makes public before npm gets the launcher. npm never accepts a version twice, so this release needs another version number. Nothing was changed.`, + }); + } + yield* Console.log( + `npm already serves executor@${version} and ${repository} has published ${tag}: a resumed publication. The draft step checks that release is this run's export, and the launcher step that npm's archive is this build's.`, + ); + } + + // npm decides whether to go on: it is read patiently, and a publication whose npm step + // would fail later, after the release is public and the image moved, stops here instead. + const npmSeen = yield* npm(patient).pipe( + Effect.catchTag("PublicReadFailed", (error) => + Effect.fail( + new ChannelGuardFailed({ + message: `Cannot read npm's dist-tags (${error.message}). Nothing was changed: the npm step would fail later, after the release is public. Re-run the failed jobs once npm answers.`, + }), + ), + ), + ); + for (const seen of npmSeen) + if (seen.current !== undefined && newerThan(seen.current, version) === undefined) + return yield* new ChannelGuardFailed({ + message: `${unorderedMessage(seen)} Nothing was changed; inspect it.`, + }); + // The others are read quickly. One that cannot be read, or names what cannot be ordered, is a + // warning: its own step checks it again, patiently, before moving it. + const warn = (line: string) => Console.log(`::warning title=Channel guard::${line}`); + const others: Array = []; + const tryRead = (label: string, read: Effect.Effect, E, R>) => + Effect.gen(function* () { + const result = yield* Effect.result(read); + if (result._tag === "Success") return others.push(...result.success); + const failure: unknown = result.failure; + const text = failure instanceof Error ? failure.message : String(failure); + yield* warn( + `Cannot read ${label} (${text}); going on, and its step checks it again before moving it.`, + ); + }); + yield* tryRead("the image", image(at)); + yield* tryRead("the published formula", publishedFormula(at)); + yield* tryRead("the desktop feed", feed(at)); + const readable: Array = []; + for (const seen of [...npmSeen, ...others]) { + if (seen.current !== undefined && newerThan(seen.current, version) === undefined) { + yield* warn(`${unorderedMessage(seen)} Going on; its step checks it again before moving it.`); + continue; + } + readable.push(seen); + } + // Any channel it could read naming a newer version refuses: a newer release half published + // shows on the image first. + const newer = readable.filter( + (seen) => seen.current !== undefined && newerThan(seen.current, version) === true, + ); + if (newer.length > 0) return yield* new ChannelGuardFailed({ message: newerMessage(newer) }); + for (const seen of readable) yield* forward(seen); +}); + +NodeRuntime.runMain(program.pipe(Effect.provide([NodeServices.layer, FetchHttpClient.layer]))); diff --git a/scripts/releases/collector-job.ts b/scripts/releases/collector-job.ts index d226ef55d2..8ac6d7636e 100644 --- a/scripts/releases/collector-job.ts +++ b/scripts/releases/collector-job.ts @@ -14,7 +14,7 @@ */ import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { Config, Effect } from "effect"; +import { Config, Effect, Schema } from "effect"; import { FetchHttpClient } from "effect/http"; import { waitForJob } from "./job-wait.ts"; @@ -30,12 +30,17 @@ const minutes = 25; NodeRuntime.runMain( Effect.gen(function* () { const target = yield* Config.String("TARGET"); + // CI's checks.yml runs as a called workflow, its jobs listed under the caller's (`checks`). + const caller = yield* Config.String("CALLER").pipe(Config.option); + // What this job needs the collector for: packaging (the release workflow) or scenarios (CI). + const use = yield* Config.schema(Schema.Literals(["packaging", "scenarios"]), "COLLECTOR_USE"); const job = `collector (${target})`; yield* waitForJob({ job, + caller: caller._tag === "Some" ? caller.value : undefined, minutes, ended: (conclusion, url) => - `The ${job} job ended ${conclusion}, so this run has no ${target} collector to package.${url === undefined ? "" : ` ${url}`}`, + `The ${job} job ended ${conclusion}, so this job has no ${target} collector for its ${use}; this failure follows from that one. Rerun the ${job} job, then this one.${url === undefined ? "" : ` ${url}`}`, late: (reason, limit) => `The ${job} job did not finish within ${limit} (${reason}). Rerun this job once it has.`, succeeded: `The ${job} job succeeded; fetching its build.`, diff --git a/scripts/releases/desktop.ts b/scripts/releases/desktop.ts index 482c39bf32..16b6b8dc37 100644 --- a/scripts/releases/desktop.ts +++ b/scripts/releases/desktop.ts @@ -7,6 +7,7 @@ import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { desktopUpdateFeed, desktopUpdateFile, nativePlatform, release } from "./config.ts"; import { developerIdMac, unsignedMac } from "./macos-signing.ts"; import { installNodeRuntime } from "./node-runtime.ts"; +import { smokePackagedCollector } from "./standalone.ts"; import { DesktopRuntimeBuild, desktopRuntimeBuildFile, @@ -170,6 +171,8 @@ const build = Effect.gen(function* () { signing.env, ); yield* signing.finalize(artifacts); + // The app's own copy of the collector, as it ships: every unpacked app this build wrote. + yield* smokePackagedCollector(root, target, artifacts); if (updates && !process.argv.includes("--dir")) { const metadata = path.join(artifacts, desktopUpdateFile(target, release.channel)); if (!(yield* fs.exists(metadata))) diff --git a/scripts/releases/job-wait.ts b/scripts/releases/job-wait.ts index b2580bd4c6..311e08a918 100644 --- a/scripts/releases/job-wait.ts +++ b/scripts/releases/job-wait.ts @@ -127,6 +127,11 @@ const spoken = (seconds: number) => export type JobWait = { /** The job's name as the jobs API lists it, such as `apps` or `collector (linux-x64)`. */ readonly job: string; + /** + * The calling workflow's job, when the waited job is in a called workflow: it is then also found + * as ` / `, and only so. Unset, the name must match exactly. + */ + readonly caller?: string | undefined; /** How long to wait in all, in minutes of elapsed time. */ readonly minutes: number; /** The failure for a job that ended `conclusion`; `url` is its page, when the API gave one. */ @@ -238,7 +243,14 @@ export const waitForJob = (options: JobWait) => ) return yield* changed; // A consistent read. Only this attempt's records count; another attempt's is never the answer. - const named = jobs.filter((candidate) => candidate.name === options.job); + // A job of a called workflow is listed under its caller's job, as `checks / collector + // (linux-x64)` in CI, which calls checks.yml as `checks`. Only a waiter that names that + // caller (`caller`) takes that one name too; every other waiter matches the name exactly. + const named = jobs.filter( + (candidate) => + candidate.name === options.job || + (options.caller !== undefined && candidate.name === `${options.caller} / ${options.job}`), + ); const matches = named.filter((candidate) => candidate.run_attempt === runAttempt); const others = named.filter((candidate) => candidate.run_attempt !== runAttempt); const duplicated = matches.length > 1; diff --git a/scripts/releases/npm-channel.ts b/scripts/releases/npm-channel.ts new file mode 100644 index 0000000000..de950d2910 --- /dev/null +++ b/scripts/releases/npm-channel.ts @@ -0,0 +1,234 @@ +/** + * Wait for npm's dist-tags to name a release that npm already serves, and fail on what was seen. + * + * npm moves a dist-tag some time after it serves the version itself: beta.51's version document + * answered at 21:16:00, and one read of the dist-tags 0.1 s later still named beta.50, though + * `npm view` named beta.51 34 s after that. So the tag is read until it names this release, for a + * bounded time, and the wait ends on the first answer that settles it: + * + * - The channel's tag names this release (and, for a beta, `latest` is what it was before + * publishing): done. + * - The channel's tag still names what it named before publishing: npm has not caught up. Ask + * again, until the limit; then fail naming what the tag still says. + * - A beta's publication changed `latest`: fail at once. A late read shows the old value, so any + * other value is a real change, which must be restored by hand. + * - The channel's tag names some third version: another publication moved it. Fail at once. + * + * A request that fails, times out or answers unreadably is not an answer, and is asked again. + * The limit has no override in production: `publish-npm.ts` passes none. + * + * Run directly, it waits once with its inputs from the environment, as e2e/tests/npm-channel.spec.ts + * runs it against a local registry: `NPM_CHANNEL`, `NPM_CHANNEL_VERSION`, `NPM_CHANNEL_BEFORE` (the + * dist-tags before publishing, as JSON), and optionally `NPM_CHANNEL_REGISTRY` and + * `NPM_CHANNEL_PACKAGE` (npm and `executor`). Against another registry only, + * `NPM_CHANNEL_LIMIT_SECONDS`, `NPM_CHANNEL_POLL_SECONDS` and `NPM_CHANNEL_ATTEMPT_SECONDS` shorten + * the wait, so a test can reach its end; against npm they are refused. + */ +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import { realpathSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { Config, Console, Duration, Effect, Schedule, Schema } from "effect"; +import { compareReleaseVersions } from "@executor-js/utils/release-version"; +import { FetchHttpClient, HttpClient } from "effect/http"; + +export const npmRegistry = "https://registry.npmjs.org"; + +/** Whether `registry` is npm's, however it is written (a trailing slash, a path, another case). */ +const isNpm = (registry: string) => { + try { + return new URL(registry).hostname.toLowerCase() === new URL(npmRegistry).hostname; + } catch { + // Not a URL: no request to it can reach npm either, so its limits may stand. + return false; + } +}; + +/** A version `compareReleaseVersions` can order: `2.0.0`, `2.0.0-beta.10`, Executor 1's `1.6.11`. */ +const orderable = /^\d+\.\d+\.\d+(?:-beta\.\d+)?$/u; + +export const DistTags = Schema.Record(Schema.String, Schema.String); +export type DistTags = typeof DistTags.Type; + +export class NpmChannelFailed extends Schema.TaggedError()("NpmChannelFailed", { + message: Schema.String, +}) {} + +/** The tags have not settled yet, or the registry could not say: ask again, within the limit. */ +class NotYet extends Schema.TaggedError()("NotYet", { reason: Schema.String }) {} + +export type NpmChannelWait = { + /** The registry's origin, such as https://registry.npmjs.org. */ + readonly registry: string; + readonly packageName: string; + /** The dist-tag this release moves: `beta` or `latest`. */ + readonly channel: string; + readonly version: string; + /** The dist-tags as read before publishing. */ + readonly before: DistTags; + /** How long to wait in all; 10 minutes. */ + readonly limitSeconds?: number; + /** How long between two reads; 10 s. */ + readonly pollSeconds?: number; + /** How long one read may take, its body included; 15 s. */ + readonly attemptSeconds?: number; +}; + +const named = (tags: DistTags, tag: string) => tags[tag] ?? "nothing"; + +/** + * One read of `packageName`'s dist-tags within `attemptSeconds`, its body included. A failed, + * timed-out or unreadable answer, or one without a `latest` tag (npm always has one, so an answer + * without it is an error body), is `NotYet`, with the reason. + */ +export const readDistTags = (registry: string, packageName: string, attemptSeconds: number) => + Effect.gen(function* () { + const http = yield* HttpClient.HttpClient; + const response = yield* http + .get(`${registry}/-/package/${packageName}/dist-tags`) + .pipe(Effect.mapError(() => new NotYet({ reason: "the registry did not answer" }))); + if (response.status !== 200) { + yield* response.text.pipe(Effect.ignore); + return yield* new NotYet({ reason: `the registry answered ${response.status}` }); + } + const tags = yield* response.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(DistTags)), + Effect.mapError(() => new NotYet({ reason: "the registry answered unreadably" })), + ); + if (tags.latest === undefined) + return yield* new NotYet({ reason: "the registry answered without a latest tag" }); + return tags; + }).pipe( + Effect.timeoutOrElse({ + duration: Duration.seconds(attemptSeconds), + orElse: () => + Effect.fail( + new NotYet({ reason: `the registry did not answer within ${attemptSeconds} s` }), + ), + }), + ); + +/** + * The dist-tags before publishing, read a few times within about a minute: nothing is published + * yet, so a registry that cannot answer stops the publication with what it last said. + */ +export const distTagsBeforePublishing = (registry: string, packageName: string) => + readDistTags(registry, packageName, 15).pipe( + Effect.retry({ schedule: Schedule.spaced(Duration.seconds(5)), times: 3 }), + Effect.mapError( + (notYet) => + new NpmChannelFailed({ + message: `Cannot read npm's dist-tags for ${packageName} before publishing (${notYet.reason}). Nothing was published.`, + }), + ), + ); + +/** Reads the dist-tags until they name `version` under `channel`; returns them. */ +export const awaitNpmChannel = (options: NpmChannelWait) => + Effect.gen(function* () { + const limit = options.limitSeconds ?? 600; + const poll = options.pollSeconds ?? 10; + const attempt = options.attemptSeconds ?? 15; + const { channel, version, before } = options; + const read = readDistTags(options.registry, options.packageName, attempt); + + const look = Effect.gen(function* () { + const tags = yield* read; + if (channel !== "latest" && tags.latest !== before.latest) + return yield* new NpmChannelFailed({ + message: `npm's latest tag names ${named(tags, "latest")}, not ${named(before, "latest")} as before publishing ${version} to ${channel}. Restore latest by hand.`, + }); + if (tags[channel] === version) return tags; + const current = tags[channel]; + if ( + current !== undefined && + orderable.test(current) && + compareReleaseVersions(current, version) > 0 + ) + return yield* new NpmChannelFailed({ + message: `npm's ${channel} tag names ${current}, newer than ${version}: a newer release has published since, so this run is stale. Do not re-run it; the newer release carries everything.`, + }); + if (tags[channel] === before[channel]) + return yield* new NotYet({ + reason: `npm's ${channel} tag still names ${named(tags, channel)}`, + }); + return yield* new NpmChannelFailed({ + message: `npm's ${channel} tag names ${named(tags, channel)}: neither ${version} nor ${named(before, channel)}, which it named before publishing. Another publication moved it.`, + }); + }); + + let last = "no answer yet"; + const follow: Effect.Effect = Effect.suspend( + () => + look.pipe( + Effect.catchTag("NotYet", (notYet) => { + last = notYet.reason; + return Console.log( + `Waiting for npm's ${channel} tag to name ${version}: ${notYet.reason}`, + ).pipe(Effect.andThen(Effect.sleep(Duration.seconds(poll))), Effect.andThen(follow)); + }), + ), + ); + return yield* follow.pipe( + Effect.timeoutOrElse({ + duration: Duration.seconds(limit), + orElse: () => + Effect.fail( + new NpmChannelFailed({ + message: `npm serves ${options.packageName}@${version}, but its ${channel} tag did not name it within ${limit} s; the last read: ${last}. Before publishing, ${channel} named ${named(before, channel)} and latest ${named(before, "latest")}. Check \`npm view ${options.packageName} dist-tags\`: once the tag names ${version}, re-running the failed jobs resumes over the published version; if it names a version newer than ${version}, this run is stale and must not be re-run.`, + }), + ), + }), + ); + }); + +/** The wait as a command, with its inputs from the environment. */ +const command = Effect.gen(function* () { + const registry = yield* Config.String("NPM_CHANNEL_REGISTRY").pipe( + Config.withDefault(npmRegistry), + ); + const seconds = (name: string) => Config.Number(name).pipe(Config.option); + const limits = { + limitSeconds: yield* seconds("NPM_CHANNEL_LIMIT_SECONDS"), + pollSeconds: yield* seconds("NPM_CHANNEL_POLL_SECONDS"), + attemptSeconds: yield* seconds("NPM_CHANNEL_ATTEMPT_SECONDS"), + }; + const shortened = Object.values(limits).some((value) => value._tag === "Some"); + if (shortened && isNpm(registry)) + return yield* new NpmChannelFailed({ + message: `The NPM_CHANNEL_*_SECONDS limits apply only to another registry, not ${npmRegistry}.`, + }); + const channel = yield* Config.String("NPM_CHANNEL"); + const version = yield* Config.String("NPM_CHANNEL_VERSION"); + const before = yield* Config.String("NPM_CHANNEL_BEFORE").pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(DistTags))), + Effect.mapError( + () => + new NpmChannelFailed({ + message: "NPM_CHANNEL_BEFORE must be the dist-tags before publishing, as a JSON object", + }), + ), + ); + const packageName = yield* Config.String("NPM_CHANNEL_PACKAGE").pipe( + Config.withDefault("executor"), + ); + const tags = yield* awaitNpmChannel({ + registry, + packageName, + channel, + version, + before, + ...(limits.limitSeconds._tag === "Some" ? { limitSeconds: limits.limitSeconds.value } : {}), + ...(limits.pollSeconds._tag === "Some" ? { pollSeconds: limits.pollSeconds.value } : {}), + ...(limits.attemptSeconds._tag === "Some" + ? { attemptSeconds: limits.attemptSeconds.value } + : {}), + }); + yield* Console.log(`npm's ${channel} tag names ${version}; latest=${named(tags, "latest")}`); +}); + +// Run as the main module: Node loads it by its real path, which argv[1] may reach through a link. +if ( + process.argv[1] !== undefined && + realpathSync(process.argv[1]) === fileURLToPath(import.meta.url) +) + NodeRuntime.runMain(command.pipe(Effect.provide(FetchHttpClient.layer))); diff --git a/scripts/releases/package.ts b/scripts/releases/package.ts index 22e71f218d..670094d2f6 100644 --- a/scripts/releases/package.ts +++ b/scripts/releases/package.ts @@ -43,7 +43,37 @@ const build = Effect.gen(function* () { ), ); yield* run("bun", ["run", "apps:build"], root); - yield* run("bun", ["run", "telemetry:build"], root); + // The collector is not built here. A release run's collector job builds it once per target and + // every packaging job downloads those bytes into dist/collector, so every archive of a target + // holds the same executable; a build here could differ with the runner's tools. Locally, + // `bun run collector:build` makes it. + const collector = path.join( + root, + "packages/telemetry/dist/collector", + process.platform === "win32" ? "executor-collector.exe" : "executor-collector", + ); + if (!(yield* fs.exists(collector))) + return yield* Effect.die( + new Error(`${collector} is missing. Run bun run collector:build first.`), + ); + // It must be this target's build, whole: the platform checks and its recorded SHA-256 and size, + // without the Rust toolchain a packaging runner may not have. + const checked = yield* processes.exitCode( + ChildProcess.make( + process.execPath, + [ + path.join(root, "packages/telemetry/scripts/build-collector.ts"), + "--target", + `${target.platform}-${target.arch}`, + "--packaged", + "check", + ], + { cwd: root, stdout: "inherit", stderr: "inherit" }, + ), + ); + if (checked !== 0) + return yield* Effect.die(new Error(`${collector} is not this target's whole build.`)); + yield* run("bun", ["run", "viewer:build"], root); yield* run("bun", ["run", "web:build"], root); yield* fs.makeDirectory(output, { recursive: true }); if (yield* fs.exists(stage)) yield* fs.remove(stage, { recursive: true }); diff --git a/scripts/releases/public-state.ts b/scripts/releases/public-state.ts new file mode 100644 index 0000000000..aa41eb7a88 --- /dev/null +++ b/scripts/releases/public-state.ts @@ -0,0 +1,320 @@ +/** + * Read what a release's public surfaces name now, without credentials: the image's tags on GHCR, + * npm's dist-tags and versions, the Homebrew formula, the desktop update feed and the public GitHub + * release. `channel-guard.ts` reads them before a step moves a channel, and + * `publication-report.ts` after a failed publish job. + * + * Each read is one answer or a `PublicReadFailed`: a surface that names nothing yet (a tag, file or + * release that does not exist, 404) is `undefined`, never a failure. A read that fails, times out + * (15 s, its body included) or answers unreadably is asked again, on one of two schedules: + * + * - `quick` (the default): three more reads 2 s apart. For reads whose failure only warns (the + * preflight's image, tap and feed) or only leaves a row unread (the report). + * - `patient` (`patiently(at)`): four more reads after 5, 10, 20 and 40 s, 75 s of waiting and at + * most about 2.5 minutes when every read runs to its limit. For reads whose failure stops a + * publication: each moving step's own check, in the middle of a publication, and the preflight's + * npm read. A short outage or rate limit then costs a minute instead of a stopped release; one + * that lasts longer stops it loudly, and re-running the failed jobs finishes it, where a channel + * moved backwards would be silent. + * + * A first publication under a new image name cannot be read: GHCR answers 403, not 404, to an + * anonymous token for a repository that does not exist yet, and a new package starts private, + * which answers the same. channel-guard.ts's preflight only warns about it, but the image step's + * own check stops, before anything public has moved. The person doing that first publication + * makes the package public once the `docker` job's push has created it (the package's settings on + * GitHub), then re-runs the failed jobs. + * + * The origins default to the real services. `EXECUTOR_PUBLIC_NPM`, `EXECUTOR_PUBLIC_GHCR`, + * `EXECUTOR_PUBLIC_GITHUB`, `EXECUTOR_PUBLIC_GITHUB_API` and `EXECUTOR_PUBLIC_RAW` replace them; + * e2e/tests/publish-guard.spec.ts points them at one local stub, and shortens the spacing of the + * repeated reads with `EXECUTOR_PUBLIC_RETRY_SECONDS` (2 s). The release workflow sets none. + */ +import { Config, Duration, Effect, Schedule, Schema } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/http"; +import { + compareReleaseVersions, + desktopUpdateFeed, + desktopUpdateFile, + platforms, + type ReleaseChannel, +} from "./config.ts"; + +export class PublicReadFailed extends Schema.TaggedError()("PublicReadFailed", { + message: Schema.String, +}) {} + +export const endpoints = Effect.gen(function* () { + const origin = (name: string, fallback: string) => + Config.String(name).pipe(Config.withDefault(fallback)); + return { + npm: yield* origin("EXECUTOR_PUBLIC_NPM", "https://registry.npmjs.org"), + ghcr: yield* origin("EXECUTOR_PUBLIC_GHCR", "https://ghcr.io"), + github: yield* origin("EXECUTOR_PUBLIC_GITHUB", "https://github.com"), + githubApi: yield* origin("EXECUTOR_PUBLIC_GITHUB_API", "https://api.github.com"), + raw: yield* origin("EXECUTOR_PUBLIC_RAW", "https://raw.githubusercontent.com"), + retrySeconds: yield* Config.Number("EXECUTOR_PUBLIC_RETRY_SECONDS").pipe(Config.withDefault(2)), + patience: "quick" as "quick" | "patient", + }; +}); +export type Endpoints = Effect.Success; + +/** The same endpoints, read on the patient schedule (above). */ +export const patiently = (at: Endpoints): Endpoints => ({ ...at, patience: "patient" }); + +/** + * The repeats after a failed read. `EXECUTOR_PUBLIC_RETRY_SECONDS` (2 s) is the quick spacing; + * the patient waits start at 2.5 times it and double: 5, 10, 20 and 40 s. + */ +const repeats = ( + at: Endpoints, +): { readonly schedule: Schedule.Schedule; readonly times: number } => + at.patience === "patient" + ? { schedule: Schedule.exponential(Duration.seconds(at.retrySeconds * 2.5), 2), times: 4 } + : { schedule: Schedule.spaced(Duration.seconds(at.retrySeconds)), times: 3 }; + +/** A release version this tooling can order: `2.0.0`, `2.0.0-beta.10`, or Executor 1's `1.6.11`. */ +const orderable = /^\d+\.\d+\.\d+(?:-beta\.\d+)?$/u; + +/** + * Whether `current`, what a channel names now, is newer than `version`: true, false, or undefined + * when `current` is not a version this tooling can order. `2.0.0-beta.10` is newer than + * `2.0.0-beta.9`, and `2.0.0` newer than every `2.0.0-beta.N`. + */ +export const newerThan = (current: string, version: string): boolean | undefined => + orderable.test(current) && orderable.test(version) + ? compareReleaseVersions(current, version) > 0 + : undefined; + +/** The Homebrew formula a channel's releases update: a beta never touches the stable one. */ +export const formulaName = (channel: ReleaseChannel) => + channel === "beta" ? "executor-beta" : "executor"; + +/** The `version "…"` line of a Homebrew formula, or undefined when it has none. */ +export const formulaVersion = (formula: string) => /^\s*version "([^"]+)"/mu.exec(formula)?.[1]; + +/** The desktop feed's file names for one channel, one per platform, as electron-updater reads them. */ +export const feedFiles = (channel: ReleaseChannel) => [ + ...new Set(platforms.map((target) => desktopUpdateFile(target, channel))), +]; + +/** One GET, once: the body, or undefined for a 404; anything else but 200 fails. */ +const getOnce = (url: string, headers: Readonly> = {}) => + Effect.gen(function* () { + const http = yield* HttpClient.HttpClient; + const response = yield* http + .execute(HttpClientRequest.get(url).pipe(HttpClientRequest.setHeaders(headers))) + .pipe(Effect.mapError(() => new PublicReadFailed({ message: `${url} did not answer` }))); + if (response.status === 404) { + yield* response.text.pipe(Effect.ignore); + return undefined; + } + const text = yield* response.text.pipe( + Effect.mapError(() => new PublicReadFailed({ message: `${url} answered unreadably` })), + ); + if (response.status !== 200) + return yield* new PublicReadFailed({ message: `${url} answered ${response.status}` }); + return text; + }).pipe( + Effect.timeoutOrElse({ + duration: Duration.seconds(15), + orElse: () => + Effect.fail(new PublicReadFailed({ message: `${url} did not answer within 15 s` })), + }), + ); + +/** One GET, asked again on `at`'s schedule while it fails. */ +const get = (at: Endpoints, url: string, headers: Readonly> = {}) => + getOnce(url, headers).pipe(Effect.retry(repeats(at))); + +const json = (schema: S, text: string, what: string) => + Schema.decodeUnknownEffect(Schema.fromJsonString(schema))(text).pipe( + Effect.mapError(() => new PublicReadFailed({ message: `${what} answered unreadably` })), + ); + +/** + * npm's dist-tags for `name`, or undefined when npm has no such package. An answer without a + * `latest` tag is unreadable, as in npm-channel.ts: npm always has one, so it is an error body, and + * taking it for "names nothing yet" would let a stale release through. It is asked again. + */ +export const npmTags = (at: Endpoints, name: string) => + getOnce(`${at.npm}/-/package/${name}/dist-tags`).pipe( + Effect.flatMap((text) => + text === undefined + ? Effect.succeed(undefined) + : json(Schema.Record(Schema.String, Schema.String), text, "npm's dist-tags").pipe( + Effect.flatMap((tags) => + tags.latest === undefined + ? Effect.fail( + new PublicReadFailed({ + message: `npm's dist-tags for ${name} answered without a latest tag`, + }), + ) + : Effect.succeed(tags), + ), + ), + ), + Effect.retry(repeats(at)), + ); + +/** + * The archive npm serves as `name@version`: its integrity and, when it is deprecated, the + * deprecation message; undefined when npm does not hold that version. + */ +export const npmArchive = (at: Endpoints, name: string, version: string) => + get(at, `${at.npm}/${name}/${version}`).pipe( + Effect.flatMap((text) => + text === undefined + ? Effect.succeed(undefined) + : json( + Schema.Struct({ + dist: Schema.Struct({ integrity: Schema.String }), + deprecated: Schema.optional(Schema.String), + }), + text, + `npm's ${name}@${version}`, + ).pipe( + Effect.map((found) => ({ + integrity: found.dist.integrity, + deprecated: found.deprecated, + })), + ), + ), + ); + +/** When npm says it published `name@version`, from the package's `time`; undefined if it does not say. */ +export const npmPublishedAt = (at: Endpoints, name: string, version: string) => + get(at, `${at.npm}/${name}`).pipe( + Effect.flatMap((text) => + text === undefined + ? Effect.succeed(undefined) + : json( + Schema.Struct({ time: Schema.optional(Schema.Record(Schema.String, Schema.String)) }), + text, + `npm's ${name} document`, + ).pipe(Effect.map((found) => found.time?.[version])), + ), + ); + +const manifestTypes = [ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +].join(", "); + +const Manifest = Schema.Struct({ + manifests: Schema.optional( + Schema.Array( + Schema.Struct({ + digest: Schema.String, + platform: Schema.optional( + Schema.Struct({ os: Schema.String, architecture: Schema.String }), + ), + }), + ), + ), + config: Schema.optional(Schema.Struct({ digest: Schema.String })), +}); +const ImageConfig = Schema.Struct({ + config: Schema.Struct({ + Labels: Schema.optional(Schema.NullOr(Schema.Record(Schema.String, Schema.String))), + }), +}); + +/** + * The version an image tag names, from its Linux image's `org.opencontainers.image.version` label + * (release-artifacts.yml sets it), or undefined when the tag does not exist. `image` is the + * workflow's IMAGE, such as `ghcr.io/usefulsoftwareco/executor-selfhost`. + */ +export const imageVersion = (at: Endpoints, image: string, tag: string) => + Effect.gen(function* () { + const repository = image.slice(image.indexOf("/") + 1); + const token = yield* get(at, `${at.ghcr}/token?scope=repository:${repository}:pull`).pipe( + Effect.flatMap((text) => + text === undefined + ? Effect.fail(new PublicReadFailed({ message: `GHCR gave no token for ${repository}` })) + : json(Schema.Struct({ token: Schema.String }), text, "GHCR's token endpoint"), + ), + ); + const headers = { authorization: `Bearer ${token.token}`, accept: manifestTypes }; + const manifest = (reference: string) => + get(at, `${at.ghcr}/v2/${repository}/manifests/${reference}`, headers).pipe( + Effect.flatMap((text) => + text === undefined + ? Effect.succeed(undefined) + : json(Manifest, text, `${image}:${reference}`), + ), + ); + const top = yield* manifest(tag); + if (top === undefined) return undefined; + const linux = top.manifests?.find((entry) => entry.platform?.os === "linux"); + const single = linux === undefined ? top : yield* manifest(linux.digest); + const config = single?.config?.digest; + if (config === undefined) + return yield* new PublicReadFailed({ message: `${image}:${tag} has no image config` }); + const blob = yield* get(at, `${at.ghcr}/v2/${repository}/blobs/${config}`, headers); + if (blob === undefined) + return yield* new PublicReadFailed({ message: `${image}:${tag}'s config is missing` }); + const labels = (yield* json(ImageConfig, blob, `${image}:${tag}'s config`)).config.Labels; + const version = labels?.["org.opencontainers.image.version"]; + if (version === undefined) + return yield* new PublicReadFailed({ message: `${image}:${tag} has no version label` }); + return version; + }); + +/** The version the tap's formula for `channel` names, or undefined when it has none yet. */ +export const publishedFormulaVersion = (at: Endpoints, tap: string, channel: ReleaseChannel) => + get(at, `${at.raw}/${tap}/HEAD/Formula/${formulaName(channel)}.rb`).pipe( + Effect.map((text) => (text === undefined ? undefined : (formulaVersion(text) ?? "no version"))), + ); + +/** The version each of `channel`'s desktop feed files names; undefined for a file not there yet. */ +export const feedVersions = (at: Endpoints, repository: string, channel: ReleaseChannel) => + Effect.forEach(feedFiles(channel), (name) => + get(at, `${at.github}/${repository}/releases/download/${desktopUpdateFeed.tag}/${name}`).pipe( + Effect.flatMap((text) => + text === undefined + ? Effect.succeed(undefined) + : json(Schema.Struct({ version: Schema.String }), text, name).pipe( + Effect.map((info) => info.version), + ), + ), + Effect.map((version) => ({ name, version })), + ), + ); + +/** + * The public GitHub release for `tag`: whether it is a draft and how many assets it holds, or + * undefined when the token (or, without one, the public) cannot see it. Only a token sees drafts. + */ +export const githubRelease = ( + at: Endpoints, + repository: string, + tag: string, + token: string | undefined, +) => + get( + at, + `${at.githubApi}/repos/${repository}/releases/tags/${encodeURIComponent(tag)}`, + token === undefined || token === "" ? {} : { authorization: `Bearer ${token}` }, + ).pipe( + Effect.flatMap((text) => + text === undefined + ? Effect.succeed(undefined) + : json( + Schema.Struct({ + draft: Schema.Boolean, + assets: Schema.Array(Schema.Unknown), + target_commitish: Schema.optional(Schema.String), + }), + text, + `the release ${tag}`, + ).pipe( + Effect.map((found) => ({ + draft: found.draft, + assets: found.assets.length, + target: found.target_commitish, + })), + ), + ), + ); diff --git a/scripts/releases/publication-report.ts b/scripts/releases/publication-report.ts new file mode 100644 index 0000000000..4388b34f0b --- /dev/null +++ b/scripts/releases/publication-report.ts @@ -0,0 +1,297 @@ +/** + * After a publish job failed, say what this release made public and what it did not, from the + * public surfaces themselves rather than from which steps passed: beta.51's npm step failed while + * npm already served the launcher with its tag. Writes a table to the job summary and the log: + * + * - the public GitHub release: public or draft, and its assets; + * - the image's version tag and its channel tag; + * - npm's version and its channel tag (and `latest`, for a beta, which it must not have moved); + * - the Homebrew formula for the channel, when the workflow has a tap; + * - each desktop feed file for the channel; + * + * and one line on whether `gh run rerun --failed` would finish the publication. + * + * It never fails: a surface it cannot read is a row that says so, and anything else unexpected is + * printed. The step runs only after a failure, so the job keeps that failure's conclusion. Reads + * VERSION, CHANNEL, IMAGE, PUBLIC_REPOSITORY, RELEASE_TAG, and optionally TAP, GH_TOKEN (to see a + * draft), GITHUB_RUN_ID and GITHUB_STEP_SUMMARY. Changes nothing anywhere (public-state.ts). + */ +import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { createHash } from "node:crypto"; +import { Cause, Config, Console, Duration, Effect, FileSystem, Option } from "effect"; +import { FetchHttpClient, type HttpClient } from "effect/http"; +import type { ReleaseChannel } from "./config.ts"; +import { + endpoints, + feedVersions, + formulaName, + githubRelease, + imageVersion, + newerThan, + npmArchive, + npmTags, + publishedFormulaVersion, + type PublicReadFailed, +} from "./public-state.ts"; + +type Row = { + readonly surface: string; + readonly now: string; + /** Done for this release; undefined when it could not be read. */ + readonly done: boolean | undefined; + /** What the surface names, when it is a channel, to tell a newer release from this one. */ + readonly names?: string | undefined; + /** npm holds this version with an archive other than this run's launcher. */ + readonly stray?: boolean; +}; + +const unread = + (surface: string) => + (error: PublicReadFailed): Effect.Effect => + Effect.succeed({ surface, now: `could not read: ${error.message}`, done: undefined }); + +const none: Effect.Effect> = Effect.succeed([]); + +const program = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const version = yield* Config.String("VERSION"); + const channel: ReleaseChannel = + (yield* Config.String("CHANNEL")) === "latest" ? "latest" : "beta"; + const image = yield* Config.String("IMAGE"); + const repository = yield* Config.String("PUBLIC_REPOSITORY"); + const tag = yield* Config.String("RELEASE_TAG"); + const tap = yield* Config.String("TAP").pipe(Config.withDefault("")); + const token = yield* Config.String("GH_TOKEN").pipe(Config.withDefault("")); + const run = yield* Config.String("GITHUB_RUN_ID").pipe(Config.withDefault("")); + const summary = yield* Config.String("GITHUB_STEP_SUMMARY").pipe(Config.option); + const at = yield* endpoints; + // The launcher this run built, when release:wrapper ran: npm's archive is compared with it. + const launcher = yield* Config.String("LAUNCHER").pipe( + Config.withDefault(`.local/releases/${version}/wrapper/executor-${version}.tgz`), + ); + const built = (yield* fs.exists(launcher).pipe(Effect.orElseSucceed(() => false))) + ? yield* fs.readFile(launcher).pipe( + Effect.map((bytes) => `sha512-${createHash("sha512").update(bytes).digest("base64")}`), + Effect.orElseSucceed(() => undefined), + ) + : undefined; + + const channelRow = (surface: string, names: string | undefined): Row => ({ + surface, + now: names ?? "nothing", + done: names === version, + names, + }); + + const feedRows = (feedChannel: ReleaseChannel, follows: boolean) => + feedVersions(at, repository, feedChannel).pipe( + Effect.map((files) => + files.map((file): Row => + follows + ? // A stable release also moves beta installs forward, but desktop-feed.ts keeps a + // newer beta there: either is done, and neither tells of a newer release. + { + surface: `desktop feed ${file.name} (beta installs follow)`, + now: file.version ?? "nothing", + done: + file.version !== undefined && + (file.version === version || newerThan(file.version, version) === true), + } + : channelRow(`desktop feed ${file.name}`, file.version), + ), + ), + Effect.catchTag("PublicReadFailed", (error) => + Effect.succeed([ + { + surface: `desktop feed (${feedChannel})`, + now: `could not read: ${error.message}`, + done: undefined, + } satisfies Row, + ]), + ), + ); + + // Every surface at once, so the step's time limit is spent on the slowest read, not their sum, + // and each group of reads within its own limit (150 s, inside the step's three minutes), so the + // table is always written: a group that has not answered by then is a row that says so. One + // group can take longer than the step otherwise: the image's four reads, or the feed's four + // files, each read up to four times for 15 s. The rows keep this order. + const groupSeconds = yield* Config.Number("EXECUTOR_PUBLIC_REPORT_GROUP_SECONDS").pipe( + Config.withDefault(150), + ); + const bounded = ( + label: string, + read: Effect.Effect, never, HttpClient.HttpClient>, + ): Effect.Effect, never, HttpClient.HttpClient> => + read.pipe( + Effect.timeoutOrElse({ + duration: Duration.seconds(groupSeconds), + orElse: () => + Effect.succeed([ + { + surface: label, + now: `could not read: no answer within ${groupSeconds} s`, + done: undefined, + } satisfies Row, + ]), + }), + ); + const reads: ReadonlyArray, never, HttpClient.HttpClient>> = [ + bounded( + `GitHub release ${tag}`, + githubRelease(at, repository, tag, token).pipe( + Effect.map((found): ReadonlyArray => [ + found === undefined + ? { + surface: `GitHub release ${tag}`, + now: token === "" ? "not public (a draft or absent)" : "absent", + done: false, + } + : { + surface: `GitHub release ${tag}`, + now: `${found.draft ? "draft" : "public"}, ${found.assets} assets`, + done: !found.draft, + }, + ]), + Effect.catchTag("PublicReadFailed", (error) => + unread(`GitHub release ${tag}`)(error).pipe(Effect.map((row) => [row])), + ), + ), + ), + bounded( + `${image}:${version}`, + imageVersion(at, image, version).pipe( + Effect.map((found): ReadonlyArray => [ + { + surface: `${image}:${version}`, + now: found === undefined ? "absent" : `present (${found})`, + done: found === version, + }, + ]), + Effect.catchTag("PublicReadFailed", (error) => + unread(`${image}:${version}`)(error).pipe(Effect.map((row) => [row])), + ), + ), + ), + bounded( + `${image}:${channel}`, + imageVersion(at, image, channel).pipe( + Effect.map((names): ReadonlyArray => [channelRow(`${image}:${channel}`, names)]), + Effect.catchTag("PublicReadFailed", (error) => + unread(`${image}:${channel}`)(error).pipe(Effect.map((row) => [row])), + ), + ), + ), + bounded( + `npm executor@${version}`, + npmArchive(at, "executor", version).pipe( + Effect.flatMap((archive) => + Effect.gen(function* () { + const row = (now: string, done: boolean, stray = false): ReadonlyArray => [ + { surface: `npm executor@${version}`, now, done, ...(stray ? { stray } : {}) }, + ]; + if (archive === undefined) return row("absent", false); + if (built !== undefined) + return archive.integrity === built + ? row("published, this run's launcher", true) + : row("npm serves a different archive under this version", false, true); + // No launcher in this run, as when the preflight refused the version: npm's archive is + // this publication's only if the release for the tag is published (npm gets the + // launcher after that). Without it, npm's archive came from elsewhere. + const release = yield* githubRelease(at, repository, tag, token); + return release === undefined || release.draft + ? row( + `npm serves an archive this publication did not put there (no published release ${tag})`, + false, + true, + ) + : row("published (not compared: this run built no launcher)", true); + }), + ), + Effect.catchTag("PublicReadFailed", (error) => + unread(`npm executor@${version}`)(error).pipe(Effect.map((row) => [row])), + ), + ), + ), + bounded( + `npm tag ${channel}`, + npmTags(at, "executor").pipe( + Effect.map((tags): ReadonlyArray => [ + channelRow(`npm tag ${channel}`, tags?.[channel]), + ...(channel === "beta" + ? [ + { + surface: "npm tag latest (a beta leaves it)", + now: tags?.latest ?? "nothing", + done: tags?.latest !== version, + } satisfies Row, + ] + : []), + ]), + Effect.catchTag("PublicReadFailed", (error) => + unread(`npm tag ${channel}`)(error).pipe(Effect.map((row) => [row])), + ), + ), + ), + tap === "" + ? none + : bounded( + `Homebrew ${formulaName(channel)}`, + publishedFormulaVersion(at, tap, channel).pipe( + Effect.map((names): ReadonlyArray => [ + channelRow(`Homebrew ${formulaName(channel)}`, names), + ]), + Effect.catchTag("PublicReadFailed", (error) => + unread(`Homebrew ${formulaName(channel)}`)(error).pipe(Effect.map((row) => [row])), + ), + ), + ), + bounded(`desktop feed (${channel})`, feedRows(channel, false)), + channel === "latest" + ? bounded("desktop feed (beta installs follow)", feedRows("beta", true)) + : none, + ]; + const groups = yield* Effect.all(reads, { concurrency: "unbounded" }); + const rows = groups.flat(); + + const newer = rows.filter( + (row) => row.names !== undefined && newerThan(row.names, version) === true, + ); + const stray = rows.some((row) => row.stray === true); + const verdict = stray + ? `Re-running cannot finish ${version}: npm already serves executor@${version} with an archive this publication did not build, and npm never accepts a version twice. This release needs another version number.` + : newer.length > 0 + ? `Re-running would not finish ${version}: ${newer.map((row) => `${row.surface} names ${row.names}`).join(", ")}, newer than it. Leave this release; the channel guards stop any step that would move a channel back.` + : rows.some((row) => row.done === undefined) + ? `Some surfaces could not be read, so this cannot say whether \`gh run rerun ${run} --failed\` would finish ${version}. Read them by hand first.` + : rows.every((row) => row.done === true) + ? `Everything ${version} publishes is public. Nothing is left to finish; the failure came after the publication.` + : `\`gh run rerun ${run} --failed\` should finish ${version}: a re-run resumes over what is public, and the steps whose work is done skip it or repeat it unchanged. It stops only where publish-npm.ts finds a different archive on npm, or a newer release has moved a channel since.`; + + const table = [ + `### What ${version} published`, + "", + "| Surface | Now | Done |", + "| --- | --- | --- |", + ...rows.map( + (row) => + `| ${row.surface} | ${row.now.replaceAll("|", "\\|")} | ${row.done === undefined ? "?" : row.done ? "done" : "not done"} |`, + ), + "", + verdict, + "", + ].join("\n"); + yield* Console.log(table); + if (Option.isSome(summary)) + yield* fs.writeFileString(summary.value, table, { flag: "a" }).pipe(Effect.ignore); +}); + +NodeRuntime.runMain( + program.pipe( + Effect.catchCause((cause) => + Console.log(`The publication report could not be completed:\n${Cause.pretty(cause)}`), + ), + Effect.provide([NodeServices.layer, FetchHttpClient.layer]), + ), +); diff --git a/scripts/releases/publish-npm.ts b/scripts/releases/publish-npm.ts index 3171628c74..5389165bda 100644 --- a/scripts/releases/publish-npm.ts +++ b/scripts/releases/publish-npm.ts @@ -4,7 +4,8 @@ * package.json records. So this refuses a launcher that does not record every platform's archive * in the artifact directory, or whose archives the public release does not list in its * SHA256SUMS. An accepted upload is polled, never uploaded again; a retried job skips a launcher - * whose registry integrity matches byte for byte. + * whose registry integrity matches byte for byte. npm moves the channel's dist-tag after it serves + * the version, so the tag is then awaited for a bounded time (npm-channel.ts). */ import { createHash } from "node:crypto"; import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; @@ -23,12 +24,12 @@ import { import { FetchHttpClient, HttpClient, HttpClientResponse } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { platformArchive, platforms, release } from "./config.ts"; +import { awaitNpmChannel, distTagsBeforePublishing, npmRegistry } from "./npm-channel.ts"; const RegistryVersion = Schema.Struct({ version: Schema.String, dist: Schema.Struct({ integrity: Schema.String }), }); -const Tags = Schema.Record(Schema.String, Schema.String); const Launcher = Schema.fromJsonString( Schema.Struct({ version: Schema.String, @@ -111,12 +112,10 @@ NodeRuntime.runMain( ); } - const registry = "https://registry.npmjs.org"; - const tags = http.get(`${registry}/-/package/executor/dist-tags`).pipe( - Effect.flatMap((response) => response.json), - Effect.flatMap(Schema.decodeUnknownEffect(Tags)), - ); - const before = yield* tags; + const registry = npmRegistry; + // A status other than 200, an answer without the tags, or none within the limit stops here, + // before anything is published (npm-channel.ts). + const before = yield* distTagsBeforePublishing(registry, "executor"); const integrity = `sha512-${(yield* digest(file, "sha512")).digest("base64")}`; // A retried job resumes only over a launcher npm already holds byte for byte. const existing = yield* http.get(`${registry}/executor/${release.version}`); @@ -172,12 +171,14 @@ NodeRuntime.runMain( Effect.retry({ schedule: Schedule.spaced(15_000), times: 160 }), ); yield* Console.log(`Verified public npm archive ${release.version}`); - const after = yield* tags; - if ( - after[release.channel] !== release.version || - (release.channel === "beta" && after.latest !== before.latest) - ) - return yield* Effect.die(new Error("The npm channel check failed.")); + // npm moves the dist-tag some time after it serves the version (npm-channel.ts). + const after = yield* awaitNpmChannel({ + registry, + packageName: "executor", + channel: release.channel, + version: release.version, + before, + }); yield* Console.log( `Published executor@${release.channel}: ${release.version}; latest=${after.latest}`, ); diff --git a/scripts/releases/runtime-bundle.ts b/scripts/releases/runtime-bundle.ts index b61d543908..7a009ae100 100644 --- a/scripts/releases/runtime-bundle.ts +++ b/scripts/releases/runtime-bundle.ts @@ -497,15 +497,28 @@ else process.exitCode = result.status ?? 1; path.join(root, "apps/local/web/dist/client"), path.join(stage, "apps/local/web/dist/client"), ); - yield* fs.copy( - path.join(root, "packages/telemetry/dist/motel"), - path.join(stage, "packages/telemetry/dist/motel"), - ); + // The telemetry collector and the viewer it serves, where packages/telemetry/src/local.ts finds + // them. The collector's build for this target is in dist/collector (`collector:build`, or the + // release run's collector job, whose download drops the executable bit). It is made executable + // here, so the packages can run it; the macOS signer signs every Mach-O file and gives those + // with an execute bit the executable entitlements. Their notices join the install's own below. + const telemetry = path.join(root, "packages/telemetry/dist"); + const collectorName = + process.platform === "win32" ? "executor-collector.exe" : "executor-collector"; + const stagedCollector = path.join(stage, "packages/telemetry/dist/collector", collectorName); + yield* fs.makeDirectory(path.dirname(stagedCollector), { recursive: true }); + yield* fs.copyFile(path.join(telemetry, "collector", collectorName), stagedCollector); + if (process.platform !== "win32") yield* fs.chmod(stagedCollector, 0o755); + for (const file of ["index.html", "assets"]) + yield* fs.copy( + path.join(telemetry, "viewer", file), + path.join(stage, "packages/telemetry/dist/viewer", file), + ); // Keep build diagnostics beside the artifact, without making every npm user // download external source maps. These files are not executable runtime assets. for (const [directory, name] of [ [output, "runtime"], - [path.join(stage, "packages/telemetry/dist/motel"), "motel"], + [path.join(stage, "packages/telemetry/dist/viewer"), "viewer"], ] as const) { for (const file of yield* fs.readDirectory(directory, { recursive: true })) { if (!file.endsWith(".map")) continue; @@ -584,6 +597,28 @@ else process.exitCode = result.status ?? 1; overwrite: true, }); } + // The collector's notices: Motel's licence, whose behaviour it follows, the licence files of + // every crate linked into it and the Rust standard library's; its crates as `cargo:name@version`. + yield* fs.copy( + path.join(telemetry, "collector/NOTICE"), + path.join(stage, "licenses/executor-collector/NOTICE"), + ); + yield* fs.copy(path.join(telemetry, "collector/licenses"), path.join(stage, "licenses"), { + overwrite: true, + }); + inventory.push("executor-collector"); + for (const line of (yield* fs.readFileString( + path.join(telemetry, "collector/cargo-packages.txt"), + )).split("\n")) + if (line !== "") inventory.push(`cargo:${line}`); + // The viewer's: the packages its build bundled, in this same layout. + yield* fs.copy(path.join(telemetry, "viewer/licenses"), path.join(stage, "licenses"), { + overwrite: true, + }); + for (const line of (yield* fs.readFileString( + path.join(telemetry, "viewer/runtime-packages.txt"), + )).split("\n")) + if (line !== "") inventory.push(line); yield* fs.writeFileString( path.join(stage, "runtime-packages.txt"), [...new Set(inventory)].sort().join("\n") + "\n", diff --git a/scripts/releases/standalone.ts b/scripts/releases/standalone.ts index 6b10b020f0..7b24db287d 100644 --- a/scripts/releases/standalone.ts +++ b/scripts/releases/standalone.ts @@ -1,15 +1,17 @@ /** * Turn the packaged runtime into the release install: one `executor` executable that carries the * runtime's code, and beside it the files it runs or reads from disk. Those are workerd, Git, the - * native bindings, PGlite, TypeScript, the dashboard and Motel. npm pack archives this directory as - * the platform's release asset: the download the `executor` npm launcher, the install scripts and - * Homebrew use. + * native bindings, PGlite, TypeScript, the dashboard and the telemetry collector with its viewer. + * npm pack archives this directory as the platform's release asset: the download the `executor` + * npm launcher, the install scripts and Homebrew use. * * Layout, relative to the executable: * - `runtime/node_modules/`: packages loaded from disk (see `installedPackages`) and resources * read through package resolution, such as the Executor app template. * - `runtime/git/`: the bundled Git. - * - `apps/local/web/dist/client/`, `packages/telemetry/dist/motel/`: static resources. + * - `apps/local/web/dist/client/`, `packages/telemetry/dist/viewer/`: static resources. + * - `packages/telemetry/dist/collector/`: the `executor-collector` executable, executable before + * the install is signed. * - `licenses/`, `runtime-packages.txt`: notices for everything bundled or copied. */ import { Config, Console, Effect, FileSystem, Option, Path } from "effect"; @@ -59,6 +61,50 @@ const bunRuntime = Effect.gen(function* () { * Build the release install from `stage`, the packaged runtime, into `/standalone`, and * pack it as the platform's release archive in `output`. Returns the install directory. */ +/** + * Starts every `executor-collector` under `directory` (a package as it ships, signed where it is + * signed) with `build-collector.ts smoke`: a packaged collector that cannot start fails the job + * that packaged it, on every target. A collector that fails to start only logs a warning in the + * product, so nothing else would turn a job red. Signing rewrites a Mach-O file, so a signed copy is + * not compared with the build's digest; `package.ts` checks the build before it is packaged. + */ +export const smokePackagedCollector = (root: string, target: Platform, directory: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const name = target.platform === "win32" ? "executor-collector.exe" : "executor-collector"; + const found: Array = []; + for (const file of yield* fs.readDirectory(directory, { recursive: true })) + if ( + path.basename(file) === name && + (yield* fs.stat(path.join(directory, file))).type === "File" + ) + found.push(path.join(directory, file)); + if (found.length === 0) return yield* Effect.die(new Error(`${directory} holds no ${name}`)); + for (const file of found) + for (const step of [["smoke"]]) { + const code = yield* processes.exitCode( + ChildProcess.make( + process.execPath, + [ + path.join(root, "packages/telemetry/scripts/build-collector.ts"), + "--target", + `${target.platform}-${target.arch}`, + "--executable", + file, + ...step, + ], + { cwd: root, stdout: "inherit", stderr: "inherit" }, + ), + ); + if (code !== 0) + return yield* Effect.die( + new Error(`The packaged collector ${file} failed ${step.join(" ")} (exit ${code})`), + ); + } + }); + /** The file beside a platform's archive that holds the digest of its install before signing. */ export const payloadDigestFile = "payload.sha256"; @@ -188,6 +234,8 @@ export const buildStandalone = ( notarize: signing === "notarize", ...(submission === undefined ? {} : { submission }), }); + // The collector as the install carries it, signed where it is signed, starts and stores telemetry. + yield* smokePackagedCollector(root, target, install); yield* fs.writeFileString( path.join(install, "package.json"),