diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b45a4fb..bd90ab4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,9 @@ jobs: steps: - uses: actions/checkout@v7 - uses: cachix/install-nix-action@v31 + - name: Allow the sandbox launcher's unprivileged user namespaces + if: runner.os == 'Linux' + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: dprint from the flake run: echo "$(nix build --no-link --print-out-paths .#dprint)/bin" >> "$GITHUB_PATH" # pnpm/setup replaces pnpm/action-setup for pnpm v11+ and folds diff --git a/docs/plans/2026-10-05-2151-feat-ratstack-scorecard-plan.md b/docs/plans/2026-10-05-2151-feat-ratstack-scorecard-plan.md index 8743ca2..366b008 100644 --- a/docs/plans/2026-10-05-2151-feat-ratstack-scorecard-plan.md +++ b/docs/plans/2026-10-05-2151-feat-ratstack-scorecard-plan.md @@ -155,6 +155,7 @@ Local rows run each side with its documented local defaults plus throwaway value - KTD14. **Readiness calls URL Scanner v2 with `agentReadiness: true`, then polls the result.** It submits `POST /accounts/{account_id}/urlscanner/v2/scan` with `agentReadiness: true` and polls `GET …/v2/result/{scan_id}` every 15 s. The first live call fixes where the option sits in the request (top level or `options`) and the result path. The cross-check posts `{ "url": … }` to `https://isitagentready.com/api/scan` and reads `level`, as rat-stack's `apps/mischief/scripts/smoke.sh` does. When the levels differ, the row is flagged. This implements the scanner Key Decision (session-settled: user-directed — chosen over isitagentready as primary: first-party reproducible JSON through our token). - KTD15. **Tests are admitted by layer, with refusal as the default** (`skill://test-layer-selection`). Every decision lives in a `*.workflow.ts` module with a colocated `*.workflow.property.test.ts`, mutated at break 100 by sfs stryker-js in CI on push to `main` (D5), never locally. Family runners, harness, side adapters and REST clients are executors and adapters: they get no tests of their own, and each unit's family smoke run plus its PR sabotage proves them. Three process-isolated journeys under `evals/ratstack-scorecard/journeys/` observe what only the seam can see: J1 launcher isolation (U4), J2 the static family over a fixture tree through the real parser in the sandbox (U2), and J3 the CLI's exit status and schema-valid JSON on a ratchet failure (U3). Refused: tests of `registry.ts` (a declaration), unit tests of runners (they would spawn processes or mock the subject), and assertions on rendered Markdown wording. - KTD16. **A mutated gate is graded only against a green baseline.** Every family that edits a clone and runs a gate (M13, M20, M22, M24, M25) first runs the unmodified gate on that side over warm caches, and a red baseline makes those cells `unmeasurable` with the first failing task. The cold path (M26) is its own baseline: a red cold gate makes M26 `unmeasurable` and leaves the other families alone. This keeps a kill rate or removal rate from being computed over a gate that was already failing. +- KTD17. **Journeys that need the launcher run in two phases; only the host driver starts the launcher** (Kiro ruling, 2026-10-06). The launcher cannot nest: `bwrap --unshare-all` inside the prm#5 sandbox (`1801228`, bwrap 0.12.0) fails with `bwrap: setting up uid map: Operation not permitted`, and loosening the launcher is ruled out. So `scorecard journeys` (the zero-third-party Deno driver) first produces each journey declared as data in `journeys/manifest.json`, running the real launcher work (for example `measureStatic` end to end over two fixture git repos built from `journeys/__fixtures__/repos/`), and writes one record per journey to `journeys/__records__/.json`: argv, exit code, stdout, stderr, output files, egress log, wall time, and an input hash over the declared inputs plus the launcher store path. It then runs the single vitest project inside the launcher. A test is a journey because it imports `journeys/launcher-run.ts`, whose `launcherRun(id)` decodes the record with Effect Schema and fails red with `MissingLauncherRecord` or `StaleLauncherRecord`, never a skip (CONST-T12). `scorecard check` (in the root `check:ci`) first runs the import-graph rules (the driver's graph from `src/main.ts` loads no third-party module; no file under `src/` imports the journey fixture), then `scorecard journeys`. ### High-Level Technical Design diff --git a/evals/ratstack-scorecard/.gitignore b/evals/ratstack-scorecard/.gitignore index ceddaa3..596e435 100644 --- a/evals/ratstack-scorecard/.gitignore +++ b/evals/ratstack-scorecard/.gitignore @@ -1 +1,2 @@ .cache/ +journeys/__records__/ diff --git a/evals/ratstack-scorecard/README.md b/evals/ratstack-scorecard/README.md index e701639..fb3caae 100644 --- a/evals/ratstack-scorecard/README.md +++ b/evals/ratstack-scorecard/README.md @@ -19,7 +19,16 @@ The scorecard measures the starter against [rat-stack](https://github.com/joelho ## Running it -Everything that loads third-party code runs inside the sandbox launcher from `systemfsoftware/pnpm-release-management` (`packages..sandbox`), with this directory as the sandbox project. The instrument's own flake (`flake.nix`) pins nixpkgs (pnpm 12.9.0, Node 24) and the launcher, and builds the tools' pnpm store from `pnpm-lock.yaml` as a fixed-output derivation (`tools-store`). The install is offline from that store; the sandbox gets no network at all. +Everything that loads third-party code runs inside the sandbox launcher from `systemfsoftware/pnpm-release-management` (`packages..sandbox`). + +The instrument has its own flake (`flake.nix`, pinned nixpkgs and launcher). Its `scorecard` package bundles Deno, the launcher, the rat-stack source at `ratstack.pin.json`, and the offline pnpm store for the tools (`tools-store`). The family runners install the tools from that store inside the sandbox, so nothing reaches the registry at measurement time. + +```sh +scorecard=$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#scorecard) +$scorecard/bin/scorecard measure --family static --out static.json +``` + +The instrument's own tests run inside the launcher against the same offline store: ```sh cd evals/ratstack-scorecard @@ -27,4 +36,4 @@ nix develop --command sh -c 'SANDBOX_PROJECT=$PWD sandbox --pnpm-store "$SANDBOX nix develop --command sh -c 'SANDBOX_PROJECT=$PWD sandbox -- pnpm vitest run' ``` -The decision modules (`src/model/*.workflow.ts`) and the orchestrator import only Deno APIs, `node:` builtins and each other, so `deno check src/` type-checks them without any third-party code. +`src/main.ts` and everything it imports use only Deno APIs, `node:` builtins and each other, so `DENO_NO_PACKAGE_JSON=1 deno check src/` type-checks the orchestrator and the decision core without third-party code. The Node scripts in `src/tools/` are the only code that loads npm packages, and they only ever run inside the launcher. diff --git a/evals/ratstack-scorecard/deno.json b/evals/ratstack-scorecard/deno.json index 0b79998..927072f 100644 --- a/evals/ratstack-scorecard/deno.json +++ b/evals/ratstack-scorecard/deno.json @@ -4,6 +4,6 @@ "exactOptionalPropertyTypes": true, "noUncheckedIndexedAccess": true }, - "exclude": ["node_modules/", "**/*.test.ts", "**/*.arbitrary.ts", "vitest.config.ts"], + "exclude": ["node_modules/", "src/tools/", "journeys/", "**/*.test.ts", "**/*.arbitrary.ts", "vitest.config.ts"], "lock": false } diff --git a/evals/ratstack-scorecard/flake.lock b/evals/ratstack-scorecard/flake.lock index c464b2c..244be87 100644 --- a/evals/ratstack-scorecard/flake.lock +++ b/evals/ratstack-scorecard/flake.lock @@ -9,11 +9,11 @@ "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1788142907, - "narHash": "sha256-arhhsgBbDTOTtnG9gphKARLXkCj3emOZUm3sP0a0jnI=", + "lastModified": 1790884787, + "narHash": "sha256-XZz0bdSY5zjlRDme0dfC98oYcAf3WOpMCueoU39iXjY=", "owner": "systemfsoftware", "repo": "comment-checker", - "rev": "60e60bbedb2686c807a9ca64a31f7bba4e75b97b", + "rev": "3bfedd853c1d6dca666e65e71e9f8a31d47db336", "type": "github" }, "original": { @@ -46,17 +46,17 @@ ] }, "locked": { - "lastModified": 1791237748, - "narHash": "sha256-IWdNptNMXzjs2yG3DRQJKUkRcOmVvyDQ0x0dN2maEVU=", + "lastModified": 1791289049, + "narHash": "sha256-EaB7hYyD684/oCH9VgDiHcJw2vAuLf8InlZGFPxN2hE=", "owner": "systemfsoftware", "repo": "pnpm-release-management", - "rev": "180122866dd537fa728b5563fb1820fbd2af88cc", + "rev": "8f1984418fef130956a3d1f50dc471fd1984d2ec", "type": "github" }, "original": { "owner": "systemfsoftware", "repo": "pnpm-release-management", - "rev": "180122866dd537fa728b5563fb1820fbd2af88cc", + "rev": "8f1984418fef130956a3d1f50dc471fd1984d2ec", "type": "github" } }, @@ -75,11 +75,11 @@ ] }, "locked": { - "lastModified": 1787367552, - "narHash": "sha256-YT4Fs2k7bi+7YzuLt93EtIRgjpwHK5ZfsQEIh5dEQSk=", + "lastModified": 1790756160, + "narHash": "sha256-NPs4nNLxCpho3Ctj8O7t/bzmX0zR8r0zndKghAFEz5U=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "fd2ebb9cc4323d0c5a1336138dab5c3c5a5d8bd9", + "rev": "ed3a19fd0439ed618ec5fe1e12f0ba69a8be38b5", "type": "github" }, "original": { diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index f54425e..f294a97 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -4,7 +4,7 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/494ce7fd23ff6a5dff39e1fb11e9b6f2ac74bf25"; pnpm-release-management = { - url = "github:systemfsoftware/pnpm-release-management/180122866dd537fa728b5563fb1820fbd2af88cc"; + url = "github:systemfsoftware/pnpm-release-management/8f1984418fef130956a3d1f50dc471fd1984d2ec"; inputs.nixpkgs.follows = "nixpkgs"; }; }; @@ -13,6 +13,12 @@ let systems = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ]; forEachSystem = fn: nixpkgs.lib.genAttrs systems (system: fn nixpkgs.legacyPackages.${system}); + pin = builtins.fromJSON (builtins.readFile ./ratstack.pin.json); + toolsStoreHash = { + x86_64-linux = "sha256-jUyIV2ysOxkIij/Ff5VGuIADTibbY0SsCPGglmIvdn0="; + aarch64-linux = "sha256-jUyIV2ysOxkIij/Ff5VGuIADTibbY0SsCPGglmIvdn0="; + aarch64-darwin = "sha256-XWYBTy3xZbQwJxdGbAK4+R69XNmL3RRyXu6+nY+XPXc="; + }; in { packages = forEachSystem (pkgs: @@ -24,11 +30,34 @@ src = self; pname = "ratstack-scorecard"; pnpm = pkgs.pnpm_12; - hash = "sha256-TylxLEQflTlKx6QDk9mFlBOEInUvvBBeOVFUYCURmYo="; + hash = toolsStoreHash.${system}; }).pnpm-store; + ratstack-src = pkgs.fetchFromGitHub { + inherit (pin) owner repo; + rev = pin.commit; + hash = pin.narHash; + }; + scorecard = pkgs.writeShellApplication { + name = "scorecard"; + runtimeInputs = [ pkgs.deno pkgs.git ]; + text = '' + export SCORECARD_INSTRUMENT=${self} + export SCORECARD_SANDBOX=${sandbox}/bin/sandbox + export SCORECARD_TOOLS_STORE=${tools-store} + export SCORECARD_RATSTACK_SRC=${ratstack-src} + export SCORECARD_TOOL_PATH=${pkgs.lib.makeBinPath [ pkgs.nodejs_24 pkgs.pnpm_12 pkgs.coreutils ]} + export SCORECARD_NODE_VERSION=${pkgs.nodejs_24.version} + export SCORECARD_PNPM_VERSION=${pkgs.pnpm_12.version} + export DENO_NO_PACKAGE_JSON=1 + exec deno run --no-config --allow-read --allow-write --allow-env --allow-sys=hostname \ + --allow-run=git,${sandbox}/bin/sandbox \ + ${self}/src/main.ts "$@" + ''; + }; in { - inherit sandbox tools-store; + inherit sandbox tools-store ratstack-src scorecard; + default = scorecard; }); devShells = forEachSystem (pkgs: diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm11.yaml b/evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm11.yaml new file mode 100644 index 0000000..16c761c --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm11.yaml @@ -0,0 +1,73 @@ +--- +lockfileVersion: "9.0" + +importers: + .: + configDependencies: {} + packageManagerDependencies: + "@pnpm/exe": + specifier: 11.3.0 + version: 11.3.0 + pnpm: + specifier: 11.3.0 + version: 11.3.0 + +packages: + "@pnpm/exe@11.3.0": + resolution: { + integrity: sha512-1ItrG3GdA8HC7IUMy79SmYqynjjfwXtIMlbpx9MzrU3ZXMYMfw3yuwjIXW7Aw2z0rRxxa2KtTYcLxqjIaVjUmg==, + } + hasBin: true + + pnpm@11.3.0: + resolution: { integrity: sha512-AAAA } + hasBin: true + +snapshots: + "@pnpm/exe@11.3.0": {} + + pnpm@11.3.0: {} + +--- +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + .: + dependencies: + effect: + specifier: 4.0.0 + version: 4.0.0 + devDependencies: + "@effect/vitest": + specifier: 4.0.0 + version: 4.0.0(effect@4.0.0)(vitest@5.0.3) + vitest: + specifier: 5.0.3 + version: 5.0.3 + +packages: + "@effect/vitest@4.0.0": + resolution: { integrity: sha512-BBBB } + peerDependencies: + effect: ^4.0.0 + vitest: ^5.0.0 + + effect@4.0.0: + resolution: { integrity: sha512-CCCC } + + vitest@5.0.3: + resolution: { integrity: sha512-DDDD } + +snapshots: + "@effect/vitest@4.0.0(effect@4.0.0)(vitest@5.0.3)": + dependencies: + effect: 4.0.0 + vitest: 5.0.3 + + effect@4.0.0: {} + + vitest@5.0.3: {} diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm12.yaml b/evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm12.yaml new file mode 100644 index 0000000..df58f0f --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm12.yaml @@ -0,0 +1,78 @@ +--- +lockfileVersion: "9.0" + +importers: + .: + configDependencies: {} + packageManagerDependencies: + pnpm: + specifier: 12.4.2 + version: 12.4.2 + +packages: + "@pnpm/exe.linux-x64@12.4.2": + resolution: { integrity: sha512-EEEE } + cpu: [x64] + os: [linux] + + pnpm@12.4.2: + resolution: { integrity: sha512-FFFF } + hasBin: true + +snapshots: + "@pnpm/exe.linux-x64@12.4.2": + optional: true + + pnpm@12.4.2: + optionalDependencies: + "@pnpm/exe.linux-x64": 12.4.2 + +--- +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +catalogs: + default: + effect: + specifier: ^4.0.0 + version: 4.0.0 + +importers: + .: + dependencies: + effect: + specifier: "catalog:" + version: 4.0.0 + devDependencies: + "@effect/vitest": + specifier: 4.0.0 + version: 4.0.0(effect@4.0.0)(vitest@5.0.3) + vitest: + specifier: 5.0.3 + version: 5.0.3 + +packages: + "@effect/vitest@4.0.0": + resolution: { integrity: sha512-BBBB } + peerDependencies: + effect: ^4.0.0 + vitest: ^5.0.0 + + effect@4.0.0: + resolution: { integrity: sha512-CCCC } + + vitest@5.0.3: + resolution: { integrity: sha512-DDDD } + +snapshots: + "@effect/vitest@4.0.0(effect@4.0.0)(vitest@5.0.3)": + dependencies: + effect: 4.0.0 + vitest: 5.0.3 + + effect@4.0.0: {} + + vitest@5.0.3: {} diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/pnpm-lock.yaml b/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/pnpm-lock.yaml new file mode 100644 index 0000000..16c761c --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/pnpm-lock.yaml @@ -0,0 +1,73 @@ +--- +lockfileVersion: "9.0" + +importers: + .: + configDependencies: {} + packageManagerDependencies: + "@pnpm/exe": + specifier: 11.3.0 + version: 11.3.0 + pnpm: + specifier: 11.3.0 + version: 11.3.0 + +packages: + "@pnpm/exe@11.3.0": + resolution: { + integrity: sha512-1ItrG3GdA8HC7IUMy79SmYqynjjfwXtIMlbpx9MzrU3ZXMYMfw3yuwjIXW7Aw2z0rRxxa2KtTYcLxqjIaVjUmg==, + } + hasBin: true + + pnpm@11.3.0: + resolution: { integrity: sha512-AAAA } + hasBin: true + +snapshots: + "@pnpm/exe@11.3.0": {} + + pnpm@11.3.0: {} + +--- +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + .: + dependencies: + effect: + specifier: 4.0.0 + version: 4.0.0 + devDependencies: + "@effect/vitest": + specifier: 4.0.0 + version: 4.0.0(effect@4.0.0)(vitest@5.0.3) + vitest: + specifier: 5.0.3 + version: 5.0.3 + +packages: + "@effect/vitest@4.0.0": + resolution: { integrity: sha512-BBBB } + peerDependencies: + effect: ^4.0.0 + vitest: ^5.0.0 + + effect@4.0.0: + resolution: { integrity: sha512-CCCC } + + vitest@5.0.3: + resolution: { integrity: sha512-DDDD } + +snapshots: + "@effect/vitest@4.0.0(effect@4.0.0)(vitest@5.0.3)": + dependencies: + effect: 4.0.0 + vitest: 5.0.3 + + effect@4.0.0: {} + + vitest@5.0.3: {} diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/src/intake.ts b/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/src/intake.ts new file mode 100644 index 0000000..7b74db8 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/src/intake.ts @@ -0,0 +1,4 @@ +// oxlint-disable-next-line no-console -- counted +console.log('intake') +// @ts-expect-error counted +export const n: number = 'x' diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/vendor/copied.ts b/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/vendor/copied.ts new file mode 100644 index 0000000..69bd8f4 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/vendor/copied.ts @@ -0,0 +1,2 @@ +// @ts-nocheck vendored, excluded +export const v = 1 diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/pnpm-lock.yaml b/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/pnpm-lock.yaml new file mode 100644 index 0000000..df58f0f --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/pnpm-lock.yaml @@ -0,0 +1,78 @@ +--- +lockfileVersion: "9.0" + +importers: + .: + configDependencies: {} + packageManagerDependencies: + pnpm: + specifier: 12.4.2 + version: 12.4.2 + +packages: + "@pnpm/exe.linux-x64@12.4.2": + resolution: { integrity: sha512-EEEE } + cpu: [x64] + os: [linux] + + pnpm@12.4.2: + resolution: { integrity: sha512-FFFF } + hasBin: true + +snapshots: + "@pnpm/exe.linux-x64@12.4.2": + optional: true + + pnpm@12.4.2: + optionalDependencies: + "@pnpm/exe.linux-x64": 12.4.2 + +--- +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +catalogs: + default: + effect: + specifier: ^4.0.0 + version: 4.0.0 + +importers: + .: + dependencies: + effect: + specifier: "catalog:" + version: 4.0.0 + devDependencies: + "@effect/vitest": + specifier: 4.0.0 + version: 4.0.0(effect@4.0.0)(vitest@5.0.3) + vitest: + specifier: 5.0.3 + version: 5.0.3 + +packages: + "@effect/vitest@4.0.0": + resolution: { integrity: sha512-BBBB } + peerDependencies: + effect: ^4.0.0 + vitest: ^5.0.0 + + effect@4.0.0: + resolution: { integrity: sha512-CCCC } + + vitest@5.0.3: + resolution: { integrity: sha512-DDDD } + +snapshots: + "@effect/vitest@4.0.0(effect@4.0.0)(vitest@5.0.3)": + dependencies: + effect: 4.0.0 + vitest: 5.0.3 + + effect@4.0.0: {} + + vitest@5.0.3: {} diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/repos/subtree.ts b/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/repos/subtree.ts new file mode 100644 index 0000000..66b66a7 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/repos/subtree.ts @@ -0,0 +1,2 @@ +// @ts-ignore subtree, excluded +export const s = 1 diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/src/claim.ts b/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/src/claim.ts new file mode 100644 index 0000000..43fcc80 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/src/claim.ts @@ -0,0 +1,3 @@ +// eslint-disable-next-line no-var -- counted +export const claim = 1 +export const text = 'oxlint-disable inside a string is not counted' diff --git a/evals/ratstack-scorecard/journeys/launcher-run.ts b/evals/ratstack-scorecard/journeys/launcher-run.ts new file mode 100644 index 0000000..285a85c --- /dev/null +++ b/evals/ratstack-scorecard/journeys/launcher-run.ts @@ -0,0 +1,83 @@ +import { Effect, Schema } from 'effect' +import { createHash } from 'node:crypto' +import { readdir, readFile, stat } from 'node:fs/promises' +import { join } from 'node:path' + +const instrument = join(import.meta.dirname, '..') + +const Manifest = Schema.Struct({ + journeys: Schema.Array(Schema.Struct({ id: Schema.String, inputs: Schema.Array(Schema.String) })), +}) + +export const LauncherRecord = Schema.Struct({ + id: Schema.String, + inputHash: Schema.String, + argv: Schema.Array(Schema.String), + code: Schema.Number, + stdout: Schema.String, + stderr: Schema.String, + files: Schema.Record(Schema.String, Schema.String), + egressLog: Schema.Null, + wallMs: Schema.Number, +}) +export type LauncherRecord = typeof LauncherRecord.Type + +export class MissingLauncherRecord extends Schema.TaggedError()('MissingLauncherRecord', { + journey: Schema.String, +}) { + override get message(): string { + return `journey ${this.journey} has no launcher record: run \`scorecard journeys\`, which produces it before vitest` + } +} + +export class StaleLauncherRecord extends Schema.TaggedError()('StaleLauncherRecord', { + journey: Schema.String, + recorded: Schema.String, + current: Schema.String, +}) { + override get message(): string { + return `journey ${this.journey}'s launcher record is stale (inputs ${this.recorded} recorded, ${this.current} now)` + } +} + +const sha256 = (bytes: Uint8Array | string): string => createHash('sha256').update(bytes).digest('hex') + +const filesUnder = async (path: string): Promise => { + const root = join(instrument, path) + if ((await stat(root)).isFile()) return [path] + const entries = await readdir(root, { recursive: true, withFileTypes: true }) + return entries + .filter((entry) => entry.isFile()) + .map((entry) => `${path}/${join(entry.parentPath, entry.name).slice(root.length + 1)}`) +} + +const inputHashOf = async (inputs: readonly string[], launcher: string): Promise => { + const files = (await Promise.all(inputs.map(filesUnder))).flat().sort() + const parts = await Promise.all( + files.map(async (file) => `${file}\0${sha256(await readFile(join(instrument, file)))}`), + ) + return sha256([launcher, ...parts].join('\n')) +} + +const readJson = (path: string) => Effect.promise(async () => JSON.parse(await readFile(path, 'utf8')) as unknown) + +const launcherRunEffect = (journey: string) => + Effect.gen(function*() { + const manifest = yield* Schema.decodeUnknownEffect(Manifest)( + yield* readJson(join(instrument, 'journeys/manifest.json')), + ) + const entry = manifest.journeys.find((candidate) => candidate.id === journey) + if (entry === undefined) return yield* new MissingLauncherRecord({ journey }) + const raw = yield* Effect.tryPromise({ + try: async () => JSON.parse(await readFile(join(instrument, 'journeys/__records__', `${journey}.json`), 'utf8')), + catch: () => new MissingLauncherRecord({ journey }), + }) + const record = yield* Schema.decodeUnknownEffect(LauncherRecord)(raw) + const current = yield* Effect.promise(() => inputHashOf(entry.inputs, process.env['SCORECARD_LAUNCHER'] ?? '')) + if (record.inputHash !== current) { + return yield* new StaleLauncherRecord({ journey, recorded: record.inputHash, current }) + } + return record + }) + +export const launcherRun = (journey: string): Promise => Effect.runPromise(launcherRunEffect(journey)) diff --git a/evals/ratstack-scorecard/journeys/manifest.json b/evals/ratstack-scorecard/journeys/manifest.json new file mode 100644 index 0000000..5f8ed77 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/manifest.json @@ -0,0 +1,34 @@ +{ + "journeys": [ + { + "id": "static-family", + "produce": { + "kind": "measure-static", + "ratstackRepo": "journeys/__fixtures__/repos/ratstack", + "starterRepo": "journeys/__fixtures__/repos/starter", + "failingTool": null + }, + "inputs": [ + "journeys/__fixtures__/repos", + "src", + "pnpm-lock.yaml", + "flake.lock" + ] + }, + { + "id": "static-family-failing-tool", + "produce": { + "kind": "measure-static", + "ratstackRepo": "journeys/__fixtures__/repos/ratstack", + "starterRepo": "journeys/__fixtures__/repos/starter", + "failingTool": "extract-comments.mjs" + }, + "inputs": [ + "journeys/__fixtures__/repos", + "src", + "pnpm-lock.yaml", + "flake.lock" + ] + } + ] +} diff --git a/evals/ratstack-scorecard/journeys/static-family.test.ts b/evals/ratstack-scorecard/journeys/static-family.test.ts new file mode 100644 index 0000000..4054c3e --- /dev/null +++ b/evals/ratstack-scorecard/journeys/static-family.test.ts @@ -0,0 +1,47 @@ +import { Schema } from 'effect' +import { describe, expect, test } from 'vitest' +import { launcherRun } from './launcher-run.ts' + +const Cell = Schema.Union([ + Schema.Struct({ _tag: Schema.Literal('Measured'), runs: Schema.Array(Schema.Number) }), + Schema.Struct({ _tag: Schema.Literal('InstrumentError'), error: Schema.String }), +]) + +const Family = Schema.Struct({ + family: Schema.Literal('static'), + cells: Schema.Array( + Schema.Struct({ + id: Schema.String, + side: Schema.Literals(['ratstack', 'starter']), + measured: Schema.Struct({ cell: Cell }), + }), + ), +}) + +const familyOf = async (journey: string) => + Schema.decodeUnknownPromise(Family)(JSON.parse((await launcherRun(journey)).files['family.json'] ?? 'null')) + +const cell = (family: typeof Family.Type, id: string, side: 'ratstack' | 'starter') => + family.cells.filter((c) => c.id === id && c.side === side).map((c) => c.measured.cell) + +describe('measureStatic end to end in the launcher, over two fixture git repos', () => { + test('each side gets one M23 and one M28 cell of three runs, counting only unvendored comment directives', async () => { + const family = await familyOf('static-family') + expect(cell(family, 'M23', 'ratstack')).toEqual([{ _tag: 'Measured', runs: [2, 2, 2] }]) + expect(cell(family, 'M23', 'starter')).toEqual([{ _tag: 'Measured', runs: [1, 1, 1] }]) + const [ratstackPackages] = cell(family, 'M28', 'ratstack') + const [starterPackages] = cell(family, 'M28', 'starter') + expect(ratstackPackages?._tag).toBe('Measured') + expect(starterPackages).toEqual(ratstackPackages) + expect(family.cells).toHaveLength(4) + }) + + test('a tool that exits non-zero gives an instrument error carrying its exit code, on both sides', async () => { + const family = await familyOf('static-family-failing-tool') + for (const side of ['ratstack', 'starter'] as const) { + const [m23] = cell(family, 'M23', side) + expect(m23?._tag).toBe('InstrumentError') + expect(m23?._tag === 'InstrumentError' && m23.error).toContain('extract-comments exited 3') + } + }) +}) diff --git a/evals/ratstack-scorecard/journeys/static-tools.test.ts b/evals/ratstack-scorecard/journeys/static-tools.test.ts new file mode 100644 index 0000000..992e946 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/static-tools.test.ts @@ -0,0 +1,68 @@ +import { execFile } from 'node:child_process' +import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { promisify } from 'node:util' +import { describe, expect, test } from 'vitest' +import { isSourceFile, tallyDirectives, totalDirectives, vendoredRootsOf } from '../src/model/directives.ts' +import { distinctPackages } from '../src/model/lockfile.ts' + +const run = promisify(execFile) +const instrument = join(import.meta.dirname, '..') +const fixtures = join(import.meta.dirname, '__fixtures__') + +const debtTree: Readonly> = { + 'src/flagged.ts': [ + '// oxlint-disable-next-line no-console -- the one directive in this tree', + "const text = 'oxlint-disable-next-line written inside a string'", + 'const pattern = /@ts-ignore/u', + 'const template = `eslint-disable ${text}`', + 'export { pattern, template, text }', + '', + ].join('\n'), + 'vendor/copied.ts': '// @ts-nocheck vendored code keeps its own directives\nexport const copied = 1\n', + 'README.md': '\n', +} + +const writeTree = async (root: string, files: Readonly>): Promise => { + for (const [path, content] of Object.entries(files)) { + await mkdir(dirname(join(root, path)), { recursive: true }) + await writeFile(join(root, path), content) + } +} + +const tool = async (name: string, args: readonly string[]): Promise => { + const output = join(await mkdtemp(join(tmpdir(), 'journey-')), 'out.json') + await run('node', [join(instrument, 'src/tools', name), ...args, output], { cwd: instrument }) + return JSON.parse(await readFile(output, 'utf8')) +} + +describe('static family through the real tools (J2)', () => { + test('only the directive in a comment counts; strings, regex literals, templates and vendored files do not', async () => { + const root = await mkdtemp(join(tmpdir(), 'debt-tree-')) + await writeTree(root, debtTree) + const files = Object.keys(debtTree) + .filter(isSourceFile) + .filter((file) => vendoredRootsOf(file, ['vendor/']).length === 0) + const request = join(root, 'request.json') + await writeFile(request, JSON.stringify({ root, files })) + const extracted = await tool('extract-comments.mjs', [request]) + const comments = (extracted as { files: { comments: string[] }[] }).files.flatMap((file) => file.comments) + const tally = tallyDirectives(comments) + expect(files).toEqual(['src/flagged.ts']) + expect(tally.oxlint).toBe(1) + expect(totalDirectives(tally)).toBe(1) + }) + + test('pnpm 11 and pnpm 12 lockfiles list their packages across both YAML documents, without peer suffixes', async () => { + const packagesOf = async (fixture: string) => { + const parsed = await tool('parse-lockfile.mjs', [join(fixtures, 'lockfiles', fixture)]) + return distinctPackages((parsed as { documents: { packages: string[] }[] }).documents.flatMap((d) => d.packages)) + } + const application = ['@effect/vitest@4.0.0', 'effect@4.0.0', 'vitest@5.0.3'] + expect(await packagesOf('pnpm11.yaml')).toEqual([...application, '@pnpm/exe@11.3.0', 'pnpm@11.3.0'].sort()) + expect(await packagesOf('pnpm12.yaml')).toEqual( + [...application, '@pnpm/exe.linux-x64@12.4.2', 'pnpm@12.4.2'].sort(), + ) + }) +}) diff --git a/evals/ratstack-scorecard/package.json b/evals/ratstack-scorecard/package.json index 7f59206..0ff3560 100644 --- a/evals/ratstack-scorecard/package.json +++ b/evals/ratstack-scorecard/package.json @@ -2,10 +2,15 @@ "name": "@starter/ratstack-scorecard", "private": true, "type": "module", + "dependencies": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + }, "devDependencies": { "@fast-check/vitest": "0.5.0", "ajv": "8.20.0", "fast-check": "4.10.2", - "vitest": "5.0.3" + "vitest": "5.0.3", + "effect": "4.0.1" } } diff --git a/evals/ratstack-scorecard/pnpm-lock.yaml b/evals/ratstack-scorecard/pnpm-lock.yaml index 05f1fa0..3ee8937 100644 --- a/evals/ratstack-scorecard/pnpm-lock.yaml +++ b/evals/ratstack-scorecard/pnpm-lock.yaml @@ -7,19 +7,29 @@ settings: importers: .: + dependencies: + oxc-parser: + specifier: 0.152.0 + version: 0.152.0 + yaml: + specifier: 2.9.1 + version: 2.9.1 devDependencies: '@fast-check/vitest': specifier: 0.5.0 - version: 0.5.0(vitest@5.0.3(vite@8.3.2)) + version: 0.5.0(vitest@5.0.3(vite@8.3.2(yaml@2.9.1))) ajv: specifier: 8.20.0 version: 8.20.0 + effect: + specifier: 4.0.1 + version: 4.0.1 fast-check: specifier: 4.10.2 version: 4.10.2 vitest: specifier: 5.0.3 - version: 5.0.3(vite@8.3.2) + version: 5.0.3(vite@8.3.2(yaml@2.9.1)) packages: @@ -38,6 +48,128 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@oxc-parser/binding-android-arm-eabi@0.152.0': + resolution: {integrity: sha512-393VtirINkae4XHBTkq/yQxhOwEr4V+r0et37P2fp+sYZSdYgZTRDPo5Um8vBh0QjMZ+RmaMy4PP0qwr4cTMwA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [android] + + '@oxc-parser/binding-android-arm64@0.152.0': + resolution: {integrity: sha512-iYFcExATJNZ0rZj0o02THbD1Nc9jAW0nVgIQPee6ejbSASpbZXlIMw8jxtpzaG14JQTWwGSZ6U5dEtRJXE6V3Q==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@oxc-parser/binding-darwin-arm64@0.152.0': + resolution: {integrity: sha512-fnU+DfUzi5/rFy0LzmipYRz06PAUUkOS8kCJQ3Eg6oqlunC1XZXoffFqxmdL51RmcbYtUG3P8DVG1U5fckoa2A==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@oxc-parser/binding-darwin-x64@0.152.0': + resolution: {integrity: sha512-Y3U2okryWTs7hDld6UhjJwLNf8/do+x4g4C+1U5SiGwrTLwbO83WJYJA8+l9iFSilWBgnfpd/+U76tfeNxUHzA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@oxc-parser/binding-freebsd-x64@0.152.0': + resolution: {integrity: sha512-XkhkRrrPffEw+ZLv/d/5pEni0buxinm/E/N6fLbJCunD1DBbeCTtVP23ESV+4CWCWdSz397asks1ViGslpmWiQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@oxc-parser/binding-linux-arm-gnueabihf@0.152.0': + resolution: {integrity: sha512-MLzsRRbdfmdzo6JEhbC1yKU9ItfvM8tEf18HM32jCiNYZEz6J3bRBRl3cTkKR0vBdrk9ZVlcARXtwHccs+epDA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@oxc-parser/binding-linux-arm-musleabihf@0.152.0': + resolution: {integrity: sha512-HHerTEJz7Iy10Ji6BGsk4gqG6PU0UzFuBu91jIEOX89FnrI5ECWSEr7BLpkYQhp7U0QFtMnBhNfQKI0hYZK78w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@oxc-parser/binding-linux-arm64-gnu@0.152.0': + resolution: {integrity: sha512-9Bb8GyzZ2uSLzRDZ27usaKokr07mxiBe2dDF5b6bd2janmaJTT1gL51UfP7AU30yFiXrJvcRyFKREAgKwZYs0w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-arm64-musl@0.152.0': + resolution: {integrity: sha512-om7iDMUroI74A/lfRRWY9/Ev3VWrVsrUZ4tyvyh8raBey/sdcfyAD7tYWc+iykAiotnmftyKTpUKFr1QnaBrAQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@oxc-parser/binding-linux-ppc64-gnu@0.152.0': + resolution: {integrity: sha512-Lf1OL2lMTqwDMFC6jnRb30B+WYHKV0XT+bPHcFM6xHtNVwxpobiTzUIvKH6a0LWpvfwyru8Yu6zCGHqqFHH+gg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-riscv64-gnu@0.152.0': + resolution: {integrity: sha512-BYWSMQK+YJ9ZILStPcmmiSVq8KEG9Qdq/c4BHPC3rUEq/fEDCVjUIwdj5ZzjgZ9P9is9PKu0tcAXzq1ZXg4urQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-riscv64-musl@0.152.0': + resolution: {integrity: sha512-d2LqHNNNqdH185sRK5dPKWZu201JrNu9/CRFuVEuT+ht7n/QaVblEMWT9MALQPrDWYvhG10P+NSKJFquDWZ09w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@oxc-parser/binding-linux-s390x-gnu@0.152.0': + resolution: {integrity: sha512-8gBeHt5rA/8+pKhHl+zLggfa3L/V/L2at96p7ty6pBE7id6lFO19nFUc15u2PcjC4OYeYK465MKy1fhhbpFIGw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-x64-gnu@0.152.0': + resolution: {integrity: sha512-ltgrSF6L+yPTVsIdaAyqKq3zTqzj/1VlNbxWZuxAe8flR3sWdyNsKvdXukoO8/ucJbLickhmieRBge+o/e8Ugw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-x64-musl@0.152.0': + resolution: {integrity: sha512-LcMp6BvrBnewSfdkLHoSqidhoAq4syKOTBV5TYHWd+2VnGkfD/nPRRVIzTBt/Lkbt7pfOFDGtkiTntFS0p6sQQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@oxc-parser/binding-openharmony-arm64@0.152.0': + resolution: {integrity: sha512-jbhoMCslVxzRCRNQ8TSHxjYdSKRvk8O6ciLVDkahJvmyHDzBqba/bMQHM6ULKj/rgHNNbSzlIiakdx8SEfkJzA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@oxc-parser/binding-win32-arm64-msvc@0.152.0': + resolution: {integrity: sha512-JZYVlhv36mr/nertnRB7vchJECOBb1Lz6lltalILsB+o9M/is+H6BEDRLQKdqA4JKs6NjJszo46+iM5sfwj6jQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@oxc-parser/binding-win32-ia32-msvc@0.152.0': + resolution: {integrity: sha512-1X9oganvVsunRH0lH10UU05N+H9atpX6WsOybByUjwnfcd3h/0tYwvuv16XuHglrIUOnxV2TodbLVEwwafd64g==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ia32] + os: [win32] + + '@oxc-parser/binding-win32-x64-msvc@0.152.0': + resolution: {integrity: sha512-sw0M80/dn9rcpu+Rqqph/C+MjvDDce6MlXmFjsNQtPEymha0eJy3ml6+foSUTqFfpDIDyHZZ4eoJ43IoMe8bJw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + '@oxc-project/types@0.152.0': resolution: {integrity: sha512-oM/5rLBm2tPkg0iBgkH/FOeR3PCDpY19GTgAZjMFM8h9WI9VW7cLgzp6nwtarYKmovavIQZ+Fe/RKX/8C8O/Rw==} @@ -178,6 +310,9 @@ packages: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} + effect@4.0.1: + resolution: {integrity: sha512-b1VlQG9g8fwxE5QnIZuPoOP/0MsqHJSRKxUijjoM80E5q95FXrX5yWtY5XI8G+GJsBQAVSDbLS458z9a++uVvw==} + es-module-lexer@2.3.2: resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==} @@ -301,6 +436,10 @@ packages: resolution: {integrity: sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==} engines: {node: '>=12.20.0'} + oxc-parser@0.152.0: + resolution: {integrity: sha512-ZztsZgaCWmPeKYMdoTT5e0oV/oxD1NeLWNR3kAtr/G707cUPYaaozRlVDHLCGnszHJRszN9mJwmPNpqgwloC0A==} + engines: {node: ^20.19.0 || >=22.12.0} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -432,12 +571,17 @@ packages: engines: {node: '>=20.11'} hasBin: true + yaml@2.9.1: + resolution: {integrity: sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==} + engines: {node: '>= 14.6'} + hasBin: true + snapshots: - '@fast-check/vitest@0.5.0(vitest@5.0.3(vite@8.3.2))': + '@fast-check/vitest@0.5.0(vitest@5.0.3(vite@8.3.2(yaml@2.9.1)))': dependencies: fast-check: 4.10.2 - vitest: 5.0.3(vite@8.3.2) + vitest: 5.0.3(vite@8.3.2(yaml@2.9.1)) '@jridgewell/resolve-uri@3.1.2': {} @@ -448,6 +592,63 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.6.0 + '@oxc-parser/binding-android-arm-eabi@0.152.0': + optional: true + + '@oxc-parser/binding-android-arm64@0.152.0': + optional: true + + '@oxc-parser/binding-darwin-arm64@0.152.0': + optional: true + + '@oxc-parser/binding-darwin-x64@0.152.0': + optional: true + + '@oxc-parser/binding-freebsd-x64@0.152.0': + optional: true + + '@oxc-parser/binding-linux-arm-gnueabihf@0.152.0': + optional: true + + '@oxc-parser/binding-linux-arm-musleabihf@0.152.0': + optional: true + + '@oxc-parser/binding-linux-arm64-gnu@0.152.0': + optional: true + + '@oxc-parser/binding-linux-arm64-musl@0.152.0': + optional: true + + '@oxc-parser/binding-linux-ppc64-gnu@0.152.0': + optional: true + + '@oxc-parser/binding-linux-riscv64-gnu@0.152.0': + optional: true + + '@oxc-parser/binding-linux-riscv64-musl@0.152.0': + optional: true + + '@oxc-parser/binding-linux-s390x-gnu@0.152.0': + optional: true + + '@oxc-parser/binding-linux-x64-gnu@0.152.0': + optional: true + + '@oxc-parser/binding-linux-x64-musl@0.152.0': + optional: true + + '@oxc-parser/binding-openharmony-arm64@0.152.0': + optional: true + + '@oxc-parser/binding-win32-arm64-msvc@0.152.0': + optional: true + + '@oxc-parser/binding-win32-ia32-msvc@0.152.0': + optional: true + + '@oxc-parser/binding-win32-x64-msvc@0.152.0': + optional: true + '@oxc-project/types@0.152.0': {} '@rolldown/binding-android-arm-eabi@1.2.12': @@ -506,14 +707,14 @@ snapshots: '@types/estree@1.0.9': {} - '@vitest/mocker@5.0.3(vite@8.3.2)': + '@vitest/mocker@5.0.3(vite@8.3.2(yaml@2.9.1))': dependencies: '@jridgewell/trace-mapping': 0.3.31 '@vitest/spy': 5.0.3 estree-walker: 3.0.3 magic-string: 1.4.2 optionalDependencies: - vite: 8.3.2 + vite: 8.3.2(yaml@2.9.1) '@vitest/spy@5.0.3': {} @@ -530,6 +731,8 @@ snapshots: detect-libc@2.1.2: {} + effect@4.0.1: {} + es-module-lexer@2.3.2: {} estree-walker@3.0.3: @@ -612,6 +815,30 @@ snapshots: obug@2.2.1: {} + oxc-parser@0.152.0: + dependencies: + '@oxc-project/types': 0.152.0 + optionalDependencies: + '@oxc-parser/binding-android-arm-eabi': 0.152.0 + '@oxc-parser/binding-android-arm64': 0.152.0 + '@oxc-parser/binding-darwin-arm64': 0.152.0 + '@oxc-parser/binding-darwin-x64': 0.152.0 + '@oxc-parser/binding-freebsd-x64': 0.152.0 + '@oxc-parser/binding-linux-arm-gnueabihf': 0.152.0 + '@oxc-parser/binding-linux-arm-musleabihf': 0.152.0 + '@oxc-parser/binding-linux-arm64-gnu': 0.152.0 + '@oxc-parser/binding-linux-arm64-musl': 0.152.0 + '@oxc-parser/binding-linux-ppc64-gnu': 0.152.0 + '@oxc-parser/binding-linux-riscv64-gnu': 0.152.0 + '@oxc-parser/binding-linux-riscv64-musl': 0.152.0 + '@oxc-parser/binding-linux-s390x-gnu': 0.152.0 + '@oxc-parser/binding-linux-x64-gnu': 0.152.0 + '@oxc-parser/binding-linux-x64-musl': 0.152.0 + '@oxc-parser/binding-openharmony-arm64': 0.152.0 + '@oxc-parser/binding-win32-arm64-msvc': 0.152.0 + '@oxc-parser/binding-win32-ia32-msvc': 0.152.0 + '@oxc-parser/binding-win32-x64-msvc': 0.152.0 + picocolors@1.1.1: {} picomatch@4.0.7: {} @@ -660,7 +887,7 @@ snapshots: fdir: 6.5.0(picomatch@4.0.7) picomatch: 4.0.7 - vite@8.3.2: + vite@8.3.2(yaml@2.9.1): dependencies: lightningcss: 1.33.0 picomatch: 4.0.7 @@ -669,11 +896,12 @@ snapshots: tinyglobby: 0.2.17 optionalDependencies: fsevents: 2.3.3 + yaml: 2.9.1 - vitest@5.0.3(vite@8.3.2): + vitest@5.0.3(vite@8.3.2(yaml@2.9.1)): dependencies: '@types/chai': 5.2.3 - '@vitest/mocker': 5.0.3(vite@8.3.2) + '@vitest/mocker': 5.0.3(vite@8.3.2(yaml@2.9.1)) chai: 6.3.0 es-module-lexer: 2.3.2 expect-type: 1.4.0 @@ -684,9 +912,11 @@ snapshots: tinybench: 6.2.0 tinyexec: 1.3.1 tinyglobby: 0.2.17 - vite: 8.3.2 + vite: 8.3.2(yaml@2.9.1) why-is-node-running: 3.2.1 transitivePeerDependencies: - msw why-is-node-running@3.2.1: {} + + yaml@2.9.1: {} diff --git a/evals/ratstack-scorecard/pnpm-workspace.yaml b/evals/ratstack-scorecard/pnpm-workspace.yaml index d05a7e7..a63a95d 100644 --- a/evals/ratstack-scorecard/pnpm-workspace.yaml +++ b/evals/ratstack-scorecard/pnpm-workspace.yaml @@ -1,2 +1,7 @@ packages: - . + +supportedArchitectures: + os: [linux, darwin] + cpu: [x64, arm64] + libc: [glibc, musl] diff --git a/evals/ratstack-scorecard/ratstack.pin.json b/evals/ratstack-scorecard/ratstack.pin.json new file mode 100644 index 0000000..c2e0cd6 --- /dev/null +++ b/evals/ratstack-scorecard/ratstack.pin.json @@ -0,0 +1,6 @@ +{ + "owner": "joelhooks", + "repo": "rat-stack", + "commit": "54d356037c994f89698760a4727be71d0005a087", + "narHash": "sha256-JVHgmV8rqDS3EjfcLbeb1VGVRS1e5STcOONL9ktIKgA=" +} diff --git a/evals/ratstack-scorecard/scorecard.schema.json b/evals/ratstack-scorecard/scorecard.schema.json index fdbdcab..62f1d3b 100644 --- a/evals/ratstack-scorecard/scorecard.schema.json +++ b/evals/ratstack-scorecard/scorecard.schema.json @@ -38,7 +38,11 @@ "runner": { "type": "string", "minLength": 1 }, "measuredAt": { "type": "string", "minLength": 1 }, "tools": { "type": "object", "additionalProperties": { "type": "string" } }, - "liveCommit": { "$ref": "#/definitions/sha" } + "liveCommit": { "$ref": "#/definitions/sha" }, + "detail": { + "type": "object", + "additionalProperties": { "anyOf": [{ "type": "number" }, { "type": "string" }] } + } } }, "citation": { diff --git a/evals/ratstack-scorecard/src/check-imports.ts b/evals/ratstack-scorecard/src/check-imports.ts new file mode 100644 index 0000000..2b2a324 --- /dev/null +++ b/evals/ratstack-scorecard/src/check-imports.ts @@ -0,0 +1,41 @@ +import { join } from 'node:path' +import { walkFiles } from './harness/instrument.ts' + +const specifiers = (source: string): readonly string[] => + [ + ...source.matchAll( + /(?:^|\n)\s*(?:import|export)\b[^'"]*?from\s*['"]([^'"]+)['"]|(?:^|\n)\s*import\s*['"]([^'"]+)['"]|import\(\s*['"]([^'"]+)['"]\s*\)/gu, + ), + ].map((match) => match[1] ?? match[2] ?? match[3] ?? '') + +const isThirdParty = (specifier: string): boolean => + !specifier.startsWith('.') && !specifier.startsWith('/') && !specifier.startsWith('node:') + +const driverGraph = async (root: string, entry: string, seen: Set): Promise => { + if (seen.has(entry)) return [] + seen.add(entry) + const found: string[] = [] + for (const specifier of specifiers(await Deno.readTextFile(join(root, entry)))) { + if (isThirdParty(specifier) || specifier.includes('node_modules')) found.push(`${entry} imports ${specifier}`) + else if (specifier.startsWith('.')) found.push(...await driverGraph(root, join(entry, '..', specifier), seen)) + } + return found +} + +export const checkImports = async (root: string): Promise => { + const driver = await driverGraph(root, 'src/main.ts', new Set()) + const fixtureImports = (await walkFiles(join(root, 'src'))) + .filter((file) => file.endsWith('.ts')) + .map((file) => `src/${file}`) + const misplacedJourneys: string[] = [] + for (const file of fixtureImports) { + const imports = specifiers(await Deno.readTextFile(join(root, file))) + if (imports.some((specifier) => specifier.endsWith('journeys/launcher-run.ts'))) { + misplacedJourneys.push(`${file} imports the journey fixture; journeys live in journeys/, never beside the model`) + } + } + return [ + ...driver.map((line) => `host driver loads third-party code: ${line}`), + ...misplacedJourneys, + ] +} diff --git a/evals/ratstack-scorecard/src/families/static.ts b/evals/ratstack-scorecard/src/families/static.ts new file mode 100644 index 0000000..45b7414 --- /dev/null +++ b/evals/ratstack-scorecard/src/families/static.ts @@ -0,0 +1,142 @@ +import { join } from 'node:path' +import { arrayAt, numberAt, stringAt } from '../harness/decode.ts' +import { + type Instrument, + materializeRatstack, + materializeStarter, + prepareTools, + provenanceFor, + trackedFiles, + walkFiles, +} from '../harness/instrument.ts' +import { runSandboxed } from '../harness/sandbox.ts' +import type { Cell, FamilyResult, Side, SideCell } from '../model/cell.ts' +import { isSourceFile, tallyDirectives, totalDirectives, vendoredRootsOf } from '../model/directives.ts' +import { distinctPackages } from '../model/lockfile.ts' +import { ratstack } from '../sides/ratstack.ts' +import type { SideAdapter } from '../sides/side-adapter.ts' +import { starter } from '../sides/starter.ts' + +const runs = 3 + +interface Subject { + readonly adapter: SideAdapter + readonly root: string + readonly files: readonly string[] +} + +interface CountedRun { + readonly cell: Cell + readonly detail: Readonly> +} + +const instrumentError = (error: string): CountedRun => ({ cell: { _tag: 'InstrumentError', error }, detail: {} }) + +const node = async (instrument: Instrument, work: string, tools: string, args: readonly string[]) => + await runSandboxed(instrument.launcher, { + project: work, + cwd: tools, + command: ['node', ...args], + deadlineMs: 15 * 60_000, + }) + +const subjectOf = async (instrument: Instrument, work: string, side: Side): Promise => + side === 'ratstack' + ? { + adapter: ratstack, + root: await materializeRatstack(instrument, work), + files: await walkFiles(instrument.ratstackSrc), + } + : { + adapter: starter, + root: await materializeStarter(instrument, work), + files: await trackedFiles(instrument.checkout), + } + +const countDirectives = async ( + instrument: Instrument, + work: string, + tools: string, + subject: Subject, +): Promise => { + const sources = subject.files.filter(isSourceFile) + const excluded = subject.adapter.vendored.map((vendored) => ({ + ...vendored, + files: sources.filter((file) => vendoredRootsOf(file, [vendored.root]).length > 0).length, + })) + const counted = sources.filter((file) => + vendoredRootsOf(file, subject.adapter.vendored.map((v) => v.root)).length === 0 + ) + const request = join(work, `directives-${subject.adapter.side}.request.json`) + await Deno.writeTextFile(request, JSON.stringify({ root: subject.root, files: counted })) + const totals: number[] = [] + let detail: Record = {} + for (let run = 0; run < runs; run++) { + const output = join(work, `directives-${subject.adapter.side}-${run}.json`) + const result = await node(instrument, work, tools, ['src/tools/extract-comments.mjs', request, output]) + if (result.code !== 0) { + return instrumentError(`extract-comments exited ${result.code}: ${result.stderr.slice(-2000)}`) + } + const parsed = JSON.parse(await Deno.readTextFile(output)) + const files = arrayAt(parsed, ['files'], 'extract-comments output') + const comments = files.flatMap((file) => arrayAt(file, ['comments'], 'extract-comments file').map(String)) + const tally = tallyDirectives(comments) + totals.push(totalDirectives(tally)) + detail = { + ...tally, + filesCounted: counted.length, + parseErrors: files.reduce((sum, file) => sum + numberAt(file, ['errors'], 'extract-comments file'), 0), + ...Object.fromEntries(excluded.map((v) => [`excluded:${v.root}`, `${v.files} files: ${v.reason}`])), + } + } + return { cell: { _tag: 'Measured', runs: totals }, detail } +} + +const countPackages = async ( + instrument: Instrument, + work: string, + tools: string, + subject: Subject, +): Promise => { + const lockfile = join(subject.root, subject.adapter.lockfile) + const counts: number[] = [] + let detail: Record = {} + for (let run = 0; run < runs; run++) { + const output = join(work, `lockfile-${subject.adapter.side}-${run}.json`) + const result = await node(instrument, work, tools, ['src/tools/parse-lockfile.mjs', lockfile, output]) + if (result.code !== 0) return instrumentError(`parse-lockfile exited ${result.code}: ${result.stderr.slice(-2000)}`) + const documents = arrayAt(JSON.parse(await Deno.readTextFile(output)), ['documents'], 'parse-lockfile output') + const keys = documents.flatMap((doc) => arrayAt(doc, ['packages'], 'lockfile document').map(String)) + counts.push(distinctPackages(keys).length) + detail = { + lockfile: subject.adapter.lockfile, + documents: documents.length, + lockfileVersions: documents.map((doc) => stringAt(doc, ['lockfileVersion'], 'lockfile document')).join(', '), + } + } + return { cell: { _tag: 'Measured', runs: counts }, detail } +} + +export const measureStatic = async ( + instrument: Instrument, + work: string, + sides: readonly Side[], +): Promise => { + const started = performance.now() + const tools = await prepareTools(instrument, work) + const cells: SideCell[] = [] + for (const side of sides) { + const subject = await subjectOf(instrument, work, side) + const debt = await countDirectives(instrument, work, tools, subject) + const packages = await countPackages(instrument, work, tools, subject) + cells.push( + { id: 'M23', side, measured: { cell: debt.cell, provenance: provenanceFor(instrument, side, debt.detail) } }, + { + id: 'M28', + side, + measured: { cell: packages.cell, provenance: provenanceFor(instrument, side, packages.detail) }, + }, + ) + } + return { family: 'static', wallMs: Math.round(performance.now() - started), cells, flags: [] } +} diff --git a/evals/ratstack-scorecard/src/harness/decode.ts b/evals/ratstack-scorecard/src/harness/decode.ts new file mode 100644 index 0000000..e271455 --- /dev/null +++ b/evals/ratstack-scorecard/src/harness/decode.ts @@ -0,0 +1,41 @@ +export const at = (value: unknown, key: string): unknown => + typeof value === 'object' && value !== null && key in value + ? Object.getOwnPropertyDescriptor(value, key)?.value + : undefined + +const pathOf = (value: unknown, path: readonly string[]): unknown => path.reduce(at, value) + +const refuse = (what: string, path: readonly string[], expected: string): never => { + throw new Error(`${what}: expected ${expected} at ${path.join('.') || ''}`) +} + +export const stringAt = (value: unknown, path: readonly string[], what: string): string => { + const found = pathOf(value, path) + return typeof found === 'string' ? found : refuse(what, path, 'a string') +} + +export const numberAt = (value: unknown, path: readonly string[], what: string): number => { + const found = pathOf(value, path) + return typeof found === 'number' ? found : refuse(what, path, 'a number') +} + +export const arrayAt = (value: unknown, path: readonly string[], what: string): readonly unknown[] => { + const found = pathOf(value, path) + return Array.isArray(found) ? found : refuse(what, path, 'an array') +} + +export const entriesAt = ( + value: unknown, + path: readonly string[], + what: string, +): readonly (readonly [string, unknown])[] => { + const found = pathOf(value, path) + return typeof found === 'object' && found !== null && !Array.isArray(found) + ? Object.entries(found) + : refuse(what, path, 'an object') +} + +export const optionalEntriesAt = (value: unknown, path: readonly string[]): readonly (readonly [string, unknown])[] => { + const found = pathOf(value, path) + return typeof found === 'object' && found !== null && !Array.isArray(found) ? Object.entries(found) : [] +} diff --git a/evals/ratstack-scorecard/src/harness/instrument.ts b/evals/ratstack-scorecard/src/harness/instrument.ts new file mode 100644 index 0000000..7393b3f --- /dev/null +++ b/evals/ratstack-scorecard/src/harness/instrument.ts @@ -0,0 +1,151 @@ +import { dirname, join } from 'node:path' +import type { CellProvenance, Side } from '../model/cell.ts' +import { optionalEntriesAt, stringAt } from './decode.ts' +import { type Launcher, runSandboxed } from './sandbox.ts' + +export interface Pin { + readonly owner: string + readonly repo: string + readonly commit: string + readonly narHash: string +} + +export interface Instrument { + readonly dir: string + readonly launcher: Launcher + readonly toolsStore: string + readonly ratstackSrc: string + readonly checkout: string + readonly pin: Pin + readonly instrumentHash: string + readonly nixpkgsRev: string + readonly runner: string + readonly starterCommit: string + readonly toolVersions: Readonly> +} + +const decoder = new TextDecoder() + +const required = (name: string): string => { + const value = Deno.env.get(name) + if (value === undefined || value === '') { + throw new Error(`${name} is unset: run the scorecard through the flake's \`scorecard\` package`) + } + return value +} + +export const git = async (cwd: string, args: readonly string[]): Promise => { + const output = await new Deno.Command('git', { args: [...args], cwd, stdout: 'piped', stderr: 'piped' }).output() + if (!output.success) throw new Error(`git ${args.join(' ')} failed: ${decoder.decode(output.stderr)}`) + return output.stdout +} + +const gitText = async (cwd: string, args: readonly string[]): Promise => + decoder.decode(await git(cwd, args)).trim() + +const readJson = async (path: string): Promise => JSON.parse(await Deno.readTextFile(path)) + +const nixpkgsRevOf = (lock: unknown): string => stringAt(lock, ['nodes', 'nixpkgs', 'locked', 'rev'], 'flake.lock') + +const pinOf = (pin: unknown): Pin => ({ + owner: stringAt(pin, ['owner'], 'ratstack.pin.json'), + repo: stringAt(pin, ['repo'], 'ratstack.pin.json'), + commit: stringAt(pin, ['commit'], 'ratstack.pin.json'), + narHash: stringAt(pin, ['narHash'], 'ratstack.pin.json'), +}) + +const dependencyVersions = (manifest: unknown): Record => + Object.fromEntries(optionalEntriesAt(manifest, ['dependencies']).map(([name, version]) => [name, String(version)])) + +export const loadInstrument = async (checkoutArg: string | undefined): Promise => { + const dir = required('SCORECARD_INSTRUMENT') + const checkout = await gitText(checkoutArg ?? Deno.cwd(), ['rev-parse', '--show-toplevel']) + return { + dir, + launcher: { executable: required('SCORECARD_SANDBOX'), path: required('SCORECARD_TOOL_PATH') }, + toolsStore: required('SCORECARD_TOOLS_STORE'), + ratstackSrc: required('SCORECARD_RATSTACK_SRC'), + checkout, + pin: pinOf(await readJson(join(dir, 'ratstack.pin.json'))), + instrumentHash: await gitText(checkout, ['rev-parse', 'HEAD:evals/ratstack-scorecard']), + nixpkgsRev: nixpkgsRevOf(await readJson(join(dir, 'flake.lock'))), + runner: Deno.env.get('RUNNER_NAME') ?? Deno.hostname(), + starterCommit: await gitText(checkout, ['rev-parse', 'HEAD']), + toolVersions: { + ...dependencyVersions(await readJson(join(dir, 'package.json'))), + node: required('SCORECARD_NODE_VERSION'), + pnpm: required('SCORECARD_PNPM_VERSION'), + deno: Deno.version.deno, + }, + } +} + +export const provenanceFor = ( + instrument: Instrument, + side: Side, + detail: Readonly>, +): CellProvenance => ({ + side, + commit: { ratstack: instrument.pin.commit, starter: instrument.starterCommit }[side], + instrumentHash: instrument.instrumentHash, + nixpkgsRev: instrument.nixpkgsRev, + runner: instrument.runner, + measuredAt: new Date().toISOString(), + tools: instrument.toolVersions, + detail, +}) + +const copyTracked = async (from: string, to: string, files: readonly string[]): Promise => { + for (const file of files) { + const target = join(to, file) + await Deno.mkdir(dirname(target), { recursive: true }) + const info = await Deno.lstat(join(from, file)) + if (info.isSymlink) await Deno.symlink(await Deno.readLink(join(from, file)), target) + else await Deno.copyFile(join(from, file), target) + } +} + +export const trackedFiles = async (checkout: string): Promise => + decoder.decode(await git(checkout, ['ls-files', '-z'])).split('\0').filter((file) => file !== '') + +export const walkFiles = async (root: string, prefix = ''): Promise => { + const found: string[] = [] + for await (const entry of Deno.readDir(join(root, prefix))) { + const path = prefix === '' ? entry.name : `${prefix}/${entry.name}` + if (entry.isDirectory) found.push(...await walkFiles(root, path)) + else found.push(path) + } + return found.sort() +} + +export const materializeStarter = async (instrument: Instrument, work: string): Promise => { + const target = join(work, 'starter') + await copyTracked(instrument.checkout, target, await trackedFiles(instrument.checkout)) + return target +} + +export const materializeRatstack = async (instrument: Instrument, work: string): Promise => { + const target = join(work, 'ratstack') + await copyTracked(instrument.ratstackSrc, target, await walkFiles(instrument.ratstackSrc)) + return target +} + +export const prepareTools = async (instrument: Instrument, work: string): Promise => { + const tools = join(work, 'tools') + await Deno.mkdir(join(tools, 'src/tools'), { recursive: true }) + for (const file of ['package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml']) { + await Deno.copyFile(join(instrument.dir, file), join(tools, file)) + } + for await (const entry of Deno.readDir(join(instrument.dir, 'src/tools'))) { + await Deno.copyFile(join(instrument.dir, 'src/tools', entry.name), join(tools, 'src/tools', entry.name)) + } + const install = await runSandboxed(instrument.launcher, { + project: work, + cwd: tools, + command: ['pnpm', 'install', '--frozen-lockfile', '--prod'], + pnpmStore: instrument.toolsStore, + deadlineMs: 10 * 60_000, + }) + if (install.code !== 0) throw new Error(`installing the instrument tools failed:\n${install.stderr}`) + return tools +} diff --git a/evals/ratstack-scorecard/src/harness/sandbox.ts b/evals/ratstack-scorecard/src/harness/sandbox.ts new file mode 100644 index 0000000..ffc8e20 --- /dev/null +++ b/evals/ratstack-scorecard/src/harness/sandbox.ts @@ -0,0 +1,54 @@ +export interface SandboxRequest { + readonly project: string + readonly cwd: string + readonly command: readonly string[] + readonly allowHosts?: readonly string[] + readonly env?: Readonly> + readonly pnpmStore?: string + readonly deadlineMs: number +} + +export interface SandboxResult { + readonly code: number + readonly stdout: string + readonly stderr: string + readonly wallMs: number + readonly pastDeadline: boolean +} + +export interface Launcher { + readonly executable: string + readonly path: string +} + +const decoder = new TextDecoder() + +export const launcherArgs = (request: SandboxRequest): readonly string[] => [ + ...(request.allowHosts ?? []).flatMap((host) => ['--allow-host', host]), + ...Object.keys(request.env ?? {}).flatMap((name) => ['--pass-env', name]), + ...(request.pnpmStore === undefined ? [] : ['--pnpm-store', request.pnpmStore]), + '--', + ...request.command, +] + +export const runSandboxed = async (launcher: Launcher, request: SandboxRequest): Promise => { + const started = performance.now() + const deadline = AbortSignal.timeout(request.deadlineMs) + const output = await new Deno.Command(launcher.executable, { + args: [...launcherArgs(request)], + cwd: request.cwd, + clearEnv: true, + env: { PATH: launcher.path, HOME: '/tmp', SANDBOX_PROJECT: request.project, ...request.env }, + stdin: 'null', + stdout: 'piped', + stderr: 'piped', + signal: deadline, + }).output() + return { + code: output.code, + stdout: decoder.decode(output.stdout), + stderr: decoder.decode(output.stderr), + wallMs: Math.round(performance.now() - started), + pastDeadline: deadline.aborted, + } +} diff --git a/evals/ratstack-scorecard/src/journeys.ts b/evals/ratstack-scorecard/src/journeys.ts new file mode 100644 index 0000000..475e108 --- /dev/null +++ b/evals/ratstack-scorecard/src/journeys.ts @@ -0,0 +1,160 @@ +import { join, relative } from 'node:path' +import { measureStatic } from './families/static.ts' +import { arrayAt, at, stringAt } from './harness/decode.ts' +import { git, type Instrument, walkFiles } from './harness/instrument.ts' +import { runSandboxed } from './harness/sandbox.ts' + +interface MeasureStatic { + readonly kind: 'measure-static' + readonly ratstackRepo: string + readonly starterRepo: string + readonly failingTool: string | null +} + +interface JourneyEntry { + readonly id: string + readonly produce: MeasureStatic + readonly inputs: readonly string[] +} + +export interface LauncherRecord { + readonly id: string + readonly inputHash: string + readonly argv: readonly string[] + readonly code: number + readonly stdout: string + readonly stderr: string + readonly files: Readonly> + readonly egressLog: null + readonly wallMs: number +} + +const journeyEntry = (value: unknown): JourneyEntry => { + const produce = at(value, 'produce') + const failingTool = at(produce, 'failingTool') + return { + id: stringAt(value, ['id'], 'journey'), + produce: { + kind: 'measure-static', + ratstackRepo: stringAt(produce, ['ratstackRepo'], 'journey produce'), + starterRepo: stringAt(produce, ['starterRepo'], 'journey produce'), + failingTool: failingTool === null ? null : stringAt(produce, ['failingTool'], 'journey produce'), + }, + inputs: arrayAt(value, ['inputs'], 'journey').map((input) => stringAt({ input }, ['input'], 'journey input')), + } +} + +const encoder = new TextEncoder() + +const hex = (bytes: ArrayBuffer): string => + [...new Uint8Array(bytes)].map((b) => b.toString(16).padStart(2, '0')).join('') + +const filesUnder = async (root: string, path: string): Promise => { + const info = await Deno.stat(join(root, path)) + if (info.isFile) return [path] + return (await walkFiles(join(root, path))).map((file) => `${path}/${file}`) +} + +export const inputHashOf = async (root: string, inputs: readonly string[], launcher: string): Promise => { + const files = (await Promise.all(inputs.map((input) => filesUnder(root, input)))).flat().sort() + const parts = await Promise.all( + files.map(async (file) => + `${file}\0${hex(await crypto.subtle.digest('SHA-256', await Deno.readFile(join(root, file))))}` + ), + ) + return hex(await crypto.subtle.digest('SHA-256', encoder.encode([launcher, ...parts].join('\n')))) +} + +const gitRepoFrom = async (tree: string, target: string): Promise => { + await Deno.mkdir(target, { recursive: true }) + for (const file of await walkFiles(tree)) { + await Deno.mkdir(join(target, file, '..'), { recursive: true }) + await Deno.copyFile(join(tree, file), join(target, file)) + } + const identity = ['-c', 'user.name=journey', '-c', 'user.email=journey@invalid', '-c', 'commit.gpgsign=false'] + await git(target, ['init', '-q', '-b', 'main']) + await git(target, ['add', '-A']) + await git(target, [...identity, 'commit', '-q', '-m', 'fixture']) + return target +} + +const overlayWithFailingTool = async (instrument: Instrument, work: string, tool: string): Promise => { + const dir = join(work, 'instrument') + for (const file of await walkFiles(instrument.dir)) { + if (file.startsWith('node_modules/') || file.startsWith('journeys/__records__/')) continue + await Deno.mkdir(join(dir, file, '..'), { recursive: true }) + await Deno.copyFile(join(instrument.dir, file), join(dir, file)) + } + const failing = join(dir, 'src/tools', tool) + await Deno.remove(failing) + await Deno.writeTextFile(failing, 'process.exit(3)\n') + return dir +} + +const produce = async (instrument: Instrument, root: string, entry: JourneyEntry): Promise => { + const work = await Deno.makeTempDir({ prefix: `journey-${entry.id}-` }) + const started = performance.now() + try { + const ratstackSrc = await gitRepoFrom(join(root, entry.produce.ratstackRepo), join(work, 'sources/ratstack')) + const checkout = await gitRepoFrom(join(root, entry.produce.starterRepo), join(work, 'sources/starter-checkout')) + const project = join(work, 'project') + await Deno.mkdir(project) + const dir = entry.produce.failingTool === null + ? instrument.dir + : await overlayWithFailingTool(instrument, work, entry.produce.failingTool) + const subject: Instrument = { + ...instrument, + dir, + ratstackSrc, + checkout, + starterCommit: new TextDecoder().decode(await git(checkout, ['rev-parse', 'HEAD'])).trim(), + } + const result = await measureStatic(subject, project, ['ratstack', 'starter']) + return { + id: entry.id, + inputHash: await inputHashOf(root, entry.inputs, instrument.launcher.executable), + argv: ['scorecard', 'measure', '--family', 'static'], + code: 0, + stdout: '', + stderr: '', + files: { 'family.json': JSON.stringify(result) }, + egressLog: null, + wallMs: Math.round(performance.now() - started), + } + } finally { + await Deno.remove(work, { recursive: true }) + } +} + +export const runJourneys = async (instrument: Instrument, root: string): Promise => { + const manifest = arrayAt(JSON.parse(await Deno.readTextFile(join(root, 'journeys/manifest.json'))), [ + 'journeys', + ], 'journeys/manifest.json').map(journeyEntry) + const records = join(root, 'journeys/__records__') + await Deno.remove(records, { recursive: true }).catch(() => undefined) + await Deno.mkdir(records, { recursive: true }) + for (const entry of manifest) { + const record = await produce(instrument, root, entry) + await Deno.writeTextFile(join(records, `${entry.id}.json`), `${JSON.stringify(record, null, 2)}\n`) + console.error(`journeys: produced ${entry.id} in ${record.wallMs} ms`) + } + const install = await runSandboxed(instrument.launcher, { + project: root, + cwd: root, + command: ['pnpm', 'install', '--frozen-lockfile'], + pnpmStore: instrument.toolsStore, + deadlineMs: 10 * 60_000, + }) + if (install.code !== 0) throw new Error(`installing the instrument's test tools failed:\n${install.stderr}`) + const vitest = await runSandboxed(instrument.launcher, { + project: root, + cwd: root, + command: ['pnpm', 'exec', 'vitest', 'run'], + env: { SCORECARD_LAUNCHER: instrument.launcher.executable }, + deadlineMs: 30 * 60_000, + }) + await Deno.stdout.write(encoder.encode(vitest.stdout)) + await Deno.stderr.write(encoder.encode(vitest.stderr)) + console.error(`journeys: vitest exited ${vitest.code} (${relative(Deno.cwd(), root) || '.'})`) + return vitest.code +} diff --git a/evals/ratstack-scorecard/src/main.ts b/evals/ratstack-scorecard/src/main.ts new file mode 100644 index 0000000..2a1ca1f --- /dev/null +++ b/evals/ratstack-scorecard/src/main.ts @@ -0,0 +1,78 @@ +import { join } from 'node:path' +import { parseArgs } from 'node:util' +import { checkImports } from './check-imports.ts' +import { measureStatic } from './families/static.ts' +import { type Instrument, loadInstrument } from './harness/instrument.ts' +import { runJourneys } from './journeys.ts' +import type { FamilyResult, Side } from './model/cell.ts' + +const usage = [ + 'usage:', + ' scorecard measure --family [--side ratstack|starter] [--out ] [--checkout ]', + ' scorecard journeys [--checkout ]', + ' scorecard check [--checkout ] (import-graph rules, then journeys)', +].join('\n') + +const families: Readonly< + Record Promise> +> = { + static: measureStatic, +} + +const sidesOf = (side: string | undefined): readonly Side[] => { + if (side === undefined) return ['ratstack', 'starter'] + if (side === 'ratstack' || side === 'starter') return [side] + throw new Error(`unknown side ${side}\n${usage}`) +} + +const measure = async (args: readonly string[]): Promise => { + const { values } = parseArgs({ + args: [...args], + options: { + family: { type: 'string' }, + side: { type: 'string' }, + out: { type: 'string' }, + checkout: { type: 'string' }, + }, + strict: true, + }) + const run = families[values.family ?? ''] + if (run === undefined) throw new Error(`unknown family ${values.family}\n${usage}`) + const instrument = await loadInstrument(values.checkout) + const work = await Deno.makeTempDir({ prefix: `scorecard-${values.family}-` }) + try { + const result: FamilyResult = await run(instrument, work, sidesOf(values.side)) + const json = `${JSON.stringify(result, null, 2)}\n` + if (values.out === undefined) await Deno.stdout.write(new TextEncoder().encode(json)) + else await Deno.writeTextFile(values.out, json) + } finally { + await Deno.remove(work, { recursive: true }) + } +} + +const journeys = async (args: readonly string[]): Promise => { + const { values } = parseArgs({ args: [...args], options: { checkout: { type: 'string' } }, strict: true }) + const instrument = await loadInstrument(values.checkout) + Deno.exit(await runJourneys(instrument, join(instrument.checkout, 'evals/ratstack-scorecard'))) +} + +const check = async (args: readonly string[]): Promise => { + const { values } = parseArgs({ args: [...args], options: { checkout: { type: 'string' } }, strict: true }) + const instrument = await loadInstrument(values.checkout) + const root = join(instrument.checkout, 'evals/ratstack-scorecard') + const violations = await checkImports(root) + for (const violation of violations) console.error(`check-imports: ${violation}`) + if (violations.length > 0) Deno.exit(1) + console.error('check-imports: the host driver loads no third-party code; no model file imports the journey fixture') + Deno.exit(await runJourneys(instrument, root)) +} + +const commands: Readonly Promise>> = { measure, journeys, check } + +const [command, ...rest] = Deno.args +const handler = commands[command ?? ''] +if (handler === undefined) { + console.error(usage) + Deno.exit(2) +} +await handler(rest) diff --git a/evals/ratstack-scorecard/src/model/cell.ts b/evals/ratstack-scorecard/src/model/cell.ts index b6921d8..97f086d 100644 --- a/evals/ratstack-scorecard/src/model/cell.ts +++ b/evals/ratstack-scorecard/src/model/cell.ts @@ -65,6 +65,7 @@ export interface CellProvenance { readonly measuredAt: string readonly tools: Readonly> readonly liveCommit?: string + readonly detail?: Readonly> } export interface MeasuredCell { @@ -85,6 +86,29 @@ export interface Row { readonly flags: readonly Flag[] } +export interface DefinedRow { + readonly definition: RowDefinition + readonly hash: string +} + +export interface SideCell { + readonly id: string + readonly side: Side + readonly measured: MeasuredCell +} + +export interface RowFlag { + readonly id: string + readonly flag: Flag +} + +export interface FamilyResult { + readonly family: Family + readonly wallMs: number + readonly cells: readonly SideCell[] + readonly flags: readonly RowFlag[] +} + export type RowOutcome = | { readonly _tag: 'Held'; readonly id: string } | { readonly _tag: 'New'; readonly id: string } diff --git a/evals/ratstack-scorecard/src/model/directives.property.test.ts b/evals/ratstack-scorecard/src/model/directives.property.test.ts new file mode 100644 index 0000000..2f3c025 --- /dev/null +++ b/evals/ratstack-scorecard/src/model/directives.property.test.ts @@ -0,0 +1,50 @@ +import { fc, test } from '@fast-check/vitest' +import { describe } from 'vitest' +import { type DirectiveFamily, directivesIn, totalDirectives, vendoredRootsOf } from './directives.ts' + +const directiveText: Readonly>> = { + 'oxlint': fc.constantFrom('oxlint-disable', 'oxlint-disable-next-line', 'oxlint-disable-line'), + 'effect-diagnostics': fc.constantFrom('@effect-diagnostics', '@effect-diagnostics-next-line'), + 'typescript': fc.constantFrom('@ts-expect-error', '@ts-ignore', '@ts-nocheck'), + 'stryker': fc.constantFrom('Stryker disable', 'Stryker restore'), + 'eslint': fc.constantFrom('eslint-disable', 'eslint-disable-next-line', 'eslint-disable-line'), + 'biome': fc.constantFrom('biome-ignore', 'biome-ignore-all'), + 'dprint': fc.constantFrom('dprint-ignore', 'dprint-ignore-file'), + 'deno-lint': fc.constantFrom('deno-lint-ignore', 'deno-lint-ignore-file'), + 'coverage': fc.constantFrom('c8 ignore', 'istanbul ignore', 'v8 ignore'), +} + +const family = fc.constantFrom( + 'oxlint', + 'effect-diagnostics', + 'typescript', + 'stryker', + 'eslint', + 'biome', + 'dprint', + 'deno-lint', + 'coverage', +) + +const punctuation = fc.stringMatching(/^[0-9 ,.:;()*]{0,40}$/) + +describe('directivesIn', () => { + test.prop([family.chain((f) => fc.tuple(fc.constant(f), directiveText[f])), punctuation, punctuation])( + 'a comment carrying one directive counts once in its own family and nowhere else', + ([f, directive], before, after) => { + const tally = directivesIn(`${before} ${directive} ${after}`) + return tally[f] === 1 && totalDirectives(tally) === 1 + }, + ) +}) + +describe('vendoredRootsOf', () => { + test.prop([ + fc.stringMatching(/^[a-z]{1,8}(\/[a-z]{1,8}){0,2}\/$/), + fc.stringMatching(/^[a-z]{1,8}(\/[a-z]{1,8}){0,2}$/), + fc.stringMatching(/^\.[a-z]{1,4}$/), + ])( + 'a path under a declared root is excluded by that root, whatever its extension', + (root, rest, extension) => vendoredRootsOf(`${root}${rest}${extension}`, [root]).length === 1, + ) +}) diff --git a/evals/ratstack-scorecard/src/model/directives.ts b/evals/ratstack-scorecard/src/model/directives.ts new file mode 100644 index 0000000..29d5f36 --- /dev/null +++ b/evals/ratstack-scorecard/src/model/directives.ts @@ -0,0 +1,50 @@ +export type DirectiveFamily = + | 'oxlint' + | 'effect-diagnostics' + | 'typescript' + | 'stryker' + | 'eslint' + | 'biome' + | 'dprint' + | 'deno-lint' + | 'coverage' + +export type DirectiveTally = Readonly> + +const patterns: Readonly> = { + 'oxlint': /\boxlint-disable(?:-[a-z-]+)?\b/gu, + 'effect-diagnostics': /@effect-diagnostics(?:-next-line)?\b/gu, + 'typescript': /@ts-(?:expect-error|ignore|nocheck)\b/gu, + 'stryker': /\bStryker (?:disable|restore)\b/gu, + 'eslint': /\beslint-disable(?:-[a-z-]+)?\b/gu, + 'biome': /\bbiome-ignore(?:-[a-z-]+)?\b/gu, + 'dprint': /\bdprint-ignore(?:-[a-z-]+)?\b/gu, + 'deno-lint': /\bdeno-lint-ignore(?:-file)?\b/gu, + 'coverage': /\b(?:c8|istanbul|v8) ignore\b/gu, +} + +const tallyOf = (count: (family: DirectiveFamily) => number): DirectiveTally => ({ + 'oxlint': count('oxlint'), + 'effect-diagnostics': count('effect-diagnostics'), + 'typescript': count('typescript'), + 'stryker': count('stryker'), + 'eslint': count('eslint'), + 'biome': count('biome'), + 'dprint': count('dprint'), + 'deno-lint': count('deno-lint'), + 'coverage': count('coverage'), +}) + +export const directivesIn = (comment: string): DirectiveTally => + tallyOf((family) => [...comment.matchAll(patterns[family])].length) + +export const tallyDirectives = (comments: readonly string[]): DirectiveTally => + comments.map(directivesIn).reduce((sum, next) => tallyOf((family) => sum[family] + next[family]), tallyOf(() => 0)) + +export const totalDirectives = (tally: DirectiveTally): number => + Object.values(tally).reduce((sum, count) => sum + count, 0) + +export const isSourceFile = (path: string): boolean => /\.[cm]?[jt]sx?$/u.test(path) + +export const vendoredRootsOf = (path: string, roots: readonly string[]): readonly string[] => + roots.filter((root) => path.startsWith(root)) diff --git a/evals/ratstack-scorecard/src/model/lockfile.ts b/evals/ratstack-scorecard/src/model/lockfile.ts new file mode 100644 index 0000000..528166a --- /dev/null +++ b/evals/ratstack-scorecard/src/model/lockfile.ts @@ -0,0 +1,4 @@ +const packageIdOf = (key: string): string => key.replace(/^\//u, '').replace(/\(.*\)$/u, '') + +export const distinctPackages = (keys: readonly string[]): readonly string[] => + [...new Set(keys.map(packageIdOf))].sort() diff --git a/evals/ratstack-scorecard/src/model/scorecard-document.ts b/evals/ratstack-scorecard/src/model/scorecard-document.ts index 9c49d18..c2319b9 100644 --- a/evals/ratstack-scorecard/src/model/scorecard-document.ts +++ b/evals/ratstack-scorecard/src/model/scorecard-document.ts @@ -1,32 +1,17 @@ import type { + DefinedRow, DocumentProvenance, - Flag, MainBaseline, MeasuredCell, Row, - RowDefinition, + RowFlag, ScorecardDocument, Side, + SideCell, } from './cell.ts' import { compareWithMain } from './compare-with-main.workflow.ts' import { judgeRow } from './judge-row.workflow.ts' -export interface DefinedRow { - readonly definition: RowDefinition - readonly hash: string -} - -export interface SideCell { - readonly id: string - readonly side: Side - readonly measured: MeasuredCell -} - -export interface RowFlag { - readonly id: string - readonly flag: Flag -} - export type CellsByRow = Readonly>>>> export interface AssembleInput { diff --git a/evals/ratstack-scorecard/src/model/scorecard.arbitrary.ts b/evals/ratstack-scorecard/src/model/scorecard.arbitrary.ts index 411a908..0473681 100644 --- a/evals/ratstack-scorecard/src/model/scorecard.arbitrary.ts +++ b/evals/ratstack-scorecard/src/model/scorecard.arbitrary.ts @@ -76,7 +76,8 @@ export const cellProvenance = (side: Side): fc.Arbitrary => runner: text, measuredAt: fc.date({ noInvalidDate: true }).map((d) => d.toISOString()), tools: fc.dictionary(fc.string({ minLength: 1, maxLength: 20 }), fc.string({ maxLength: 20 })), - }) + detail: fc.dictionary(fc.string({ minLength: 1, maxLength: 20 }), fc.oneof(fc.string({ maxLength: 40 }), value)), + }, { requiredKeys: ['side', 'commit', 'instrumentHash', 'nixpkgsRev', 'runner', 'measuredAt', 'tools'] }) export const measuredCell = (row: RowDefinition, side: Side): fc.Arbitrary => fc.record({ cell: cell(row), provenance: cellProvenance(side) }) diff --git a/evals/ratstack-scorecard/src/model/verify-citation.workflow.property.test.ts b/evals/ratstack-scorecard/src/model/verify-citation.workflow.property.test.ts new file mode 100644 index 0000000..4c71f35 --- /dev/null +++ b/evals/ratstack-scorecard/src/model/verify-citation.workflow.property.test.ts @@ -0,0 +1,26 @@ +import { fc, test } from '@fast-check/vitest' +import { describe } from 'vitest' +import { verifyCitation } from './verify-citation.workflow.ts' + +const line = fc.stringMatching(/^[a-z ]{0,40}$/) +const claim = fc.stringMatching(/^[A-Z][A-Z ]{8,30}[A-Z]$/) + +const fileWith = (lines: readonly string[], at: number, text: string): string => + [...lines.slice(0, at), text, ...lines.slice(at)].join('\n') + +const checkAt = (source: string, text: string, line: number) => + verifyCitation({ citation: { file: 'f.ts', lines: [line, line], text }, source })._tag + +describe('verifyCitation', () => { + test.prop([fc.array(line, { minLength: 2, maxLength: 30 }), claim, fc.nat()])( + 'a citation verifies on the line holding the text and is contradicted one line either side', + (lines, text, seed) => { + const at = 1 + (seed % (lines.length - 1)) + const source = fileWith(lines, at, text) + const cited = at + 1 + return checkAt(source, text, cited) === 'Verified' && + checkAt(source, text, cited - 1) === 'Contradicted' && + checkAt(source, text, cited + 1) === 'Contradicted' + }, + ) +}) diff --git a/evals/ratstack-scorecard/src/model/verify-citation.workflow.ts b/evals/ratstack-scorecard/src/model/verify-citation.workflow.ts new file mode 100644 index 0000000..4f03c72 --- /dev/null +++ b/evals/ratstack-scorecard/src/model/verify-citation.workflow.ts @@ -0,0 +1,19 @@ +import type { Citation, CitationCheck } from './cell.ts' +import { firstRule } from './dispatch.ts' + +export interface VerifyCitationInput { + readonly citation: Citation + readonly source: string +} + +const normalized = (text: string): string => text.replace(/\s+/gu, ' ').trim() + +export const verifyCitation = (input: VerifyCitationInput): CitationCheck => { + const cited = normalized( + input.source.split(/\r?\n/u).slice(input.citation.lines[0] - 1, input.citation.lines[1]).join('\n'), + ) + return firstRule( + [[cited.includes(normalized(input.citation.text)), () => ({ _tag: 'Verified' })]], + () => ({ _tag: 'Contradicted', found: cited.slice(0, 200) }), + ) +} diff --git a/evals/ratstack-scorecard/src/sides/ratstack.ts b/evals/ratstack-scorecard/src/sides/ratstack.ts new file mode 100644 index 0000000..cf40926 --- /dev/null +++ b/evals/ratstack-scorecard/src/sides/ratstack.ts @@ -0,0 +1,13 @@ +import type { SideAdapter } from './side-adapter.ts' + +export const ratstack: SideAdapter = { + side: 'ratstack', + lockfile: 'pnpm-lock.yaml', + vendored: [ + { + root: 'tools/oxlint/anti-slop/', + reason: 'copied from dmmulroy/anti-slop (tools/oxlint/anti-slop/UPSTREAM.md); rat-stack’s own ledger excludes it', + }, + { root: 'vendor/', reason: 'vendored dependency sources (vendor/README.md); rat-stack’s own ledger excludes it' }, + ], +} diff --git a/evals/ratstack-scorecard/src/sides/side-adapter.ts b/evals/ratstack-scorecard/src/sides/side-adapter.ts new file mode 100644 index 0000000..23278e5 --- /dev/null +++ b/evals/ratstack-scorecard/src/sides/side-adapter.ts @@ -0,0 +1,12 @@ +import type { Side } from '../model/cell.ts' + +export interface VendoredRoot { + readonly root: string + readonly reason: string +} + +export interface SideAdapter { + readonly side: Side + readonly lockfile: string + readonly vendored: readonly VendoredRoot[] +} diff --git a/evals/ratstack-scorecard/src/sides/starter.ts b/evals/ratstack-scorecard/src/sides/starter.ts new file mode 100644 index 0000000..34ed8a9 --- /dev/null +++ b/evals/ratstack-scorecard/src/sides/starter.ts @@ -0,0 +1,10 @@ +import type { SideAdapter } from './side-adapter.ts' + +export const starter: SideAdapter = { + side: 'starter', + lockfile: 'pnpm-lock.yaml', + vendored: [ + { root: 'repos/', reason: 'git subtrees vendored from other repositories (subtrees.toml)' }, + { root: 'evals/', reason: 'the scorecard instrument itself, which grades the starter and is not starter code' }, + ], +} diff --git a/evals/ratstack-scorecard/src/tools/extract-comments.mjs b/evals/ratstack-scorecard/src/tools/extract-comments.mjs new file mode 100644 index 0000000..6ccb164 --- /dev/null +++ b/evals/ratstack-scorecard/src/tools/extract-comments.mjs @@ -0,0 +1,19 @@ +import { readFile, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { parseSync } from 'oxc-parser' + +const [request, output] = process.argv.slice(2) +const { root, files } = JSON.parse(await readFile(request, 'utf8')) + +const parsed = [] +for (const file of files) { + const source = await readFile(join(root, file), 'utf8') + const result = parseSync(file, source) + parsed.push({ + path: file, + comments: result.comments.map((comment) => comment.value), + errors: result.errors.length, + }) +} + +await writeFile(output, JSON.stringify({ parser: 'oxc-parser', files: parsed })) diff --git a/evals/ratstack-scorecard/src/tools/parse-lockfile.mjs b/evals/ratstack-scorecard/src/tools/parse-lockfile.mjs new file mode 100644 index 0000000..8d58bc9 --- /dev/null +++ b/evals/ratstack-scorecard/src/tools/parse-lockfile.mjs @@ -0,0 +1,14 @@ +import { readFile, writeFile } from 'node:fs/promises' +import { parseAllDocuments } from 'yaml' + +const [lockfile, output] = process.argv.slice(2) +const documents = parseAllDocuments(await readFile(lockfile, 'utf8')).map((document) => { + const value = document.toJS() ?? {} + return { + lockfileVersion: String(value.lockfileVersion ?? ''), + packages: Object.keys(value.packages ?? {}), + errors: document.errors.length, + } +}) + +await writeFile(output, JSON.stringify({ parser: 'yaml', documents })) diff --git a/package.json b/package.json index 5a60a0e..d158426 100644 --- a/package.json +++ b/package.json @@ -20,11 +20,12 @@ "scripts": { "attw": "turbo --concurrency=${TURBO_CONCURRENCY:-50%} attw", "build": "turbo --concurrency=${TURBO_CONCURRENCY:-50%} build", - "check:ci": "s=0; pnpm format:check || s=1; TURBO_CONCURRENCY=${TURBO_CONCURRENCY:-100%} pnpm gate:tasks || s=1; pnpm gate:dist || s=1; pnpm mutation || s=1; exit $s", + "check:ci": "s=0; pnpm format:check || s=1; TURBO_CONCURRENCY=${TURBO_CONCURRENCY:-100%} pnpm gate:tasks || s=1; pnpm gate:dist || s=1; pnpm scorecard:check || s=1; pnpm mutation || s=1; exit $s", "clean": "turbo clean", "format": "./bin/dprint fmt", "format:check": "./bin/dprint check", "gate:dist": "turbo --concurrency=${TURBO_CONCURRENCY:-100%} build", + "scorecard:check": "nix run ./evals/ratstack-scorecard#scorecard -- check", "gate:tasks": "turbo --concurrency=${TURBO_CONCURRENCY:-50%} --continue lint typecheck typecheck:node test", "lint": "turbo --concurrency=${TURBO_CONCURRENCY:-50%} lint", "mutation": "turbo --concurrency=${TURBO_CONCURRENCY:-50%} mutation",