From 2724ebea771c095efe9d6f1a571590272eea6ca2 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Mon, 5 Oct 2026 22:38:59 +0000 Subject: [PATCH 1/8] feat(repo): pin rat-stack and measure the static scorecard rows The instrument gets its own flake: nixpkgs and the pnpm-release-management sandbox launcher pinned by rev, rat-stack fetched at ratstack.pin.json, and an offline pnpm store for the instrument's npm tools built by the launcher's fixed-output fetcher. The scorecard package runs the Deno orchestrator with those paths baked in. The static family measures M23 (suppression directives in tracked source, counted by parsing comments with oxc-parser) and M28 (distinct name@version in the lockfile, read with the yaml parser across both lockfile documents) for both sides, three runs each. Every tool runs in the launcher; the orchestrator only classifies the comments and package keys the tools return. verifyCitation re-checks the cited rat-stack text that M12 relies on --- evals/ratstack-scorecard/README.md | 13 +- evals/ratstack-scorecard/deno.json | 2 +- evals/ratstack-scorecard/flake.nix | 28 +- .../__fixtures__/lockfiles/pnpm11.yaml | 73 ++++++ .../__fixtures__/lockfiles/pnpm12.yaml | 78 ++++++ .../journeys/static.journey.test.ts | 68 +++++ evals/ratstack-scorecard/package.json | 4 + evals/ratstack-scorecard/pnpm-lock.yaml | 242 +++++++++++++++++- evals/ratstack-scorecard/ratstack.pin.json | 6 + .../ratstack-scorecard/scorecard.schema.json | 6 +- .../ratstack-scorecard/src/families/static.ts | 137 ++++++++++ .../ratstack-scorecard/src/harness/decode.ts | 41 +++ .../src/harness/instrument.ts | 145 +++++++++++ .../ratstack-scorecard/src/harness/sandbox.ts | 54 ++++ evals/ratstack-scorecard/src/main.ts | 53 ++++ evals/ratstack-scorecard/src/model/cell.ts | 24 ++ .../src/model/directives.property.test.ts | 50 ++++ .../src/model/directives.ts | 50 ++++ .../ratstack-scorecard/src/model/lockfile.ts | 4 + .../src/model/scorecard-document.ts | 21 +- .../src/model/scorecard.arbitrary.ts | 3 +- .../verify-citation.workflow.property.test.ts | 26 ++ .../src/model/verify-citation.workflow.ts | 19 ++ .../ratstack-scorecard/src/sides/ratstack.ts | 13 + .../src/sides/side-adapter.ts | 12 + evals/ratstack-scorecard/src/sides/starter.ts | 10 + .../src/tools/extract-comments.mjs | 19 ++ .../src/tools/parse-lockfile.mjs | 14 + 28 files changed, 1180 insertions(+), 35 deletions(-) create mode 100644 evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm11.yaml create mode 100644 evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm12.yaml create mode 100644 evals/ratstack-scorecard/journeys/static.journey.test.ts create mode 100644 evals/ratstack-scorecard/ratstack.pin.json create mode 100644 evals/ratstack-scorecard/src/families/static.ts create mode 100644 evals/ratstack-scorecard/src/harness/decode.ts create mode 100644 evals/ratstack-scorecard/src/harness/instrument.ts create mode 100644 evals/ratstack-scorecard/src/harness/sandbox.ts create mode 100644 evals/ratstack-scorecard/src/main.ts create mode 100644 evals/ratstack-scorecard/src/model/directives.property.test.ts create mode 100644 evals/ratstack-scorecard/src/model/directives.ts create mode 100644 evals/ratstack-scorecard/src/model/lockfile.ts create mode 100644 evals/ratstack-scorecard/src/model/verify-citation.workflow.property.test.ts create mode 100644 evals/ratstack-scorecard/src/model/verify-citation.workflow.ts create mode 100644 evals/ratstack-scorecard/src/sides/ratstack.ts create mode 100644 evals/ratstack-scorecard/src/sides/side-adapter.ts create mode 100644 evals/ratstack-scorecard/src/sides/starter.ts create mode 100644 evals/ratstack-scorecard/src/tools/extract-comments.mjs create mode 100644 evals/ratstack-scorecard/src/tools/parse-lockfile.mjs diff --git a/evals/ratstack-scorecard/README.md b/evals/ratstack-scorecard/README.md index e701639..fb3caae 100644 --- a/evals/ratstack-scorecard/README.md +++ b/evals/ratstack-scorecard/README.md @@ -19,7 +19,16 @@ The scorecard measures the starter against [rat-stack](https://github.com/joelho ## Running it -Everything that loads third-party code runs inside the sandbox launcher from `systemfsoftware/pnpm-release-management` (`packages..sandbox`), with this directory as the sandbox project. The instrument's own flake (`flake.nix`) pins nixpkgs (pnpm 12.9.0, Node 24) and the launcher, and builds the tools' pnpm store from `pnpm-lock.yaml` as a fixed-output derivation (`tools-store`). The install is offline from that store; the sandbox gets no network at all. +Everything that loads third-party code runs inside the sandbox launcher from `systemfsoftware/pnpm-release-management` (`packages..sandbox`). + +The instrument has its own flake (`flake.nix`, pinned nixpkgs and launcher). Its `scorecard` package bundles Deno, the launcher, the rat-stack source at `ratstack.pin.json`, and the offline pnpm store for the tools (`tools-store`). The family runners install the tools from that store inside the sandbox, so nothing reaches the registry at measurement time. + +```sh +scorecard=$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#scorecard) +$scorecard/bin/scorecard measure --family static --out static.json +``` + +The instrument's own tests run inside the launcher against the same offline store: ```sh cd evals/ratstack-scorecard @@ -27,4 +36,4 @@ nix develop --command sh -c 'SANDBOX_PROJECT=$PWD sandbox --pnpm-store "$SANDBOX nix develop --command sh -c 'SANDBOX_PROJECT=$PWD sandbox -- pnpm vitest run' ``` -The decision modules (`src/model/*.workflow.ts`) and the orchestrator import only Deno APIs, `node:` builtins and each other, so `deno check src/` type-checks them without any third-party code. +`src/main.ts` and everything it imports use only Deno APIs, `node:` builtins and each other, so `DENO_NO_PACKAGE_JSON=1 deno check src/` type-checks the orchestrator and the decision core without third-party code. The Node scripts in `src/tools/` are the only code that loads npm packages, and they only ever run inside the launcher. diff --git a/evals/ratstack-scorecard/deno.json b/evals/ratstack-scorecard/deno.json index 0b79998..927072f 100644 --- a/evals/ratstack-scorecard/deno.json +++ b/evals/ratstack-scorecard/deno.json @@ -4,6 +4,6 @@ "exactOptionalPropertyTypes": true, "noUncheckedIndexedAccess": true }, - "exclude": ["node_modules/", "**/*.test.ts", "**/*.arbitrary.ts", "vitest.config.ts"], + "exclude": ["node_modules/", "src/tools/", "journeys/", "**/*.test.ts", "**/*.arbitrary.ts", "vitest.config.ts"], "lock": false } diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index f54425e..bd63b3a 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -13,6 +13,7 @@ let systems = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ]; forEachSystem = fn: nixpkgs.lib.genAttrs systems (system: fn nixpkgs.legacyPackages.${system}); + pin = builtins.fromJSON (builtins.readFile ./ratstack.pin.json); in { packages = forEachSystem (pkgs: @@ -24,11 +25,34 @@ src = self; pname = "ratstack-scorecard"; pnpm = pkgs.pnpm_12; - hash = "sha256-TylxLEQflTlKx6QDk9mFlBOEInUvvBBeOVFUYCURmYo="; + hash = "sha256-6AG6SAB7vZR3kLGU7ujHxBwBqD+cSrwKeTl0qLyz9ko="; }).pnpm-store; + ratstack-src = pkgs.fetchFromGitHub { + inherit (pin) owner repo; + rev = pin.commit; + hash = pin.narHash; + }; + scorecard = pkgs.writeShellApplication { + name = "scorecard"; + runtimeInputs = [ pkgs.deno pkgs.git ]; + text = '' + export SCORECARD_INSTRUMENT=${self} + export SCORECARD_SANDBOX=${sandbox}/bin/sandbox + export SCORECARD_TOOLS_STORE=${tools-store} + export SCORECARD_RATSTACK_SRC=${ratstack-src} + export SCORECARD_TOOL_PATH=${pkgs.lib.makeBinPath [ pkgs.nodejs_24 pkgs.pnpm_12 pkgs.coreutils ]} + export SCORECARD_NODE_VERSION=${pkgs.nodejs_24.version} + export SCORECARD_PNPM_VERSION=${pkgs.pnpm_12.version} + export DENO_NO_PACKAGE_JSON=1 + exec deno run --no-config --allow-read --allow-write --allow-env --allow-sys=hostname \ + --allow-run=git,${sandbox}/bin/sandbox \ + ${self}/src/main.ts "$@" + ''; + }; in { - inherit sandbox tools-store; + inherit sandbox tools-store ratstack-src scorecard; + default = scorecard; }); devShells = forEachSystem (pkgs: diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm11.yaml b/evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm11.yaml new file mode 100644 index 0000000..16c761c --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm11.yaml @@ -0,0 +1,73 @@ +--- +lockfileVersion: "9.0" + +importers: + .: + configDependencies: {} + packageManagerDependencies: + "@pnpm/exe": + specifier: 11.3.0 + version: 11.3.0 + pnpm: + specifier: 11.3.0 + version: 11.3.0 + +packages: + "@pnpm/exe@11.3.0": + resolution: { + integrity: sha512-1ItrG3GdA8HC7IUMy79SmYqynjjfwXtIMlbpx9MzrU3ZXMYMfw3yuwjIXW7Aw2z0rRxxa2KtTYcLxqjIaVjUmg==, + } + hasBin: true + + pnpm@11.3.0: + resolution: { integrity: sha512-AAAA } + hasBin: true + +snapshots: + "@pnpm/exe@11.3.0": {} + + pnpm@11.3.0: {} + +--- +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + .: + dependencies: + effect: + specifier: 4.0.0 + version: 4.0.0 + devDependencies: + "@effect/vitest": + specifier: 4.0.0 + version: 4.0.0(effect@4.0.0)(vitest@5.0.3) + vitest: + specifier: 5.0.3 + version: 5.0.3 + +packages: + "@effect/vitest@4.0.0": + resolution: { integrity: sha512-BBBB } + peerDependencies: + effect: ^4.0.0 + vitest: ^5.0.0 + + effect@4.0.0: + resolution: { integrity: sha512-CCCC } + + vitest@5.0.3: + resolution: { integrity: sha512-DDDD } + +snapshots: + "@effect/vitest@4.0.0(effect@4.0.0)(vitest@5.0.3)": + dependencies: + effect: 4.0.0 + vitest: 5.0.3 + + effect@4.0.0: {} + + vitest@5.0.3: {} diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm12.yaml b/evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm12.yaml new file mode 100644 index 0000000..df58f0f --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/lockfiles/pnpm12.yaml @@ -0,0 +1,78 @@ +--- +lockfileVersion: "9.0" + +importers: + .: + configDependencies: {} + packageManagerDependencies: + pnpm: + specifier: 12.4.2 + version: 12.4.2 + +packages: + "@pnpm/exe.linux-x64@12.4.2": + resolution: { integrity: sha512-EEEE } + cpu: [x64] + os: [linux] + + pnpm@12.4.2: + resolution: { integrity: sha512-FFFF } + hasBin: true + +snapshots: + "@pnpm/exe.linux-x64@12.4.2": + optional: true + + pnpm@12.4.2: + optionalDependencies: + "@pnpm/exe.linux-x64": 12.4.2 + +--- +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +catalogs: + default: + effect: + specifier: ^4.0.0 + version: 4.0.0 + +importers: + .: + dependencies: + effect: + specifier: "catalog:" + version: 4.0.0 + devDependencies: + "@effect/vitest": + specifier: 4.0.0 + version: 4.0.0(effect@4.0.0)(vitest@5.0.3) + vitest: + specifier: 5.0.3 + version: 5.0.3 + +packages: + "@effect/vitest@4.0.0": + resolution: { integrity: sha512-BBBB } + peerDependencies: + effect: ^4.0.0 + vitest: ^5.0.0 + + effect@4.0.0: + resolution: { integrity: sha512-CCCC } + + vitest@5.0.3: + resolution: { integrity: sha512-DDDD } + +snapshots: + "@effect/vitest@4.0.0(effect@4.0.0)(vitest@5.0.3)": + dependencies: + effect: 4.0.0 + vitest: 5.0.3 + + effect@4.0.0: {} + + vitest@5.0.3: {} diff --git a/evals/ratstack-scorecard/journeys/static.journey.test.ts b/evals/ratstack-scorecard/journeys/static.journey.test.ts new file mode 100644 index 0000000..992e946 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/static.journey.test.ts @@ -0,0 +1,68 @@ +import { execFile } from 'node:child_process' +import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { promisify } from 'node:util' +import { describe, expect, test } from 'vitest' +import { isSourceFile, tallyDirectives, totalDirectives, vendoredRootsOf } from '../src/model/directives.ts' +import { distinctPackages } from '../src/model/lockfile.ts' + +const run = promisify(execFile) +const instrument = join(import.meta.dirname, '..') +const fixtures = join(import.meta.dirname, '__fixtures__') + +const debtTree: Readonly> = { + 'src/flagged.ts': [ + '// oxlint-disable-next-line no-console -- the one directive in this tree', + "const text = 'oxlint-disable-next-line written inside a string'", + 'const pattern = /@ts-ignore/u', + 'const template = `eslint-disable ${text}`', + 'export { pattern, template, text }', + '', + ].join('\n'), + 'vendor/copied.ts': '// @ts-nocheck vendored code keeps its own directives\nexport const copied = 1\n', + 'README.md': '\n', +} + +const writeTree = async (root: string, files: Readonly>): Promise => { + for (const [path, content] of Object.entries(files)) { + await mkdir(dirname(join(root, path)), { recursive: true }) + await writeFile(join(root, path), content) + } +} + +const tool = async (name: string, args: readonly string[]): Promise => { + const output = join(await mkdtemp(join(tmpdir(), 'journey-')), 'out.json') + await run('node', [join(instrument, 'src/tools', name), ...args, output], { cwd: instrument }) + return JSON.parse(await readFile(output, 'utf8')) +} + +describe('static family through the real tools (J2)', () => { + test('only the directive in a comment counts; strings, regex literals, templates and vendored files do not', async () => { + const root = await mkdtemp(join(tmpdir(), 'debt-tree-')) + await writeTree(root, debtTree) + const files = Object.keys(debtTree) + .filter(isSourceFile) + .filter((file) => vendoredRootsOf(file, ['vendor/']).length === 0) + const request = join(root, 'request.json') + await writeFile(request, JSON.stringify({ root, files })) + const extracted = await tool('extract-comments.mjs', [request]) + const comments = (extracted as { files: { comments: string[] }[] }).files.flatMap((file) => file.comments) + const tally = tallyDirectives(comments) + expect(files).toEqual(['src/flagged.ts']) + expect(tally.oxlint).toBe(1) + expect(totalDirectives(tally)).toBe(1) + }) + + test('pnpm 11 and pnpm 12 lockfiles list their packages across both YAML documents, without peer suffixes', async () => { + const packagesOf = async (fixture: string) => { + const parsed = await tool('parse-lockfile.mjs', [join(fixtures, 'lockfiles', fixture)]) + return distinctPackages((parsed as { documents: { packages: string[] }[] }).documents.flatMap((d) => d.packages)) + } + const application = ['@effect/vitest@4.0.0', 'effect@4.0.0', 'vitest@5.0.3'] + expect(await packagesOf('pnpm11.yaml')).toEqual([...application, '@pnpm/exe@11.3.0', 'pnpm@11.3.0'].sort()) + expect(await packagesOf('pnpm12.yaml')).toEqual( + [...application, '@pnpm/exe.linux-x64@12.4.2', 'pnpm@12.4.2'].sort(), + ) + }) +}) diff --git a/evals/ratstack-scorecard/package.json b/evals/ratstack-scorecard/package.json index 7f59206..884aeb1 100644 --- a/evals/ratstack-scorecard/package.json +++ b/evals/ratstack-scorecard/package.json @@ -2,6 +2,10 @@ "name": "@starter/ratstack-scorecard", "private": true, "type": "module", + "dependencies": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + }, "devDependencies": { "@fast-check/vitest": "0.5.0", "ajv": "8.20.0", diff --git a/evals/ratstack-scorecard/pnpm-lock.yaml b/evals/ratstack-scorecard/pnpm-lock.yaml index 05f1fa0..665ab06 100644 --- a/evals/ratstack-scorecard/pnpm-lock.yaml +++ b/evals/ratstack-scorecard/pnpm-lock.yaml @@ -7,10 +7,17 @@ settings: importers: .: + dependencies: + oxc-parser: + specifier: 0.152.0 + version: 0.152.0 + yaml: + specifier: 2.9.1 + version: 2.9.1 devDependencies: '@fast-check/vitest': specifier: 0.5.0 - version: 0.5.0(vitest@5.0.3(vite@8.3.2)) + version: 0.5.0(vitest@5.0.3(vite@8.3.2(yaml@2.9.1))) ajv: specifier: 8.20.0 version: 8.20.0 @@ -19,7 +26,7 @@ importers: version: 4.10.2 vitest: specifier: 5.0.3 - version: 5.0.3(vite@8.3.2) + version: 5.0.3(vite@8.3.2(yaml@2.9.1)) packages: @@ -38,6 +45,128 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@oxc-parser/binding-android-arm-eabi@0.152.0': + resolution: {integrity: sha512-393VtirINkae4XHBTkq/yQxhOwEr4V+r0et37P2fp+sYZSdYgZTRDPo5Um8vBh0QjMZ+RmaMy4PP0qwr4cTMwA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [android] + + '@oxc-parser/binding-android-arm64@0.152.0': + resolution: {integrity: sha512-iYFcExATJNZ0rZj0o02THbD1Nc9jAW0nVgIQPee6ejbSASpbZXlIMw8jxtpzaG14JQTWwGSZ6U5dEtRJXE6V3Q==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + + '@oxc-parser/binding-darwin-arm64@0.152.0': + resolution: {integrity: sha512-fnU+DfUzi5/rFy0LzmipYRz06PAUUkOS8kCJQ3Eg6oqlunC1XZXoffFqxmdL51RmcbYtUG3P8DVG1U5fckoa2A==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + + '@oxc-parser/binding-darwin-x64@0.152.0': + resolution: {integrity: sha512-Y3U2okryWTs7hDld6UhjJwLNf8/do+x4g4C+1U5SiGwrTLwbO83WJYJA8+l9iFSilWBgnfpd/+U76tfeNxUHzA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + + '@oxc-parser/binding-freebsd-x64@0.152.0': + resolution: {integrity: sha512-XkhkRrrPffEw+ZLv/d/5pEni0buxinm/E/N6fLbJCunD1DBbeCTtVP23ESV+4CWCWdSz397asks1ViGslpmWiQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + + '@oxc-parser/binding-linux-arm-gnueabihf@0.152.0': + resolution: {integrity: sha512-MLzsRRbdfmdzo6JEhbC1yKU9ItfvM8tEf18HM32jCiNYZEz6J3bRBRl3cTkKR0vBdrk9ZVlcARXtwHccs+epDA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@oxc-parser/binding-linux-arm-musleabihf@0.152.0': + resolution: {integrity: sha512-HHerTEJz7Iy10Ji6BGsk4gqG6PU0UzFuBu91jIEOX89FnrI5ECWSEr7BLpkYQhp7U0QFtMnBhNfQKI0hYZK78w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + + '@oxc-parser/binding-linux-arm64-gnu@0.152.0': + resolution: {integrity: sha512-9Bb8GyzZ2uSLzRDZ27usaKokr07mxiBe2dDF5b6bd2janmaJTT1gL51UfP7AU30yFiXrJvcRyFKREAgKwZYs0w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-arm64-musl@0.152.0': + resolution: {integrity: sha512-om7iDMUroI74A/lfRRWY9/Ev3VWrVsrUZ4tyvyh8raBey/sdcfyAD7tYWc+iykAiotnmftyKTpUKFr1QnaBrAQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@oxc-parser/binding-linux-ppc64-gnu@0.152.0': + resolution: {integrity: sha512-Lf1OL2lMTqwDMFC6jnRb30B+WYHKV0XT+bPHcFM6xHtNVwxpobiTzUIvKH6a0LWpvfwyru8Yu6zCGHqqFHH+gg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-riscv64-gnu@0.152.0': + resolution: {integrity: sha512-BYWSMQK+YJ9ZILStPcmmiSVq8KEG9Qdq/c4BHPC3rUEq/fEDCVjUIwdj5ZzjgZ9P9is9PKu0tcAXzq1ZXg4urQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-riscv64-musl@0.152.0': + resolution: {integrity: sha512-d2LqHNNNqdH185sRK5dPKWZu201JrNu9/CRFuVEuT+ht7n/QaVblEMWT9MALQPrDWYvhG10P+NSKJFquDWZ09w==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@oxc-parser/binding-linux-s390x-gnu@0.152.0': + resolution: {integrity: sha512-8gBeHt5rA/8+pKhHl+zLggfa3L/V/L2at96p7ty6pBE7id6lFO19nFUc15u2PcjC4OYeYK465MKy1fhhbpFIGw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-x64-gnu@0.152.0': + resolution: {integrity: sha512-ltgrSF6L+yPTVsIdaAyqKq3zTqzj/1VlNbxWZuxAe8flR3sWdyNsKvdXukoO8/ucJbLickhmieRBge+o/e8Ugw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@oxc-parser/binding-linux-x64-musl@0.152.0': + resolution: {integrity: sha512-LcMp6BvrBnewSfdkLHoSqidhoAq4syKOTBV5TYHWd+2VnGkfD/nPRRVIzTBt/Lkbt7pfOFDGtkiTntFS0p6sQQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@oxc-parser/binding-openharmony-arm64@0.152.0': + resolution: {integrity: sha512-jbhoMCslVxzRCRNQ8TSHxjYdSKRvk8O6ciLVDkahJvmyHDzBqba/bMQHM6ULKj/rgHNNbSzlIiakdx8SEfkJzA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + + '@oxc-parser/binding-win32-arm64-msvc@0.152.0': + resolution: {integrity: sha512-JZYVlhv36mr/nertnRB7vchJECOBb1Lz6lltalILsB+o9M/is+H6BEDRLQKdqA4JKs6NjJszo46+iM5sfwj6jQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + + '@oxc-parser/binding-win32-ia32-msvc@0.152.0': + resolution: {integrity: sha512-1X9oganvVsunRH0lH10UU05N+H9atpX6WsOybByUjwnfcd3h/0tYwvuv16XuHglrIUOnxV2TodbLVEwwafd64g==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ia32] + os: [win32] + + '@oxc-parser/binding-win32-x64-msvc@0.152.0': + resolution: {integrity: sha512-sw0M80/dn9rcpu+Rqqph/C+MjvDDce6MlXmFjsNQtPEymha0eJy3ml6+foSUTqFfpDIDyHZZ4eoJ43IoMe8bJw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + '@oxc-project/types@0.152.0': resolution: {integrity: sha512-oM/5rLBm2tPkg0iBgkH/FOeR3PCDpY19GTgAZjMFM8h9WI9VW7cLgzp6nwtarYKmovavIQZ+Fe/RKX/8C8O/Rw==} @@ -301,6 +430,10 @@ packages: resolution: {integrity: sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==} engines: {node: '>=12.20.0'} + oxc-parser@0.152.0: + resolution: {integrity: sha512-ZztsZgaCWmPeKYMdoTT5e0oV/oxD1NeLWNR3kAtr/G707cUPYaaozRlVDHLCGnszHJRszN9mJwmPNpqgwloC0A==} + engines: {node: ^20.19.0 || >=22.12.0} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -432,12 +565,17 @@ packages: engines: {node: '>=20.11'} hasBin: true + yaml@2.9.1: + resolution: {integrity: sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==} + engines: {node: '>= 14.6'} + hasBin: true + snapshots: - '@fast-check/vitest@0.5.0(vitest@5.0.3(vite@8.3.2))': + '@fast-check/vitest@0.5.0(vitest@5.0.3(vite@8.3.2(yaml@2.9.1)))': dependencies: fast-check: 4.10.2 - vitest: 5.0.3(vite@8.3.2) + vitest: 5.0.3(vite@8.3.2(yaml@2.9.1)) '@jridgewell/resolve-uri@3.1.2': {} @@ -448,6 +586,63 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.6.0 + '@oxc-parser/binding-android-arm-eabi@0.152.0': + optional: true + + '@oxc-parser/binding-android-arm64@0.152.0': + optional: true + + '@oxc-parser/binding-darwin-arm64@0.152.0': + optional: true + + '@oxc-parser/binding-darwin-x64@0.152.0': + optional: true + + '@oxc-parser/binding-freebsd-x64@0.152.0': + optional: true + + '@oxc-parser/binding-linux-arm-gnueabihf@0.152.0': + optional: true + + '@oxc-parser/binding-linux-arm-musleabihf@0.152.0': + optional: true + + '@oxc-parser/binding-linux-arm64-gnu@0.152.0': + optional: true + + '@oxc-parser/binding-linux-arm64-musl@0.152.0': + optional: true + + '@oxc-parser/binding-linux-ppc64-gnu@0.152.0': + optional: true + + '@oxc-parser/binding-linux-riscv64-gnu@0.152.0': + optional: true + + '@oxc-parser/binding-linux-riscv64-musl@0.152.0': + optional: true + + '@oxc-parser/binding-linux-s390x-gnu@0.152.0': + optional: true + + '@oxc-parser/binding-linux-x64-gnu@0.152.0': + optional: true + + '@oxc-parser/binding-linux-x64-musl@0.152.0': + optional: true + + '@oxc-parser/binding-openharmony-arm64@0.152.0': + optional: true + + '@oxc-parser/binding-win32-arm64-msvc@0.152.0': + optional: true + + '@oxc-parser/binding-win32-ia32-msvc@0.152.0': + optional: true + + '@oxc-parser/binding-win32-x64-msvc@0.152.0': + optional: true + '@oxc-project/types@0.152.0': {} '@rolldown/binding-android-arm-eabi@1.2.12': @@ -506,14 +701,14 @@ snapshots: '@types/estree@1.0.9': {} - '@vitest/mocker@5.0.3(vite@8.3.2)': + '@vitest/mocker@5.0.3(vite@8.3.2(yaml@2.9.1))': dependencies: '@jridgewell/trace-mapping': 0.3.31 '@vitest/spy': 5.0.3 estree-walker: 3.0.3 magic-string: 1.4.2 optionalDependencies: - vite: 8.3.2 + vite: 8.3.2(yaml@2.9.1) '@vitest/spy@5.0.3': {} @@ -612,6 +807,30 @@ snapshots: obug@2.2.1: {} + oxc-parser@0.152.0: + dependencies: + '@oxc-project/types': 0.152.0 + optionalDependencies: + '@oxc-parser/binding-android-arm-eabi': 0.152.0 + '@oxc-parser/binding-android-arm64': 0.152.0 + '@oxc-parser/binding-darwin-arm64': 0.152.0 + '@oxc-parser/binding-darwin-x64': 0.152.0 + '@oxc-parser/binding-freebsd-x64': 0.152.0 + '@oxc-parser/binding-linux-arm-gnueabihf': 0.152.0 + '@oxc-parser/binding-linux-arm-musleabihf': 0.152.0 + '@oxc-parser/binding-linux-arm64-gnu': 0.152.0 + '@oxc-parser/binding-linux-arm64-musl': 0.152.0 + '@oxc-parser/binding-linux-ppc64-gnu': 0.152.0 + '@oxc-parser/binding-linux-riscv64-gnu': 0.152.0 + '@oxc-parser/binding-linux-riscv64-musl': 0.152.0 + '@oxc-parser/binding-linux-s390x-gnu': 0.152.0 + '@oxc-parser/binding-linux-x64-gnu': 0.152.0 + '@oxc-parser/binding-linux-x64-musl': 0.152.0 + '@oxc-parser/binding-openharmony-arm64': 0.152.0 + '@oxc-parser/binding-win32-arm64-msvc': 0.152.0 + '@oxc-parser/binding-win32-ia32-msvc': 0.152.0 + '@oxc-parser/binding-win32-x64-msvc': 0.152.0 + picocolors@1.1.1: {} picomatch@4.0.7: {} @@ -660,7 +879,7 @@ snapshots: fdir: 6.5.0(picomatch@4.0.7) picomatch: 4.0.7 - vite@8.3.2: + vite@8.3.2(yaml@2.9.1): dependencies: lightningcss: 1.33.0 picomatch: 4.0.7 @@ -669,11 +888,12 @@ snapshots: tinyglobby: 0.2.17 optionalDependencies: fsevents: 2.3.3 + yaml: 2.9.1 - vitest@5.0.3(vite@8.3.2): + vitest@5.0.3(vite@8.3.2(yaml@2.9.1)): dependencies: '@types/chai': 5.2.3 - '@vitest/mocker': 5.0.3(vite@8.3.2) + '@vitest/mocker': 5.0.3(vite@8.3.2(yaml@2.9.1)) chai: 6.3.0 es-module-lexer: 2.3.2 expect-type: 1.4.0 @@ -684,9 +904,11 @@ snapshots: tinybench: 6.2.0 tinyexec: 1.3.1 tinyglobby: 0.2.17 - vite: 8.3.2 + vite: 8.3.2(yaml@2.9.1) why-is-node-running: 3.2.1 transitivePeerDependencies: - msw why-is-node-running@3.2.1: {} + + yaml@2.9.1: {} diff --git a/evals/ratstack-scorecard/ratstack.pin.json b/evals/ratstack-scorecard/ratstack.pin.json new file mode 100644 index 0000000..c2e0cd6 --- /dev/null +++ b/evals/ratstack-scorecard/ratstack.pin.json @@ -0,0 +1,6 @@ +{ + "owner": "joelhooks", + "repo": "rat-stack", + "commit": "54d356037c994f89698760a4727be71d0005a087", + "narHash": "sha256-JVHgmV8rqDS3EjfcLbeb1VGVRS1e5STcOONL9ktIKgA=" +} diff --git a/evals/ratstack-scorecard/scorecard.schema.json b/evals/ratstack-scorecard/scorecard.schema.json index fdbdcab..62f1d3b 100644 --- a/evals/ratstack-scorecard/scorecard.schema.json +++ b/evals/ratstack-scorecard/scorecard.schema.json @@ -38,7 +38,11 @@ "runner": { "type": "string", "minLength": 1 }, "measuredAt": { "type": "string", "minLength": 1 }, "tools": { "type": "object", "additionalProperties": { "type": "string" } }, - "liveCommit": { "$ref": "#/definitions/sha" } + "liveCommit": { "$ref": "#/definitions/sha" }, + "detail": { + "type": "object", + "additionalProperties": { "anyOf": [{ "type": "number" }, { "type": "string" }] } + } } }, "citation": { diff --git a/evals/ratstack-scorecard/src/families/static.ts b/evals/ratstack-scorecard/src/families/static.ts new file mode 100644 index 0000000..7f0254b --- /dev/null +++ b/evals/ratstack-scorecard/src/families/static.ts @@ -0,0 +1,137 @@ +import { join } from 'node:path' +import { arrayAt, numberAt, stringAt } from '../harness/decode.ts' +import { + type Instrument, + materializeStarter, + prepareTools, + provenanceFor, + trackedFiles, + walkFiles, +} from '../harness/instrument.ts' +import { runSandboxed } from '../harness/sandbox.ts' +import type { Cell, FamilyResult, Side, SideCell } from '../model/cell.ts' +import { isSourceFile, tallyDirectives, totalDirectives, vendoredRootsOf } from '../model/directives.ts' +import { distinctPackages } from '../model/lockfile.ts' +import { ratstack } from '../sides/ratstack.ts' +import type { SideAdapter } from '../sides/side-adapter.ts' +import { starter } from '../sides/starter.ts' + +const runs = 3 + +interface Subject { + readonly adapter: SideAdapter + readonly root: string + readonly files: readonly string[] +} + +interface CountedRun { + readonly cell: Cell + readonly detail: Readonly> +} + +const instrumentError = (error: string): CountedRun => ({ cell: { _tag: 'InstrumentError', error }, detail: {} }) + +const node = async (instrument: Instrument, work: string, tools: string, args: readonly string[]) => + await runSandboxed(instrument.launcher, { + project: work, + cwd: tools, + command: ['node', ...args], + deadlineMs: 15 * 60_000, + }) + +const subjectOf = async (instrument: Instrument, work: string, side: Side): Promise => + side === 'ratstack' + ? { adapter: ratstack, root: instrument.ratstackSrc, files: await walkFiles(instrument.ratstackSrc) } + : { + adapter: starter, + root: await materializeStarter(instrument, work), + files: await trackedFiles(instrument.checkout), + } + +const countDirectives = async ( + instrument: Instrument, + work: string, + tools: string, + subject: Subject, +): Promise => { + const sources = subject.files.filter(isSourceFile) + const excluded = subject.adapter.vendored.map((vendored) => ({ + ...vendored, + files: sources.filter((file) => vendoredRootsOf(file, [vendored.root]).length > 0).length, + })) + const counted = sources.filter((file) => + vendoredRootsOf(file, subject.adapter.vendored.map((v) => v.root)).length === 0 + ) + const request = join(work, `directives-${subject.adapter.side}.request.json`) + await Deno.writeTextFile(request, JSON.stringify({ root: subject.root, files: counted })) + const totals: number[] = [] + let detail: Record = {} + for (let run = 0; run < runs; run++) { + const output = join(work, `directives-${subject.adapter.side}-${run}.json`) + const result = await node(instrument, work, tools, ['src/tools/extract-comments.mjs', request, output]) + if (result.code !== 0) { + return instrumentError(`extract-comments exited ${result.code}: ${result.stderr.slice(-2000)}`) + } + const parsed = JSON.parse(await Deno.readTextFile(output)) + const files = arrayAt(parsed, ['files'], 'extract-comments output') + const comments = files.flatMap((file) => arrayAt(file, ['comments'], 'extract-comments file').map(String)) + const tally = tallyDirectives(comments) + totals.push(totalDirectives(tally)) + detail = { + ...tally, + filesCounted: counted.length, + parseErrors: files.reduce((sum, file) => sum + numberAt(file, ['errors'], 'extract-comments file'), 0), + ...Object.fromEntries(excluded.map((v) => [`excluded:${v.root}`, `${v.files} files: ${v.reason}`])), + } + } + return { cell: { _tag: 'Measured', runs: totals }, detail } +} + +const countPackages = async ( + instrument: Instrument, + work: string, + tools: string, + subject: Subject, +): Promise => { + const lockfile = join(subject.root, subject.adapter.lockfile) + const counts: number[] = [] + let detail: Record = {} + for (let run = 0; run < runs; run++) { + const output = join(work, `lockfile-${subject.adapter.side}-${run}.json`) + const result = await node(instrument, work, tools, ['src/tools/parse-lockfile.mjs', lockfile, output]) + if (result.code !== 0) return instrumentError(`parse-lockfile exited ${result.code}: ${result.stderr.slice(-2000)}`) + const documents = arrayAt(JSON.parse(await Deno.readTextFile(output)), ['documents'], 'parse-lockfile output') + const keys = documents.flatMap((doc) => arrayAt(doc, ['packages'], 'lockfile document').map(String)) + counts.push(distinctPackages(keys).length) + detail = { + lockfile: subject.adapter.lockfile, + documents: documents.length, + lockfileVersions: documents.map((doc) => stringAt(doc, ['lockfileVersion'], 'lockfile document')).join(', '), + } + } + return { cell: { _tag: 'Measured', runs: counts }, detail } +} + +export const measureStatic = async ( + instrument: Instrument, + work: string, + sides: readonly Side[], +): Promise => { + const started = performance.now() + const tools = await prepareTools(instrument, work) + const cells: SideCell[] = [] + for (const side of sides) { + const subject = await subjectOf(instrument, work, side) + const debt = await countDirectives(instrument, work, tools, subject) + const packages = await countPackages(instrument, work, tools, subject) + cells.push( + { id: 'M23', side, measured: { cell: debt.cell, provenance: provenanceFor(instrument, side, debt.detail) } }, + { + id: 'M28', + side, + measured: { cell: packages.cell, provenance: provenanceFor(instrument, side, packages.detail) }, + }, + ) + } + return { family: 'static', wallMs: Math.round(performance.now() - started), cells, flags: [] } +} diff --git a/evals/ratstack-scorecard/src/harness/decode.ts b/evals/ratstack-scorecard/src/harness/decode.ts new file mode 100644 index 0000000..e271455 --- /dev/null +++ b/evals/ratstack-scorecard/src/harness/decode.ts @@ -0,0 +1,41 @@ +export const at = (value: unknown, key: string): unknown => + typeof value === 'object' && value !== null && key in value + ? Object.getOwnPropertyDescriptor(value, key)?.value + : undefined + +const pathOf = (value: unknown, path: readonly string[]): unknown => path.reduce(at, value) + +const refuse = (what: string, path: readonly string[], expected: string): never => { + throw new Error(`${what}: expected ${expected} at ${path.join('.') || ''}`) +} + +export const stringAt = (value: unknown, path: readonly string[], what: string): string => { + const found = pathOf(value, path) + return typeof found === 'string' ? found : refuse(what, path, 'a string') +} + +export const numberAt = (value: unknown, path: readonly string[], what: string): number => { + const found = pathOf(value, path) + return typeof found === 'number' ? found : refuse(what, path, 'a number') +} + +export const arrayAt = (value: unknown, path: readonly string[], what: string): readonly unknown[] => { + const found = pathOf(value, path) + return Array.isArray(found) ? found : refuse(what, path, 'an array') +} + +export const entriesAt = ( + value: unknown, + path: readonly string[], + what: string, +): readonly (readonly [string, unknown])[] => { + const found = pathOf(value, path) + return typeof found === 'object' && found !== null && !Array.isArray(found) + ? Object.entries(found) + : refuse(what, path, 'an object') +} + +export const optionalEntriesAt = (value: unknown, path: readonly string[]): readonly (readonly [string, unknown])[] => { + const found = pathOf(value, path) + return typeof found === 'object' && found !== null && !Array.isArray(found) ? Object.entries(found) : [] +} diff --git a/evals/ratstack-scorecard/src/harness/instrument.ts b/evals/ratstack-scorecard/src/harness/instrument.ts new file mode 100644 index 0000000..2de91c3 --- /dev/null +++ b/evals/ratstack-scorecard/src/harness/instrument.ts @@ -0,0 +1,145 @@ +import { dirname, join } from 'node:path' +import type { CellProvenance, Side } from '../model/cell.ts' +import { optionalEntriesAt, stringAt } from './decode.ts' +import { type Launcher, runSandboxed } from './sandbox.ts' + +export interface Pin { + readonly owner: string + readonly repo: string + readonly commit: string + readonly narHash: string +} + +export interface Instrument { + readonly dir: string + readonly launcher: Launcher + readonly toolsStore: string + readonly ratstackSrc: string + readonly checkout: string + readonly pin: Pin + readonly instrumentHash: string + readonly nixpkgsRev: string + readonly runner: string + readonly starterCommit: string + readonly toolVersions: Readonly> +} + +const decoder = new TextDecoder() + +const required = (name: string): string => { + const value = Deno.env.get(name) + if (value === undefined || value === '') { + throw new Error(`${name} is unset: run the scorecard through the flake's \`scorecard\` package`) + } + return value +} + +export const git = async (cwd: string, args: readonly string[]): Promise => { + const output = await new Deno.Command('git', { args: [...args], cwd, stdout: 'piped', stderr: 'piped' }).output() + if (!output.success) throw new Error(`git ${args.join(' ')} failed: ${decoder.decode(output.stderr)}`) + return output.stdout +} + +const gitText = async (cwd: string, args: readonly string[]): Promise => + decoder.decode(await git(cwd, args)).trim() + +const readJson = async (path: string): Promise => JSON.parse(await Deno.readTextFile(path)) + +const nixpkgsRevOf = (lock: unknown): string => stringAt(lock, ['nodes', 'nixpkgs', 'locked', 'rev'], 'flake.lock') + +const pinOf = (pin: unknown): Pin => ({ + owner: stringAt(pin, ['owner'], 'ratstack.pin.json'), + repo: stringAt(pin, ['repo'], 'ratstack.pin.json'), + commit: stringAt(pin, ['commit'], 'ratstack.pin.json'), + narHash: stringAt(pin, ['narHash'], 'ratstack.pin.json'), +}) + +const dependencyVersions = (manifest: unknown): Record => + Object.fromEntries(optionalEntriesAt(manifest, ['dependencies']).map(([name, version]) => [name, String(version)])) + +export const loadInstrument = async (checkoutArg: string | undefined): Promise => { + const dir = required('SCORECARD_INSTRUMENT') + const checkout = await gitText(checkoutArg ?? Deno.cwd(), ['rev-parse', '--show-toplevel']) + return { + dir, + launcher: { executable: required('SCORECARD_SANDBOX'), path: required('SCORECARD_TOOL_PATH') }, + toolsStore: required('SCORECARD_TOOLS_STORE'), + ratstackSrc: required('SCORECARD_RATSTACK_SRC'), + checkout, + pin: pinOf(await readJson(join(dir, 'ratstack.pin.json'))), + instrumentHash: await gitText(checkout, ['rev-parse', 'HEAD:evals/ratstack-scorecard']), + nixpkgsRev: nixpkgsRevOf(await readJson(join(dir, 'flake.lock'))), + runner: Deno.env.get('RUNNER_NAME') ?? Deno.hostname(), + starterCommit: await gitText(checkout, ['rev-parse', 'HEAD']), + toolVersions: { + ...dependencyVersions(await readJson(join(dir, 'package.json'))), + node: required('SCORECARD_NODE_VERSION'), + pnpm: required('SCORECARD_PNPM_VERSION'), + deno: Deno.version.deno, + }, + } +} + +export const provenanceFor = ( + instrument: Instrument, + side: Side, + detail: Readonly>, +): CellProvenance => ({ + side, + commit: { ratstack: instrument.pin.commit, starter: instrument.starterCommit }[side], + instrumentHash: instrument.instrumentHash, + nixpkgsRev: instrument.nixpkgsRev, + runner: instrument.runner, + measuredAt: new Date().toISOString(), + tools: instrument.toolVersions, + detail, +}) + +const copyTracked = async (from: string, to: string, files: readonly string[]): Promise => { + for (const file of files) { + const target = join(to, file) + await Deno.mkdir(dirname(target), { recursive: true }) + const info = await Deno.lstat(join(from, file)) + if (info.isSymlink) await Deno.symlink(await Deno.readLink(join(from, file)), target) + else await Deno.copyFile(join(from, file), target) + } +} + +export const trackedFiles = async (checkout: string): Promise => + decoder.decode(await git(checkout, ['ls-files', '-z'])).split('\0').filter((file) => file !== '') + +export const walkFiles = async (root: string, prefix = ''): Promise => { + const found: string[] = [] + for await (const entry of Deno.readDir(join(root, prefix))) { + const path = prefix === '' ? entry.name : `${prefix}/${entry.name}` + if (entry.isDirectory) found.push(...await walkFiles(root, path)) + else found.push(path) + } + return found.sort() +} + +export const materializeStarter = async (instrument: Instrument, work: string): Promise => { + const target = join(work, 'starter') + await copyTracked(instrument.checkout, target, await trackedFiles(instrument.checkout)) + return target +} + +export const prepareTools = async (instrument: Instrument, work: string): Promise => { + const tools = join(work, 'tools') + await Deno.mkdir(join(tools, 'src/tools'), { recursive: true }) + for (const file of ['package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml']) { + await Deno.copyFile(join(instrument.dir, file), join(tools, file)) + } + for await (const entry of Deno.readDir(join(instrument.dir, 'src/tools'))) { + await Deno.copyFile(join(instrument.dir, 'src/tools', entry.name), join(tools, 'src/tools', entry.name)) + } + const install = await runSandboxed(instrument.launcher, { + project: work, + cwd: tools, + command: ['pnpm', 'install', '--frozen-lockfile', '--prod'], + pnpmStore: instrument.toolsStore, + deadlineMs: 10 * 60_000, + }) + if (install.code !== 0) throw new Error(`installing the instrument tools failed:\n${install.stderr}`) + return tools +} diff --git a/evals/ratstack-scorecard/src/harness/sandbox.ts b/evals/ratstack-scorecard/src/harness/sandbox.ts new file mode 100644 index 0000000..ffc8e20 --- /dev/null +++ b/evals/ratstack-scorecard/src/harness/sandbox.ts @@ -0,0 +1,54 @@ +export interface SandboxRequest { + readonly project: string + readonly cwd: string + readonly command: readonly string[] + readonly allowHosts?: readonly string[] + readonly env?: Readonly> + readonly pnpmStore?: string + readonly deadlineMs: number +} + +export interface SandboxResult { + readonly code: number + readonly stdout: string + readonly stderr: string + readonly wallMs: number + readonly pastDeadline: boolean +} + +export interface Launcher { + readonly executable: string + readonly path: string +} + +const decoder = new TextDecoder() + +export const launcherArgs = (request: SandboxRequest): readonly string[] => [ + ...(request.allowHosts ?? []).flatMap((host) => ['--allow-host', host]), + ...Object.keys(request.env ?? {}).flatMap((name) => ['--pass-env', name]), + ...(request.pnpmStore === undefined ? [] : ['--pnpm-store', request.pnpmStore]), + '--', + ...request.command, +] + +export const runSandboxed = async (launcher: Launcher, request: SandboxRequest): Promise => { + const started = performance.now() + const deadline = AbortSignal.timeout(request.deadlineMs) + const output = await new Deno.Command(launcher.executable, { + args: [...launcherArgs(request)], + cwd: request.cwd, + clearEnv: true, + env: { PATH: launcher.path, HOME: '/tmp', SANDBOX_PROJECT: request.project, ...request.env }, + stdin: 'null', + stdout: 'piped', + stderr: 'piped', + signal: deadline, + }).output() + return { + code: output.code, + stdout: decoder.decode(output.stdout), + stderr: decoder.decode(output.stderr), + wallMs: Math.round(performance.now() - started), + pastDeadline: deadline.aborted, + } +} diff --git a/evals/ratstack-scorecard/src/main.ts b/evals/ratstack-scorecard/src/main.ts new file mode 100644 index 0000000..5167e07 --- /dev/null +++ b/evals/ratstack-scorecard/src/main.ts @@ -0,0 +1,53 @@ +import { parseArgs } from 'node:util' +import { measureStatic } from './families/static.ts' +import { type Instrument, loadInstrument } from './harness/instrument.ts' +import type { FamilyResult, Side } from './model/cell.ts' + +const usage = `usage: scorecard measure --family [--side ratstack|starter] [--out ] [--checkout ]` + +const families: Readonly< + Record Promise> +> = { + static: measureStatic, +} + +const sidesOf = (side: string | undefined): readonly Side[] => { + if (side === undefined) return ['ratstack', 'starter'] + if (side === 'ratstack' || side === 'starter') return [side] + throw new Error(`unknown side ${side}\n${usage}`) +} + +const measure = async (args: readonly string[]): Promise => { + const { values } = parseArgs({ + args: [...args], + options: { + family: { type: 'string' }, + side: { type: 'string' }, + out: { type: 'string' }, + checkout: { type: 'string' }, + }, + strict: true, + }) + const run = families[values.family ?? ''] + if (run === undefined) throw new Error(`unknown family ${values.family}\n${usage}`) + const instrument = await loadInstrument(values.checkout) + const work = await Deno.makeTempDir({ prefix: `scorecard-${values.family}-` }) + try { + const result: FamilyResult = await run(instrument, work, sidesOf(values.side)) + const json = `${JSON.stringify(result, null, 2)}\n` + if (values.out === undefined) await Deno.stdout.write(new TextEncoder().encode(json)) + else await Deno.writeTextFile(values.out, json) + } finally { + await Deno.remove(work, { recursive: true }) + } +} + +const commands: Readonly Promise>> = { measure } + +const [command, ...rest] = Deno.args +const handler = commands[command ?? ''] +if (handler === undefined) { + console.error(usage) + Deno.exit(2) +} +await handler(rest) diff --git a/evals/ratstack-scorecard/src/model/cell.ts b/evals/ratstack-scorecard/src/model/cell.ts index b6921d8..97f086d 100644 --- a/evals/ratstack-scorecard/src/model/cell.ts +++ b/evals/ratstack-scorecard/src/model/cell.ts @@ -65,6 +65,7 @@ export interface CellProvenance { readonly measuredAt: string readonly tools: Readonly> readonly liveCommit?: string + readonly detail?: Readonly> } export interface MeasuredCell { @@ -85,6 +86,29 @@ export interface Row { readonly flags: readonly Flag[] } +export interface DefinedRow { + readonly definition: RowDefinition + readonly hash: string +} + +export interface SideCell { + readonly id: string + readonly side: Side + readonly measured: MeasuredCell +} + +export interface RowFlag { + readonly id: string + readonly flag: Flag +} + +export interface FamilyResult { + readonly family: Family + readonly wallMs: number + readonly cells: readonly SideCell[] + readonly flags: readonly RowFlag[] +} + export type RowOutcome = | { readonly _tag: 'Held'; readonly id: string } | { readonly _tag: 'New'; readonly id: string } diff --git a/evals/ratstack-scorecard/src/model/directives.property.test.ts b/evals/ratstack-scorecard/src/model/directives.property.test.ts new file mode 100644 index 0000000..2f3c025 --- /dev/null +++ b/evals/ratstack-scorecard/src/model/directives.property.test.ts @@ -0,0 +1,50 @@ +import { fc, test } from '@fast-check/vitest' +import { describe } from 'vitest' +import { type DirectiveFamily, directivesIn, totalDirectives, vendoredRootsOf } from './directives.ts' + +const directiveText: Readonly>> = { + 'oxlint': fc.constantFrom('oxlint-disable', 'oxlint-disable-next-line', 'oxlint-disable-line'), + 'effect-diagnostics': fc.constantFrom('@effect-diagnostics', '@effect-diagnostics-next-line'), + 'typescript': fc.constantFrom('@ts-expect-error', '@ts-ignore', '@ts-nocheck'), + 'stryker': fc.constantFrom('Stryker disable', 'Stryker restore'), + 'eslint': fc.constantFrom('eslint-disable', 'eslint-disable-next-line', 'eslint-disable-line'), + 'biome': fc.constantFrom('biome-ignore', 'biome-ignore-all'), + 'dprint': fc.constantFrom('dprint-ignore', 'dprint-ignore-file'), + 'deno-lint': fc.constantFrom('deno-lint-ignore', 'deno-lint-ignore-file'), + 'coverage': fc.constantFrom('c8 ignore', 'istanbul ignore', 'v8 ignore'), +} + +const family = fc.constantFrom( + 'oxlint', + 'effect-diagnostics', + 'typescript', + 'stryker', + 'eslint', + 'biome', + 'dprint', + 'deno-lint', + 'coverage', +) + +const punctuation = fc.stringMatching(/^[0-9 ,.:;()*]{0,40}$/) + +describe('directivesIn', () => { + test.prop([family.chain((f) => fc.tuple(fc.constant(f), directiveText[f])), punctuation, punctuation])( + 'a comment carrying one directive counts once in its own family and nowhere else', + ([f, directive], before, after) => { + const tally = directivesIn(`${before} ${directive} ${after}`) + return tally[f] === 1 && totalDirectives(tally) === 1 + }, + ) +}) + +describe('vendoredRootsOf', () => { + test.prop([ + fc.stringMatching(/^[a-z]{1,8}(\/[a-z]{1,8}){0,2}\/$/), + fc.stringMatching(/^[a-z]{1,8}(\/[a-z]{1,8}){0,2}$/), + fc.stringMatching(/^\.[a-z]{1,4}$/), + ])( + 'a path under a declared root is excluded by that root, whatever its extension', + (root, rest, extension) => vendoredRootsOf(`${root}${rest}${extension}`, [root]).length === 1, + ) +}) diff --git a/evals/ratstack-scorecard/src/model/directives.ts b/evals/ratstack-scorecard/src/model/directives.ts new file mode 100644 index 0000000..29d5f36 --- /dev/null +++ b/evals/ratstack-scorecard/src/model/directives.ts @@ -0,0 +1,50 @@ +export type DirectiveFamily = + | 'oxlint' + | 'effect-diagnostics' + | 'typescript' + | 'stryker' + | 'eslint' + | 'biome' + | 'dprint' + | 'deno-lint' + | 'coverage' + +export type DirectiveTally = Readonly> + +const patterns: Readonly> = { + 'oxlint': /\boxlint-disable(?:-[a-z-]+)?\b/gu, + 'effect-diagnostics': /@effect-diagnostics(?:-next-line)?\b/gu, + 'typescript': /@ts-(?:expect-error|ignore|nocheck)\b/gu, + 'stryker': /\bStryker (?:disable|restore)\b/gu, + 'eslint': /\beslint-disable(?:-[a-z-]+)?\b/gu, + 'biome': /\bbiome-ignore(?:-[a-z-]+)?\b/gu, + 'dprint': /\bdprint-ignore(?:-[a-z-]+)?\b/gu, + 'deno-lint': /\bdeno-lint-ignore(?:-file)?\b/gu, + 'coverage': /\b(?:c8|istanbul|v8) ignore\b/gu, +} + +const tallyOf = (count: (family: DirectiveFamily) => number): DirectiveTally => ({ + 'oxlint': count('oxlint'), + 'effect-diagnostics': count('effect-diagnostics'), + 'typescript': count('typescript'), + 'stryker': count('stryker'), + 'eslint': count('eslint'), + 'biome': count('biome'), + 'dprint': count('dprint'), + 'deno-lint': count('deno-lint'), + 'coverage': count('coverage'), +}) + +export const directivesIn = (comment: string): DirectiveTally => + tallyOf((family) => [...comment.matchAll(patterns[family])].length) + +export const tallyDirectives = (comments: readonly string[]): DirectiveTally => + comments.map(directivesIn).reduce((sum, next) => tallyOf((family) => sum[family] + next[family]), tallyOf(() => 0)) + +export const totalDirectives = (tally: DirectiveTally): number => + Object.values(tally).reduce((sum, count) => sum + count, 0) + +export const isSourceFile = (path: string): boolean => /\.[cm]?[jt]sx?$/u.test(path) + +export const vendoredRootsOf = (path: string, roots: readonly string[]): readonly string[] => + roots.filter((root) => path.startsWith(root)) diff --git a/evals/ratstack-scorecard/src/model/lockfile.ts b/evals/ratstack-scorecard/src/model/lockfile.ts new file mode 100644 index 0000000..528166a --- /dev/null +++ b/evals/ratstack-scorecard/src/model/lockfile.ts @@ -0,0 +1,4 @@ +const packageIdOf = (key: string): string => key.replace(/^\//u, '').replace(/\(.*\)$/u, '') + +export const distinctPackages = (keys: readonly string[]): readonly string[] => + [...new Set(keys.map(packageIdOf))].sort() diff --git a/evals/ratstack-scorecard/src/model/scorecard-document.ts b/evals/ratstack-scorecard/src/model/scorecard-document.ts index 9c49d18..c2319b9 100644 --- a/evals/ratstack-scorecard/src/model/scorecard-document.ts +++ b/evals/ratstack-scorecard/src/model/scorecard-document.ts @@ -1,32 +1,17 @@ import type { + DefinedRow, DocumentProvenance, - Flag, MainBaseline, MeasuredCell, Row, - RowDefinition, + RowFlag, ScorecardDocument, Side, + SideCell, } from './cell.ts' import { compareWithMain } from './compare-with-main.workflow.ts' import { judgeRow } from './judge-row.workflow.ts' -export interface DefinedRow { - readonly definition: RowDefinition - readonly hash: string -} - -export interface SideCell { - readonly id: string - readonly side: Side - readonly measured: MeasuredCell -} - -export interface RowFlag { - readonly id: string - readonly flag: Flag -} - export type CellsByRow = Readonly>>>> export interface AssembleInput { diff --git a/evals/ratstack-scorecard/src/model/scorecard.arbitrary.ts b/evals/ratstack-scorecard/src/model/scorecard.arbitrary.ts index 411a908..0473681 100644 --- a/evals/ratstack-scorecard/src/model/scorecard.arbitrary.ts +++ b/evals/ratstack-scorecard/src/model/scorecard.arbitrary.ts @@ -76,7 +76,8 @@ export const cellProvenance = (side: Side): fc.Arbitrary => runner: text, measuredAt: fc.date({ noInvalidDate: true }).map((d) => d.toISOString()), tools: fc.dictionary(fc.string({ minLength: 1, maxLength: 20 }), fc.string({ maxLength: 20 })), - }) + detail: fc.dictionary(fc.string({ minLength: 1, maxLength: 20 }), fc.oneof(fc.string({ maxLength: 40 }), value)), + }, { requiredKeys: ['side', 'commit', 'instrumentHash', 'nixpkgsRev', 'runner', 'measuredAt', 'tools'] }) export const measuredCell = (row: RowDefinition, side: Side): fc.Arbitrary => fc.record({ cell: cell(row), provenance: cellProvenance(side) }) diff --git a/evals/ratstack-scorecard/src/model/verify-citation.workflow.property.test.ts b/evals/ratstack-scorecard/src/model/verify-citation.workflow.property.test.ts new file mode 100644 index 0000000..4c71f35 --- /dev/null +++ b/evals/ratstack-scorecard/src/model/verify-citation.workflow.property.test.ts @@ -0,0 +1,26 @@ +import { fc, test } from '@fast-check/vitest' +import { describe } from 'vitest' +import { verifyCitation } from './verify-citation.workflow.ts' + +const line = fc.stringMatching(/^[a-z ]{0,40}$/) +const claim = fc.stringMatching(/^[A-Z][A-Z ]{8,30}[A-Z]$/) + +const fileWith = (lines: readonly string[], at: number, text: string): string => + [...lines.slice(0, at), text, ...lines.slice(at)].join('\n') + +const checkAt = (source: string, text: string, line: number) => + verifyCitation({ citation: { file: 'f.ts', lines: [line, line], text }, source })._tag + +describe('verifyCitation', () => { + test.prop([fc.array(line, { minLength: 2, maxLength: 30 }), claim, fc.nat()])( + 'a citation verifies on the line holding the text and is contradicted one line either side', + (lines, text, seed) => { + const at = 1 + (seed % (lines.length - 1)) + const source = fileWith(lines, at, text) + const cited = at + 1 + return checkAt(source, text, cited) === 'Verified' && + checkAt(source, text, cited - 1) === 'Contradicted' && + checkAt(source, text, cited + 1) === 'Contradicted' + }, + ) +}) diff --git a/evals/ratstack-scorecard/src/model/verify-citation.workflow.ts b/evals/ratstack-scorecard/src/model/verify-citation.workflow.ts new file mode 100644 index 0000000..4f03c72 --- /dev/null +++ b/evals/ratstack-scorecard/src/model/verify-citation.workflow.ts @@ -0,0 +1,19 @@ +import type { Citation, CitationCheck } from './cell.ts' +import { firstRule } from './dispatch.ts' + +export interface VerifyCitationInput { + readonly citation: Citation + readonly source: string +} + +const normalized = (text: string): string => text.replace(/\s+/gu, ' ').trim() + +export const verifyCitation = (input: VerifyCitationInput): CitationCheck => { + const cited = normalized( + input.source.split(/\r?\n/u).slice(input.citation.lines[0] - 1, input.citation.lines[1]).join('\n'), + ) + return firstRule( + [[cited.includes(normalized(input.citation.text)), () => ({ _tag: 'Verified' })]], + () => ({ _tag: 'Contradicted', found: cited.slice(0, 200) }), + ) +} diff --git a/evals/ratstack-scorecard/src/sides/ratstack.ts b/evals/ratstack-scorecard/src/sides/ratstack.ts new file mode 100644 index 0000000..cf40926 --- /dev/null +++ b/evals/ratstack-scorecard/src/sides/ratstack.ts @@ -0,0 +1,13 @@ +import type { SideAdapter } from './side-adapter.ts' + +export const ratstack: SideAdapter = { + side: 'ratstack', + lockfile: 'pnpm-lock.yaml', + vendored: [ + { + root: 'tools/oxlint/anti-slop/', + reason: 'copied from dmmulroy/anti-slop (tools/oxlint/anti-slop/UPSTREAM.md); rat-stack’s own ledger excludes it', + }, + { root: 'vendor/', reason: 'vendored dependency sources (vendor/README.md); rat-stack’s own ledger excludes it' }, + ], +} diff --git a/evals/ratstack-scorecard/src/sides/side-adapter.ts b/evals/ratstack-scorecard/src/sides/side-adapter.ts new file mode 100644 index 0000000..23278e5 --- /dev/null +++ b/evals/ratstack-scorecard/src/sides/side-adapter.ts @@ -0,0 +1,12 @@ +import type { Side } from '../model/cell.ts' + +export interface VendoredRoot { + readonly root: string + readonly reason: string +} + +export interface SideAdapter { + readonly side: Side + readonly lockfile: string + readonly vendored: readonly VendoredRoot[] +} diff --git a/evals/ratstack-scorecard/src/sides/starter.ts b/evals/ratstack-scorecard/src/sides/starter.ts new file mode 100644 index 0000000..34ed8a9 --- /dev/null +++ b/evals/ratstack-scorecard/src/sides/starter.ts @@ -0,0 +1,10 @@ +import type { SideAdapter } from './side-adapter.ts' + +export const starter: SideAdapter = { + side: 'starter', + lockfile: 'pnpm-lock.yaml', + vendored: [ + { root: 'repos/', reason: 'git subtrees vendored from other repositories (subtrees.toml)' }, + { root: 'evals/', reason: 'the scorecard instrument itself, which grades the starter and is not starter code' }, + ], +} diff --git a/evals/ratstack-scorecard/src/tools/extract-comments.mjs b/evals/ratstack-scorecard/src/tools/extract-comments.mjs new file mode 100644 index 0000000..6ccb164 --- /dev/null +++ b/evals/ratstack-scorecard/src/tools/extract-comments.mjs @@ -0,0 +1,19 @@ +import { readFile, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { parseSync } from 'oxc-parser' + +const [request, output] = process.argv.slice(2) +const { root, files } = JSON.parse(await readFile(request, 'utf8')) + +const parsed = [] +for (const file of files) { + const source = await readFile(join(root, file), 'utf8') + const result = parseSync(file, source) + parsed.push({ + path: file, + comments: result.comments.map((comment) => comment.value), + errors: result.errors.length, + }) +} + +await writeFile(output, JSON.stringify({ parser: 'oxc-parser', files: parsed })) diff --git a/evals/ratstack-scorecard/src/tools/parse-lockfile.mjs b/evals/ratstack-scorecard/src/tools/parse-lockfile.mjs new file mode 100644 index 0000000..8d58bc9 --- /dev/null +++ b/evals/ratstack-scorecard/src/tools/parse-lockfile.mjs @@ -0,0 +1,14 @@ +import { readFile, writeFile } from 'node:fs/promises' +import { parseAllDocuments } from 'yaml' + +const [lockfile, output] = process.argv.slice(2) +const documents = parseAllDocuments(await readFile(lockfile, 'utf8')).map((document) => { + const value = document.toJS() ?? {} + return { + lockfileVersion: String(value.lockfileVersion ?? ''), + packages: Object.keys(value.packages ?? {}), + errors: document.errors.length, + } +}) + +await writeFile(output, JSON.stringify({ parser: 'yaml', documents })) From 54f6fe677fb399d4f603233a746a90ceb44791d5 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Oct 2026 01:38:05 +0000 Subject: [PATCH 2/8] fix(repo): refresh the scorecard tools store hash for pnpm 12.9.0 The fixed-output pnpm deps hash changed under pnpm 12.9.0; the local build reused the old output path because its hash was still set, and CI's clean store refused it (got sha256-1sxiCRZ...) --- evals/ratstack-scorecard/flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index bd63b3a..b3155f9 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -25,7 +25,7 @@ src = self; pname = "ratstack-scorecard"; pnpm = pkgs.pnpm_12; - hash = "sha256-6AG6SAB7vZR3kLGU7ujHxBwBqD+cSrwKeTl0qLyz9ko="; + hash = "sha256-1sxiCRZTB0wEkZZGf371K7Cl6yj3t9WlLJjVUXQ1+rk="; }).pnpm-store; ratstack-src = pkgs.fetchFromGitHub { inherit (pin) owner repo; From a2a1fc608b934cd8d8a9f7d47c5b719f3fec3fd2 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Oct 2026 04:40:16 +0000 Subject: [PATCH 3/8] test(repo): run launcher journeys in two phases behind the zero-dependency driver Review fixes #1/#3 and #11 under Kiro's ruling: the launcher cannot nest (bwrap: setting up uid map: Operation not permitted), so scorecard journeys produces each manifest journey on the host (measureStatic end to end over two fixture git repos, and once with a tool that exits 3) and records it, then runs the one vitest project in the launcher. launcherRun(id) decodes the record with Effect Schema and fails red when it is missing or stale. scorecard check adds the import-graph rules and runs in check:ci --- .github/workflows/ci.yml | 3 + ...10-05-2151-feat-ratstack-scorecard-plan.md | 1 + evals/ratstack-scorecard/.gitignore | 1 + evals/ratstack-scorecard/flake.nix | 2 +- .../repos/ratstack/pnpm-lock.yaml | 73 ++++++++ .../__fixtures__/repos/ratstack/src/intake.ts | 4 + .../repos/ratstack/vendor/copied.ts | 2 + .../__fixtures__/repos/starter/pnpm-lock.yaml | 78 +++++++++ .../repos/starter/repos/subtree.ts | 2 + .../__fixtures__/repos/starter/src/claim.ts | 3 + .../journeys/launcher-run.ts | 83 ++++++++++ .../ratstack-scorecard/journeys/manifest.json | 34 ++++ .../journeys/static-family.test.ts | 47 ++++++ ...c.journey.test.ts => static-tools.test.ts} | 0 evals/ratstack-scorecard/package.json | 3 +- evals/ratstack-scorecard/pnpm-lock.yaml | 8 + evals/ratstack-scorecard/src/check-imports.ts | 41 +++++ evals/ratstack-scorecard/src/journeys.ts | 156 ++++++++++++++++++ evals/ratstack-scorecard/src/main.ts | 29 +++- package.json | 3 +- 20 files changed, 568 insertions(+), 5 deletions(-) create mode 100644 evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/pnpm-lock.yaml create mode 100644 evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/src/intake.ts create mode 100644 evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/vendor/copied.ts create mode 100644 evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/pnpm-lock.yaml create mode 100644 evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/repos/subtree.ts create mode 100644 evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/src/claim.ts create mode 100644 evals/ratstack-scorecard/journeys/launcher-run.ts create mode 100644 evals/ratstack-scorecard/journeys/manifest.json create mode 100644 evals/ratstack-scorecard/journeys/static-family.test.ts rename evals/ratstack-scorecard/journeys/{static.journey.test.ts => static-tools.test.ts} (100%) create mode 100644 evals/ratstack-scorecard/src/check-imports.ts create mode 100644 evals/ratstack-scorecard/src/journeys.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b45a4fb..bd90ab4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,9 @@ jobs: steps: - uses: actions/checkout@v7 - uses: cachix/install-nix-action@v31 + - name: Allow the sandbox launcher's unprivileged user namespaces + if: runner.os == 'Linux' + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: dprint from the flake run: echo "$(nix build --no-link --print-out-paths .#dprint)/bin" >> "$GITHUB_PATH" # pnpm/setup replaces pnpm/action-setup for pnpm v11+ and folds diff --git a/docs/plans/2026-10-05-2151-feat-ratstack-scorecard-plan.md b/docs/plans/2026-10-05-2151-feat-ratstack-scorecard-plan.md index 8743ca2..366b008 100644 --- a/docs/plans/2026-10-05-2151-feat-ratstack-scorecard-plan.md +++ b/docs/plans/2026-10-05-2151-feat-ratstack-scorecard-plan.md @@ -155,6 +155,7 @@ Local rows run each side with its documented local defaults plus throwaway value - KTD14. **Readiness calls URL Scanner v2 with `agentReadiness: true`, then polls the result.** It submits `POST /accounts/{account_id}/urlscanner/v2/scan` with `agentReadiness: true` and polls `GET …/v2/result/{scan_id}` every 15 s. The first live call fixes where the option sits in the request (top level or `options`) and the result path. The cross-check posts `{ "url": … }` to `https://isitagentready.com/api/scan` and reads `level`, as rat-stack's `apps/mischief/scripts/smoke.sh` does. When the levels differ, the row is flagged. This implements the scanner Key Decision (session-settled: user-directed — chosen over isitagentready as primary: first-party reproducible JSON through our token). - KTD15. **Tests are admitted by layer, with refusal as the default** (`skill://test-layer-selection`). Every decision lives in a `*.workflow.ts` module with a colocated `*.workflow.property.test.ts`, mutated at break 100 by sfs stryker-js in CI on push to `main` (D5), never locally. Family runners, harness, side adapters and REST clients are executors and adapters: they get no tests of their own, and each unit's family smoke run plus its PR sabotage proves them. Three process-isolated journeys under `evals/ratstack-scorecard/journeys/` observe what only the seam can see: J1 launcher isolation (U4), J2 the static family over a fixture tree through the real parser in the sandbox (U2), and J3 the CLI's exit status and schema-valid JSON on a ratchet failure (U3). Refused: tests of `registry.ts` (a declaration), unit tests of runners (they would spawn processes or mock the subject), and assertions on rendered Markdown wording. - KTD16. **A mutated gate is graded only against a green baseline.** Every family that edits a clone and runs a gate (M13, M20, M22, M24, M25) first runs the unmodified gate on that side over warm caches, and a red baseline makes those cells `unmeasurable` with the first failing task. The cold path (M26) is its own baseline: a red cold gate makes M26 `unmeasurable` and leaves the other families alone. This keeps a kill rate or removal rate from being computed over a gate that was already failing. +- KTD17. **Journeys that need the launcher run in two phases; only the host driver starts the launcher** (Kiro ruling, 2026-10-06). The launcher cannot nest: `bwrap --unshare-all` inside the prm#5 sandbox (`1801228`, bwrap 0.12.0) fails with `bwrap: setting up uid map: Operation not permitted`, and loosening the launcher is ruled out. So `scorecard journeys` (the zero-third-party Deno driver) first produces each journey declared as data in `journeys/manifest.json`, running the real launcher work (for example `measureStatic` end to end over two fixture git repos built from `journeys/__fixtures__/repos/`), and writes one record per journey to `journeys/__records__/.json`: argv, exit code, stdout, stderr, output files, egress log, wall time, and an input hash over the declared inputs plus the launcher store path. It then runs the single vitest project inside the launcher. A test is a journey because it imports `journeys/launcher-run.ts`, whose `launcherRun(id)` decodes the record with Effect Schema and fails red with `MissingLauncherRecord` or `StaleLauncherRecord`, never a skip (CONST-T12). `scorecard check` (in the root `check:ci`) first runs the import-graph rules (the driver's graph from `src/main.ts` loads no third-party module; no file under `src/` imports the journey fixture), then `scorecard journeys`. ### High-Level Technical Design diff --git a/evals/ratstack-scorecard/.gitignore b/evals/ratstack-scorecard/.gitignore index ceddaa3..596e435 100644 --- a/evals/ratstack-scorecard/.gitignore +++ b/evals/ratstack-scorecard/.gitignore @@ -1 +1,2 @@ .cache/ +journeys/__records__/ diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index b3155f9..32c3159 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -25,7 +25,7 @@ src = self; pname = "ratstack-scorecard"; pnpm = pkgs.pnpm_12; - hash = "sha256-1sxiCRZTB0wEkZZGf371K7Cl6yj3t9WlLJjVUXQ1+rk="; + hash = "sha256-1uk+a9MkcIEm8BohE8vKzI8FM9DfX6OU6gjgfZhpeqE="; }).pnpm-store; ratstack-src = pkgs.fetchFromGitHub { inherit (pin) owner repo; diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/pnpm-lock.yaml b/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/pnpm-lock.yaml new file mode 100644 index 0000000..16c761c --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/pnpm-lock.yaml @@ -0,0 +1,73 @@ +--- +lockfileVersion: "9.0" + +importers: + .: + configDependencies: {} + packageManagerDependencies: + "@pnpm/exe": + specifier: 11.3.0 + version: 11.3.0 + pnpm: + specifier: 11.3.0 + version: 11.3.0 + +packages: + "@pnpm/exe@11.3.0": + resolution: { + integrity: sha512-1ItrG3GdA8HC7IUMy79SmYqynjjfwXtIMlbpx9MzrU3ZXMYMfw3yuwjIXW7Aw2z0rRxxa2KtTYcLxqjIaVjUmg==, + } + hasBin: true + + pnpm@11.3.0: + resolution: { integrity: sha512-AAAA } + hasBin: true + +snapshots: + "@pnpm/exe@11.3.0": {} + + pnpm@11.3.0: {} + +--- +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + .: + dependencies: + effect: + specifier: 4.0.0 + version: 4.0.0 + devDependencies: + "@effect/vitest": + specifier: 4.0.0 + version: 4.0.0(effect@4.0.0)(vitest@5.0.3) + vitest: + specifier: 5.0.3 + version: 5.0.3 + +packages: + "@effect/vitest@4.0.0": + resolution: { integrity: sha512-BBBB } + peerDependencies: + effect: ^4.0.0 + vitest: ^5.0.0 + + effect@4.0.0: + resolution: { integrity: sha512-CCCC } + + vitest@5.0.3: + resolution: { integrity: sha512-DDDD } + +snapshots: + "@effect/vitest@4.0.0(effect@4.0.0)(vitest@5.0.3)": + dependencies: + effect: 4.0.0 + vitest: 5.0.3 + + effect@4.0.0: {} + + vitest@5.0.3: {} diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/src/intake.ts b/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/src/intake.ts new file mode 100644 index 0000000..7b74db8 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/src/intake.ts @@ -0,0 +1,4 @@ +// oxlint-disable-next-line no-console -- counted +console.log('intake') +// @ts-expect-error counted +export const n: number = 'x' diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/vendor/copied.ts b/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/vendor/copied.ts new file mode 100644 index 0000000..69bd8f4 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/repos/ratstack/vendor/copied.ts @@ -0,0 +1,2 @@ +// @ts-nocheck vendored, excluded +export const v = 1 diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/pnpm-lock.yaml b/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/pnpm-lock.yaml new file mode 100644 index 0000000..df58f0f --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/pnpm-lock.yaml @@ -0,0 +1,78 @@ +--- +lockfileVersion: "9.0" + +importers: + .: + configDependencies: {} + packageManagerDependencies: + pnpm: + specifier: 12.4.2 + version: 12.4.2 + +packages: + "@pnpm/exe.linux-x64@12.4.2": + resolution: { integrity: sha512-EEEE } + cpu: [x64] + os: [linux] + + pnpm@12.4.2: + resolution: { integrity: sha512-FFFF } + hasBin: true + +snapshots: + "@pnpm/exe.linux-x64@12.4.2": + optional: true + + pnpm@12.4.2: + optionalDependencies: + "@pnpm/exe.linux-x64": 12.4.2 + +--- +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +catalogs: + default: + effect: + specifier: ^4.0.0 + version: 4.0.0 + +importers: + .: + dependencies: + effect: + specifier: "catalog:" + version: 4.0.0 + devDependencies: + "@effect/vitest": + specifier: 4.0.0 + version: 4.0.0(effect@4.0.0)(vitest@5.0.3) + vitest: + specifier: 5.0.3 + version: 5.0.3 + +packages: + "@effect/vitest@4.0.0": + resolution: { integrity: sha512-BBBB } + peerDependencies: + effect: ^4.0.0 + vitest: ^5.0.0 + + effect@4.0.0: + resolution: { integrity: sha512-CCCC } + + vitest@5.0.3: + resolution: { integrity: sha512-DDDD } + +snapshots: + "@effect/vitest@4.0.0(effect@4.0.0)(vitest@5.0.3)": + dependencies: + effect: 4.0.0 + vitest: 5.0.3 + + effect@4.0.0: {} + + vitest@5.0.3: {} diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/repos/subtree.ts b/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/repos/subtree.ts new file mode 100644 index 0000000..66b66a7 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/repos/subtree.ts @@ -0,0 +1,2 @@ +// @ts-ignore subtree, excluded +export const s = 1 diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/src/claim.ts b/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/src/claim.ts new file mode 100644 index 0000000..43fcc80 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/repos/starter/src/claim.ts @@ -0,0 +1,3 @@ +// eslint-disable-next-line no-var -- counted +export const claim = 1 +export const text = 'oxlint-disable inside a string is not counted' diff --git a/evals/ratstack-scorecard/journeys/launcher-run.ts b/evals/ratstack-scorecard/journeys/launcher-run.ts new file mode 100644 index 0000000..285a85c --- /dev/null +++ b/evals/ratstack-scorecard/journeys/launcher-run.ts @@ -0,0 +1,83 @@ +import { Effect, Schema } from 'effect' +import { createHash } from 'node:crypto' +import { readdir, readFile, stat } from 'node:fs/promises' +import { join } from 'node:path' + +const instrument = join(import.meta.dirname, '..') + +const Manifest = Schema.Struct({ + journeys: Schema.Array(Schema.Struct({ id: Schema.String, inputs: Schema.Array(Schema.String) })), +}) + +export const LauncherRecord = Schema.Struct({ + id: Schema.String, + inputHash: Schema.String, + argv: Schema.Array(Schema.String), + code: Schema.Number, + stdout: Schema.String, + stderr: Schema.String, + files: Schema.Record(Schema.String, Schema.String), + egressLog: Schema.Null, + wallMs: Schema.Number, +}) +export type LauncherRecord = typeof LauncherRecord.Type + +export class MissingLauncherRecord extends Schema.TaggedError()('MissingLauncherRecord', { + journey: Schema.String, +}) { + override get message(): string { + return `journey ${this.journey} has no launcher record: run \`scorecard journeys\`, which produces it before vitest` + } +} + +export class StaleLauncherRecord extends Schema.TaggedError()('StaleLauncherRecord', { + journey: Schema.String, + recorded: Schema.String, + current: Schema.String, +}) { + override get message(): string { + return `journey ${this.journey}'s launcher record is stale (inputs ${this.recorded} recorded, ${this.current} now)` + } +} + +const sha256 = (bytes: Uint8Array | string): string => createHash('sha256').update(bytes).digest('hex') + +const filesUnder = async (path: string): Promise => { + const root = join(instrument, path) + if ((await stat(root)).isFile()) return [path] + const entries = await readdir(root, { recursive: true, withFileTypes: true }) + return entries + .filter((entry) => entry.isFile()) + .map((entry) => `${path}/${join(entry.parentPath, entry.name).slice(root.length + 1)}`) +} + +const inputHashOf = async (inputs: readonly string[], launcher: string): Promise => { + const files = (await Promise.all(inputs.map(filesUnder))).flat().sort() + const parts = await Promise.all( + files.map(async (file) => `${file}\0${sha256(await readFile(join(instrument, file)))}`), + ) + return sha256([launcher, ...parts].join('\n')) +} + +const readJson = (path: string) => Effect.promise(async () => JSON.parse(await readFile(path, 'utf8')) as unknown) + +const launcherRunEffect = (journey: string) => + Effect.gen(function*() { + const manifest = yield* Schema.decodeUnknownEffect(Manifest)( + yield* readJson(join(instrument, 'journeys/manifest.json')), + ) + const entry = manifest.journeys.find((candidate) => candidate.id === journey) + if (entry === undefined) return yield* new MissingLauncherRecord({ journey }) + const raw = yield* Effect.tryPromise({ + try: async () => JSON.parse(await readFile(join(instrument, 'journeys/__records__', `${journey}.json`), 'utf8')), + catch: () => new MissingLauncherRecord({ journey }), + }) + const record = yield* Schema.decodeUnknownEffect(LauncherRecord)(raw) + const current = yield* Effect.promise(() => inputHashOf(entry.inputs, process.env['SCORECARD_LAUNCHER'] ?? '')) + if (record.inputHash !== current) { + return yield* new StaleLauncherRecord({ journey, recorded: record.inputHash, current }) + } + return record + }) + +export const launcherRun = (journey: string): Promise => Effect.runPromise(launcherRunEffect(journey)) diff --git a/evals/ratstack-scorecard/journeys/manifest.json b/evals/ratstack-scorecard/journeys/manifest.json new file mode 100644 index 0000000..5f8ed77 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/manifest.json @@ -0,0 +1,34 @@ +{ + "journeys": [ + { + "id": "static-family", + "produce": { + "kind": "measure-static", + "ratstackRepo": "journeys/__fixtures__/repos/ratstack", + "starterRepo": "journeys/__fixtures__/repos/starter", + "failingTool": null + }, + "inputs": [ + "journeys/__fixtures__/repos", + "src", + "pnpm-lock.yaml", + "flake.lock" + ] + }, + { + "id": "static-family-failing-tool", + "produce": { + "kind": "measure-static", + "ratstackRepo": "journeys/__fixtures__/repos/ratstack", + "starterRepo": "journeys/__fixtures__/repos/starter", + "failingTool": "extract-comments.mjs" + }, + "inputs": [ + "journeys/__fixtures__/repos", + "src", + "pnpm-lock.yaml", + "flake.lock" + ] + } + ] +} diff --git a/evals/ratstack-scorecard/journeys/static-family.test.ts b/evals/ratstack-scorecard/journeys/static-family.test.ts new file mode 100644 index 0000000..4054c3e --- /dev/null +++ b/evals/ratstack-scorecard/journeys/static-family.test.ts @@ -0,0 +1,47 @@ +import { Schema } from 'effect' +import { describe, expect, test } from 'vitest' +import { launcherRun } from './launcher-run.ts' + +const Cell = Schema.Union([ + Schema.Struct({ _tag: Schema.Literal('Measured'), runs: Schema.Array(Schema.Number) }), + Schema.Struct({ _tag: Schema.Literal('InstrumentError'), error: Schema.String }), +]) + +const Family = Schema.Struct({ + family: Schema.Literal('static'), + cells: Schema.Array( + Schema.Struct({ + id: Schema.String, + side: Schema.Literals(['ratstack', 'starter']), + measured: Schema.Struct({ cell: Cell }), + }), + ), +}) + +const familyOf = async (journey: string) => + Schema.decodeUnknownPromise(Family)(JSON.parse((await launcherRun(journey)).files['family.json'] ?? 'null')) + +const cell = (family: typeof Family.Type, id: string, side: 'ratstack' | 'starter') => + family.cells.filter((c) => c.id === id && c.side === side).map((c) => c.measured.cell) + +describe('measureStatic end to end in the launcher, over two fixture git repos', () => { + test('each side gets one M23 and one M28 cell of three runs, counting only unvendored comment directives', async () => { + const family = await familyOf('static-family') + expect(cell(family, 'M23', 'ratstack')).toEqual([{ _tag: 'Measured', runs: [2, 2, 2] }]) + expect(cell(family, 'M23', 'starter')).toEqual([{ _tag: 'Measured', runs: [1, 1, 1] }]) + const [ratstackPackages] = cell(family, 'M28', 'ratstack') + const [starterPackages] = cell(family, 'M28', 'starter') + expect(ratstackPackages?._tag).toBe('Measured') + expect(starterPackages).toEqual(ratstackPackages) + expect(family.cells).toHaveLength(4) + }) + + test('a tool that exits non-zero gives an instrument error carrying its exit code, on both sides', async () => { + const family = await familyOf('static-family-failing-tool') + for (const side of ['ratstack', 'starter'] as const) { + const [m23] = cell(family, 'M23', side) + expect(m23?._tag).toBe('InstrumentError') + expect(m23?._tag === 'InstrumentError' && m23.error).toContain('extract-comments exited 3') + } + }) +}) diff --git a/evals/ratstack-scorecard/journeys/static.journey.test.ts b/evals/ratstack-scorecard/journeys/static-tools.test.ts similarity index 100% rename from evals/ratstack-scorecard/journeys/static.journey.test.ts rename to evals/ratstack-scorecard/journeys/static-tools.test.ts diff --git a/evals/ratstack-scorecard/package.json b/evals/ratstack-scorecard/package.json index 884aeb1..0ff3560 100644 --- a/evals/ratstack-scorecard/package.json +++ b/evals/ratstack-scorecard/package.json @@ -10,6 +10,7 @@ "@fast-check/vitest": "0.5.0", "ajv": "8.20.0", "fast-check": "4.10.2", - "vitest": "5.0.3" + "vitest": "5.0.3", + "effect": "4.0.1" } } diff --git a/evals/ratstack-scorecard/pnpm-lock.yaml b/evals/ratstack-scorecard/pnpm-lock.yaml index 665ab06..3ee8937 100644 --- a/evals/ratstack-scorecard/pnpm-lock.yaml +++ b/evals/ratstack-scorecard/pnpm-lock.yaml @@ -21,6 +21,9 @@ importers: ajv: specifier: 8.20.0 version: 8.20.0 + effect: + specifier: 4.0.1 + version: 4.0.1 fast-check: specifier: 4.10.2 version: 4.10.2 @@ -307,6 +310,9 @@ packages: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} + effect@4.0.1: + resolution: {integrity: sha512-b1VlQG9g8fwxE5QnIZuPoOP/0MsqHJSRKxUijjoM80E5q95FXrX5yWtY5XI8G+GJsBQAVSDbLS458z9a++uVvw==} + es-module-lexer@2.3.2: resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==} @@ -725,6 +731,8 @@ snapshots: detect-libc@2.1.2: {} + effect@4.0.1: {} + es-module-lexer@2.3.2: {} estree-walker@3.0.3: diff --git a/evals/ratstack-scorecard/src/check-imports.ts b/evals/ratstack-scorecard/src/check-imports.ts new file mode 100644 index 0000000..2b2a324 --- /dev/null +++ b/evals/ratstack-scorecard/src/check-imports.ts @@ -0,0 +1,41 @@ +import { join } from 'node:path' +import { walkFiles } from './harness/instrument.ts' + +const specifiers = (source: string): readonly string[] => + [ + ...source.matchAll( + /(?:^|\n)\s*(?:import|export)\b[^'"]*?from\s*['"]([^'"]+)['"]|(?:^|\n)\s*import\s*['"]([^'"]+)['"]|import\(\s*['"]([^'"]+)['"]\s*\)/gu, + ), + ].map((match) => match[1] ?? match[2] ?? match[3] ?? '') + +const isThirdParty = (specifier: string): boolean => + !specifier.startsWith('.') && !specifier.startsWith('/') && !specifier.startsWith('node:') + +const driverGraph = async (root: string, entry: string, seen: Set): Promise => { + if (seen.has(entry)) return [] + seen.add(entry) + const found: string[] = [] + for (const specifier of specifiers(await Deno.readTextFile(join(root, entry)))) { + if (isThirdParty(specifier) || specifier.includes('node_modules')) found.push(`${entry} imports ${specifier}`) + else if (specifier.startsWith('.')) found.push(...await driverGraph(root, join(entry, '..', specifier), seen)) + } + return found +} + +export const checkImports = async (root: string): Promise => { + const driver = await driverGraph(root, 'src/main.ts', new Set()) + const fixtureImports = (await walkFiles(join(root, 'src'))) + .filter((file) => file.endsWith('.ts')) + .map((file) => `src/${file}`) + const misplacedJourneys: string[] = [] + for (const file of fixtureImports) { + const imports = specifiers(await Deno.readTextFile(join(root, file))) + if (imports.some((specifier) => specifier.endsWith('journeys/launcher-run.ts'))) { + misplacedJourneys.push(`${file} imports the journey fixture; journeys live in journeys/, never beside the model`) + } + } + return [ + ...driver.map((line) => `host driver loads third-party code: ${line}`), + ...misplacedJourneys, + ] +} diff --git a/evals/ratstack-scorecard/src/journeys.ts b/evals/ratstack-scorecard/src/journeys.ts new file mode 100644 index 0000000..c3eec88 --- /dev/null +++ b/evals/ratstack-scorecard/src/journeys.ts @@ -0,0 +1,156 @@ +import { join, relative } from 'node:path' +import { measureStatic } from './families/static.ts' +import { arrayAt, at, stringAt } from './harness/decode.ts' +import { git, type Instrument, walkFiles } from './harness/instrument.ts' +import { runSandboxed } from './harness/sandbox.ts' + +interface MeasureStatic { + readonly kind: 'measure-static' + readonly ratstackRepo: string + readonly starterRepo: string + readonly failingTool: string | null +} + +interface JourneyEntry { + readonly id: string + readonly produce: MeasureStatic + readonly inputs: readonly string[] +} + +export interface LauncherRecord { + readonly id: string + readonly inputHash: string + readonly argv: readonly string[] + readonly code: number + readonly stdout: string + readonly stderr: string + readonly files: Readonly> + readonly egressLog: null + readonly wallMs: number +} + +const journeyEntry = (value: unknown): JourneyEntry => { + const produce = at(value, 'produce') + const failingTool = at(produce, 'failingTool') + return { + id: stringAt(value, ['id'], 'journey'), + produce: { + kind: 'measure-static', + ratstackRepo: stringAt(produce, ['ratstackRepo'], 'journey produce'), + starterRepo: stringAt(produce, ['starterRepo'], 'journey produce'), + failingTool: failingTool === null ? null : stringAt(produce, ['failingTool'], 'journey produce'), + }, + inputs: arrayAt(value, ['inputs'], 'journey').map((input) => stringAt({ input }, ['input'], 'journey input')), + } +} + +const encoder = new TextEncoder() + +const hex = (bytes: ArrayBuffer): string => + [...new Uint8Array(bytes)].map((b) => b.toString(16).padStart(2, '0')).join('') + +const filesUnder = async (root: string, path: string): Promise => { + const info = await Deno.stat(join(root, path)) + if (info.isFile) return [path] + return (await walkFiles(join(root, path))).map((file) => `${path}/${file}`) +} + +export const inputHashOf = async (root: string, inputs: readonly string[], launcher: string): Promise => { + const files = (await Promise.all(inputs.map((input) => filesUnder(root, input)))).flat().sort() + const parts = await Promise.all( + files.map(async (file) => + `${file}\0${hex(await crypto.subtle.digest('SHA-256', await Deno.readFile(join(root, file))))}` + ), + ) + return hex(await crypto.subtle.digest('SHA-256', encoder.encode([launcher, ...parts].join('\n')))) +} + +const gitRepoFrom = async (tree: string, target: string): Promise => { + await Deno.mkdir(target, { recursive: true }) + for (const file of await walkFiles(tree)) { + await Deno.mkdir(join(target, file, '..'), { recursive: true }) + await Deno.copyFile(join(tree, file), join(target, file)) + } + const identity = ['-c', 'user.name=journey', '-c', 'user.email=journey@invalid', '-c', 'commit.gpgsign=false'] + await git(target, ['init', '-q', '-b', 'main']) + await git(target, ['add', '-A']) + await git(target, [...identity, 'commit', '-q', '-m', 'fixture']) + return target +} + +const overlayWithFailingTool = async (instrument: Instrument, work: string, tool: string): Promise => { + const dir = join(work, 'instrument') + for (const file of await walkFiles(instrument.dir)) { + if (file.startsWith('node_modules/') || file.startsWith('journeys/__records__/')) continue + await Deno.mkdir(join(dir, file, '..'), { recursive: true }) + await Deno.copyFile(join(instrument.dir, file), join(dir, file)) + } + await Deno.writeTextFile(join(dir, 'src/tools', tool), 'process.exit(3)\n') + return dir +} + +const produce = async (instrument: Instrument, root: string, entry: JourneyEntry): Promise => { + const work = await Deno.makeTempDir({ prefix: `journey-${entry.id}-` }) + const started = performance.now() + try { + const ratstackSrc = await gitRepoFrom(join(root, entry.produce.ratstackRepo), join(work, 'ratstack')) + const checkout = await gitRepoFrom(join(root, entry.produce.starterRepo), join(work, 'starter-checkout')) + const dir = entry.produce.failingTool === null + ? instrument.dir + : await overlayWithFailingTool(instrument, work, entry.produce.failingTool) + const subject: Instrument = { + ...instrument, + dir, + ratstackSrc, + checkout, + starterCommit: new TextDecoder().decode(await git(checkout, ['rev-parse', 'HEAD'])).trim(), + } + const result = await measureStatic(subject, work, ['ratstack', 'starter']) + return { + id: entry.id, + inputHash: await inputHashOf(root, entry.inputs, instrument.launcher.executable), + argv: ['scorecard', 'measure', '--family', 'static'], + code: 0, + stdout: '', + stderr: '', + files: { 'family.json': JSON.stringify(result) }, + egressLog: null, + wallMs: Math.round(performance.now() - started), + } + } finally { + await Deno.remove(work, { recursive: true }) + } +} + +export const runJourneys = async (instrument: Instrument, root: string): Promise => { + const manifest = arrayAt(JSON.parse(await Deno.readTextFile(join(root, 'journeys/manifest.json'))), [ + 'journeys', + ], 'journeys/manifest.json').map(journeyEntry) + const records = join(root, 'journeys/__records__') + await Deno.remove(records, { recursive: true }).catch(() => undefined) + await Deno.mkdir(records, { recursive: true }) + for (const entry of manifest) { + const record = await produce(instrument, root, entry) + await Deno.writeTextFile(join(records, `${entry.id}.json`), `${JSON.stringify(record, null, 2)}\n`) + console.error(`journeys: produced ${entry.id} in ${record.wallMs} ms`) + } + const install = await runSandboxed(instrument.launcher, { + project: root, + cwd: root, + command: ['pnpm', 'install', '--frozen-lockfile'], + pnpmStore: instrument.toolsStore, + deadlineMs: 10 * 60_000, + }) + if (install.code !== 0) throw new Error(`installing the instrument's test tools failed:\n${install.stderr}`) + const vitest = await runSandboxed(instrument.launcher, { + project: root, + cwd: root, + command: ['pnpm', 'exec', 'vitest', 'run'], + env: { SCORECARD_LAUNCHER: instrument.launcher.executable }, + deadlineMs: 30 * 60_000, + }) + await Deno.stdout.write(encoder.encode(vitest.stdout)) + await Deno.stderr.write(encoder.encode(vitest.stderr)) + console.error(`journeys: vitest exited ${vitest.code} (${relative(Deno.cwd(), root) || '.'})`) + return vitest.code +} diff --git a/evals/ratstack-scorecard/src/main.ts b/evals/ratstack-scorecard/src/main.ts index 5167e07..2a1ca1f 100644 --- a/evals/ratstack-scorecard/src/main.ts +++ b/evals/ratstack-scorecard/src/main.ts @@ -1,9 +1,17 @@ +import { join } from 'node:path' import { parseArgs } from 'node:util' +import { checkImports } from './check-imports.ts' import { measureStatic } from './families/static.ts' import { type Instrument, loadInstrument } from './harness/instrument.ts' +import { runJourneys } from './journeys.ts' import type { FamilyResult, Side } from './model/cell.ts' -const usage = `usage: scorecard measure --family [--side ratstack|starter] [--out ] [--checkout ]` +const usage = [ + 'usage:', + ' scorecard measure --family [--side ratstack|starter] [--out ] [--checkout ]', + ' scorecard journeys [--checkout ]', + ' scorecard check [--checkout ] (import-graph rules, then journeys)', +].join('\n') const families: Readonly< Record Promise> @@ -42,7 +50,24 @@ const measure = async (args: readonly string[]): Promise => { } } -const commands: Readonly Promise>> = { measure } +const journeys = async (args: readonly string[]): Promise => { + const { values } = parseArgs({ args: [...args], options: { checkout: { type: 'string' } }, strict: true }) + const instrument = await loadInstrument(values.checkout) + Deno.exit(await runJourneys(instrument, join(instrument.checkout, 'evals/ratstack-scorecard'))) +} + +const check = async (args: readonly string[]): Promise => { + const { values } = parseArgs({ args: [...args], options: { checkout: { type: 'string' } }, strict: true }) + const instrument = await loadInstrument(values.checkout) + const root = join(instrument.checkout, 'evals/ratstack-scorecard') + const violations = await checkImports(root) + for (const violation of violations) console.error(`check-imports: ${violation}`) + if (violations.length > 0) Deno.exit(1) + console.error('check-imports: the host driver loads no third-party code; no model file imports the journey fixture') + Deno.exit(await runJourneys(instrument, root)) +} + +const commands: Readonly Promise>> = { measure, journeys, check } const [command, ...rest] = Deno.args const handler = commands[command ?? ''] diff --git a/package.json b/package.json index 5a60a0e..d158426 100644 --- a/package.json +++ b/package.json @@ -20,11 +20,12 @@ "scripts": { "attw": "turbo --concurrency=${TURBO_CONCURRENCY:-50%} attw", "build": "turbo --concurrency=${TURBO_CONCURRENCY:-50%} build", - "check:ci": "s=0; pnpm format:check || s=1; TURBO_CONCURRENCY=${TURBO_CONCURRENCY:-100%} pnpm gate:tasks || s=1; pnpm gate:dist || s=1; pnpm mutation || s=1; exit $s", + "check:ci": "s=0; pnpm format:check || s=1; TURBO_CONCURRENCY=${TURBO_CONCURRENCY:-100%} pnpm gate:tasks || s=1; pnpm gate:dist || s=1; pnpm scorecard:check || s=1; pnpm mutation || s=1; exit $s", "clean": "turbo clean", "format": "./bin/dprint fmt", "format:check": "./bin/dprint check", "gate:dist": "turbo --concurrency=${TURBO_CONCURRENCY:-100%} build", + "scorecard:check": "nix run ./evals/ratstack-scorecard#scorecard -- check", "gate:tasks": "turbo --concurrency=${TURBO_CONCURRENCY:-50%} --continue lint typecheck typecheck:node test", "lint": "turbo --concurrency=${TURBO_CONCURRENCY:-50%} lint", "mutation": "turbo --concurrency=${TURBO_CONCURRENCY:-50%} mutation", From dbe7a69dc74f98b92099de426df5bb0b6360e702 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Oct 2026 06:43:25 +0000 Subject: [PATCH 4/8] fix(repo): fetch every platform's packages into the scorecard tools store The macOS leg failed with a hash mismatch on the tools store: pnpm fetches only the current platform's optional packages, and on Linux it also skips musl-only bindings. supportedArchitectures lists linux and darwin, x64 and arm64, glibc and musl, so the fetched set and its fixed-output hash are the same on every system --- evals/ratstack-scorecard/flake.nix | 2 +- evals/ratstack-scorecard/pnpm-workspace.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index 32c3159..706a53a 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -25,7 +25,7 @@ src = self; pname = "ratstack-scorecard"; pnpm = pkgs.pnpm_12; - hash = "sha256-1uk+a9MkcIEm8BohE8vKzI8FM9DfX6OU6gjgfZhpeqE="; + hash = "sha256-jUyIV2ysOxkIij/Ff5VGuIADTibbY0SsCPGglmIvdn0="; }).pnpm-store; ratstack-src = pkgs.fetchFromGitHub { inherit (pin) owner repo; diff --git a/evals/ratstack-scorecard/pnpm-workspace.yaml b/evals/ratstack-scorecard/pnpm-workspace.yaml index d05a7e7..a63a95d 100644 --- a/evals/ratstack-scorecard/pnpm-workspace.yaml +++ b/evals/ratstack-scorecard/pnpm-workspace.yaml @@ -1,2 +1,7 @@ packages: - . + +supportedArchitectures: + os: [linux, darwin] + cpu: [x64, arm64] + libc: [glibc, musl] From b40e9a55beac84356529f89b5c31571a79366e2a Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Oct 2026 07:10:07 +0000 Subject: [PATCH 5/8] fix(repo): pin the scorecard tools store hash per system pnpm fetches a different file set on darwin than on linux even with supportedArchitectures listing both, so one fixed-output hash cannot hold for every system. The darwin value is the one the macOS CI leg computed for this lockfile --- evals/ratstack-scorecard/flake.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index 706a53a..f24e709 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -14,6 +14,11 @@ systems = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ]; forEachSystem = fn: nixpkgs.lib.genAttrs systems (system: fn nixpkgs.legacyPackages.${system}); pin = builtins.fromJSON (builtins.readFile ./ratstack.pin.json); + toolsStoreHash = { + x86_64-linux = "sha256-jUyIV2ysOxkIij/Ff5VGuIADTibbY0SsCPGglmIvdn0="; + aarch64-linux = "sha256-jUyIV2ysOxkIij/Ff5VGuIADTibbY0SsCPGglmIvdn0="; + aarch64-darwin = "sha256-XWYBTy3xZbQwJxdGbAK4+R69XNmL3RRyXu6+nY+XPXc="; + }; in { packages = forEachSystem (pkgs: @@ -25,7 +30,7 @@ src = self; pname = "ratstack-scorecard"; pnpm = pkgs.pnpm_12; - hash = "sha256-jUyIV2ysOxkIij/Ff5VGuIADTibbY0SsCPGglmIvdn0="; + hash = toolsStoreHash.${system}; }).pnpm-store; ratstack-src = pkgs.fetchFromGitHub { inherit (pin) owner repo; From b9e1ef33945b6531f8cac3430cb66cf56c084698 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Oct 2026 17:28:36 +0000 Subject: [PATCH 6/8] build(repo): pin the scorecard launcher to the darwin runtime-dir fix pnpm-release-management #8 (8f19844) gives macOS sandbox runs a private XDG_RUNTIME_DIR, so pnpm 12's store lock no longer lands in the denied /tmp; its Sandbox workflow proves the offline install on macos-latest. Pinned as a PR snapshot per the Nix distribution ruling; repin to main's rev once #8 merges --- evals/ratstack-scorecard/flake.lock | 20 ++++++++++---------- evals/ratstack-scorecard/flake.nix | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/evals/ratstack-scorecard/flake.lock b/evals/ratstack-scorecard/flake.lock index c464b2c..244be87 100644 --- a/evals/ratstack-scorecard/flake.lock +++ b/evals/ratstack-scorecard/flake.lock @@ -9,11 +9,11 @@ "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1788142907, - "narHash": "sha256-arhhsgBbDTOTtnG9gphKARLXkCj3emOZUm3sP0a0jnI=", + "lastModified": 1790884787, + "narHash": "sha256-XZz0bdSY5zjlRDme0dfC98oYcAf3WOpMCueoU39iXjY=", "owner": "systemfsoftware", "repo": "comment-checker", - "rev": "60e60bbedb2686c807a9ca64a31f7bba4e75b97b", + "rev": "3bfedd853c1d6dca666e65e71e9f8a31d47db336", "type": "github" }, "original": { @@ -46,17 +46,17 @@ ] }, "locked": { - "lastModified": 1791237748, - "narHash": "sha256-IWdNptNMXzjs2yG3DRQJKUkRcOmVvyDQ0x0dN2maEVU=", + "lastModified": 1791289049, + "narHash": "sha256-EaB7hYyD684/oCH9VgDiHcJw2vAuLf8InlZGFPxN2hE=", "owner": "systemfsoftware", "repo": "pnpm-release-management", - "rev": "180122866dd537fa728b5563fb1820fbd2af88cc", + "rev": "8f1984418fef130956a3d1f50dc471fd1984d2ec", "type": "github" }, "original": { "owner": "systemfsoftware", "repo": "pnpm-release-management", - "rev": "180122866dd537fa728b5563fb1820fbd2af88cc", + "rev": "8f1984418fef130956a3d1f50dc471fd1984d2ec", "type": "github" } }, @@ -75,11 +75,11 @@ ] }, "locked": { - "lastModified": 1787367552, - "narHash": "sha256-YT4Fs2k7bi+7YzuLt93EtIRgjpwHK5ZfsQEIh5dEQSk=", + "lastModified": 1790756160, + "narHash": "sha256-NPs4nNLxCpho3Ctj8O7t/bzmX0zR8r0zndKghAFEz5U=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "fd2ebb9cc4323d0c5a1336138dab5c3c5a5d8bd9", + "rev": "ed3a19fd0439ed618ec5fe1e12f0ba69a8be38b5", "type": "github" }, "original": { diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index f24e709..f294a97 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -4,7 +4,7 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/494ce7fd23ff6a5dff39e1fb11e9b6f2ac74bf25"; pnpm-release-management = { - url = "github:systemfsoftware/pnpm-release-management/180122866dd537fa728b5563fb1820fbd2af88cc"; + url = "github:systemfsoftware/pnpm-release-management/8f1984418fef130956a3d1f50dc471fd1984d2ec"; inputs.nixpkgs.follows = "nixpkgs"; }; }; From a50be7836c33d7fc287b2127edbceef9abeea509 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Oct 2026 17:38:54 +0000 Subject: [PATCH 7/8] fix(repo): replace the copied tool in the failing-tool journey instead of writing over it The overlay copies the instrument out of the read-only Nix store, so each copy keeps mode 0444. Root on the Linux runner ignores the mode; the macOS runner's user cannot write over it (PermissionDenied). Reproduced as uid 65534: overwrite fails, remove then write succeeds --- evals/ratstack-scorecard/src/journeys.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/evals/ratstack-scorecard/src/journeys.ts b/evals/ratstack-scorecard/src/journeys.ts index c3eec88..1890c3e 100644 --- a/evals/ratstack-scorecard/src/journeys.ts +++ b/evals/ratstack-scorecard/src/journeys.ts @@ -85,7 +85,9 @@ const overlayWithFailingTool = async (instrument: Instrument, work: string, tool await Deno.mkdir(join(dir, file, '..'), { recursive: true }) await Deno.copyFile(join(instrument.dir, file), join(dir, file)) } - await Deno.writeTextFile(join(dir, 'src/tools', tool), 'process.exit(3)\n') + const failing = join(dir, 'src/tools', tool) + await Deno.remove(failing) + await Deno.writeTextFile(failing, 'process.exit(3)\n') return dir } From cbb6ac89d7517c7d0980dfa1e0f49679f9c8566f Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Oct 2026 17:49:39 +0000 Subject: [PATCH 8/8] fix(repo): measure rat-stack from a copy inside the sandboxed project From pnpm-release-management 8f19844 the launcher mounts only the invocation's /nix/store closure, so the rat-stack source (a store path outside it) was invisible inside the sandbox and the comment extractor failed with ENOENT, turning M23 into an instrument error on the Scorecard job. The static family now copies rat-stack's source into its project, as it already did for the starter. The static-family journey keeps its fixture sources outside the measured project, as production does: it fails with the same ENOENT without this fix and passes with it --- evals/ratstack-scorecard/src/families/static.ts | 7 ++++++- evals/ratstack-scorecard/src/harness/instrument.ts | 6 ++++++ evals/ratstack-scorecard/src/journeys.ts | 8 +++++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/evals/ratstack-scorecard/src/families/static.ts b/evals/ratstack-scorecard/src/families/static.ts index 7f0254b..45b7414 100644 --- a/evals/ratstack-scorecard/src/families/static.ts +++ b/evals/ratstack-scorecard/src/families/static.ts @@ -2,6 +2,7 @@ import { join } from 'node:path' import { arrayAt, numberAt, stringAt } from '../harness/decode.ts' import { type Instrument, + materializeRatstack, materializeStarter, prepareTools, provenanceFor, @@ -41,7 +42,11 @@ const node = async (instrument: Instrument, work: string, tools: string, args: r const subjectOf = async (instrument: Instrument, work: string, side: Side): Promise => side === 'ratstack' - ? { adapter: ratstack, root: instrument.ratstackSrc, files: await walkFiles(instrument.ratstackSrc) } + ? { + adapter: ratstack, + root: await materializeRatstack(instrument, work), + files: await walkFiles(instrument.ratstackSrc), + } : { adapter: starter, root: await materializeStarter(instrument, work), diff --git a/evals/ratstack-scorecard/src/harness/instrument.ts b/evals/ratstack-scorecard/src/harness/instrument.ts index 2de91c3..7393b3f 100644 --- a/evals/ratstack-scorecard/src/harness/instrument.ts +++ b/evals/ratstack-scorecard/src/harness/instrument.ts @@ -124,6 +124,12 @@ export const materializeStarter = async (instrument: Instrument, work: string): return target } +export const materializeRatstack = async (instrument: Instrument, work: string): Promise => { + const target = join(work, 'ratstack') + await copyTracked(instrument.ratstackSrc, target, await walkFiles(instrument.ratstackSrc)) + return target +} + export const prepareTools = async (instrument: Instrument, work: string): Promise => { const tools = join(work, 'tools') await Deno.mkdir(join(tools, 'src/tools'), { recursive: true }) diff --git a/evals/ratstack-scorecard/src/journeys.ts b/evals/ratstack-scorecard/src/journeys.ts index 1890c3e..475e108 100644 --- a/evals/ratstack-scorecard/src/journeys.ts +++ b/evals/ratstack-scorecard/src/journeys.ts @@ -95,8 +95,10 @@ const produce = async (instrument: Instrument, root: string, entry: JourneyEntry const work = await Deno.makeTempDir({ prefix: `journey-${entry.id}-` }) const started = performance.now() try { - const ratstackSrc = await gitRepoFrom(join(root, entry.produce.ratstackRepo), join(work, 'ratstack')) - const checkout = await gitRepoFrom(join(root, entry.produce.starterRepo), join(work, 'starter-checkout')) + const ratstackSrc = await gitRepoFrom(join(root, entry.produce.ratstackRepo), join(work, 'sources/ratstack')) + const checkout = await gitRepoFrom(join(root, entry.produce.starterRepo), join(work, 'sources/starter-checkout')) + const project = join(work, 'project') + await Deno.mkdir(project) const dir = entry.produce.failingTool === null ? instrument.dir : await overlayWithFailingTool(instrument, work, entry.produce.failingTool) @@ -107,7 +109,7 @@ const produce = async (instrument: Instrument, root: string, entry: JourneyEntry checkout, starterCommit: new TextDecoder().decode(await git(checkout, ['rev-parse', 'HEAD'])).trim(), } - const result = await measureStatic(subject, work, ['ratstack', 'starter']) + const result = await measureStatic(subject, project, ['ratstack', 'starter']) return { id: entry.id, inputHash: await inputHashOf(root, entry.inputs, instrument.launcher.executable),