From 7b9d39e6e498919ff270f5177377113240bf3778 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Mon, 5 Oct 2026 23:08:17 +0000 Subject: [PATCH 1/6] ci(ci): publish the scorecard on every PR to main scorecard.yml plans one matrix entry per built family, measures each family in its own GitHub-hosted job with the rat-stack side cached by rat-stack commit, instrument tree hash and nixpkgs rev, then judges and ratchets against the latest successful main run's scorecard artifact. The table goes to the job summary and scorecard.json is uploaded. Off pull requests a pin job opens the rat-stack pin-bump PR through a GitHub App token scoped to contents and pull requests. The CLI gains plan, aggregate, latest-main-run and pin; the family results carry definition hashes so a changed metric re-baselines --- .github/workflows/scorecard.yml | 169 ++++++++++++++++ evals/ratstack-scorecard/README.md | 11 ++ evals/ratstack-scorecard/flake.nix | 4 +- .../aggregate/families/ratstack-static.json | 69 +++++++ .../aggregate/families/starter-static.json | 69 +++++++ .../aggregate/main/scorecard.json | 151 ++++++++++++++ .../journeys/aggregate.journey.test.ts | 61 ++++++ .../ratstack-scorecard/src/families/static.ts | 30 ++- .../ratstack-scorecard/src/harness/decode.ts | 95 ++++++--- .../src/harness/instrument.ts | 56 ++++-- .../src/harness/scorecard-codec.ts | 146 ++++++++++++++ evals/ratstack-scorecard/src/main.ts | 187 ++++++++++++++++-- .../src/model/cache-key.property.test.ts | 25 +++ .../ratstack-scorecard/src/model/cache-key.ts | 17 ++ evals/ratstack-scorecard/src/model/cell.ts | 1 + .../plan-pin-bump.workflow.property.test.ts | 23 +++ .../src/model/plan-pin-bump.workflow.ts | 16 ++ 17 files changed, 1058 insertions(+), 72 deletions(-) create mode 100644 .github/workflows/scorecard.yml create mode 100644 evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/ratstack-static.json create mode 100644 evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/starter-static.json create mode 100644 evals/ratstack-scorecard/journeys/__fixtures__/aggregate/main/scorecard.json create mode 100644 evals/ratstack-scorecard/journeys/aggregate.journey.test.ts create mode 100644 evals/ratstack-scorecard/src/harness/scorecard-codec.ts create mode 100644 evals/ratstack-scorecard/src/model/cache-key.property.test.ts create mode 100644 evals/ratstack-scorecard/src/model/cache-key.ts create mode 100644 evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.property.test.ts create mode 100644 evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.ts diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 0000000..815f23e --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,169 @@ +name: Scorecard + +on: + pull_request: + branches: [main] + push: + branches: [main] + schedule: + - cron: "17 4 * * *" + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: scorecard-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + SCORECARD_FLAKE: ./evals/ratstack-scorecard + +jobs: + plan: + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@v7 + - uses: cachix/install-nix-action@v31 + - id: plan + name: Families and rat-stack cache keys + run: | + scorecard="$(nix build --no-link --print-out-paths "$SCORECARD_FLAKE#scorecard")" + matrix="$("$scorecard/bin/scorecard" plan)" + echo "$matrix" | jq . + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + + measure: + name: measure (${{ matrix.family }}) + needs: plan + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + runs-on: ubuntu-latest + timeout-minutes: ${{ matrix.timeoutMinutes }} + steps: + - uses: actions/checkout@v7 + - uses: cachix/install-nix-action@v31 + - name: Allow the launcher's unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Build the instrument + run: echo "SCORECARD=$(nix build --no-link --print-out-paths "$SCORECARD_FLAKE#scorecard")/bin/scorecard" >> "$GITHUB_ENV" + - id: ratstack-cache + name: Restore the rat-stack side + uses: actions/cache/restore@v6 + with: + path: scorecard-out/ratstack-${{ matrix.family }}.json + key: ${{ matrix.cacheKey }} + - name: Measure the rat-stack side + if: steps.ratstack-cache.outputs.cache-hit != 'true' + run: | + mkdir -p scorecard-out + "$SCORECARD" measure --family "${{ matrix.family }}" --side ratstack --out "scorecard-out/ratstack-${{ matrix.family }}.json" + - name: Save the rat-stack side for later runs + if: steps.ratstack-cache.outputs.cache-hit != 'true' && github.event_name != 'pull_request' + uses: actions/cache/save@v6 + with: + path: scorecard-out/ratstack-${{ matrix.family }}.json + key: ${{ matrix.cacheKey }} + - name: Measure the starter side + run: | + mkdir -p scorecard-out + "$SCORECARD" measure --family "${{ matrix.family }}" --side starter --out "scorecard-out/starter-${{ matrix.family }}.json" + - uses: actions/upload-artifact@v7 + with: + name: family-${{ matrix.family }} + path: scorecard-out/ + if-no-files-found: error + + aggregate: + needs: [plan, measure] + if: ${{ !cancelled() && needs.plan.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: cachix/install-nix-action@v31 + - name: Build the instrument + run: echo "SCORECARD=$(nix build --no-link --print-out-paths "$SCORECARD_FLAKE#scorecard")/bin/scorecard" >> "$GITHUB_ENV" + - uses: actions/download-artifact@v8 + with: + pattern: family-* + merge-multiple: true + path: families + - id: main + name: Find the latest scorecard on main + env: + GITHUB_TOKEN: ${{ github.token }} + run: echo "run=$("$SCORECARD" latest-main-run)" >> "$GITHUB_OUTPUT" + - name: Download main's scorecard + if: steps.main.outputs.run != '' + uses: actions/download-artifact@v8 + with: + name: scorecard + run-id: ${{ steps.main.outputs.run }} + github-token: ${{ github.token }} + path: main + - name: Judge, ratchet and publish + run: | + main_args=() + if [ -f main/scorecard.json ]; then main_args=(--main main/scorecard.json); fi + status=0 + "$SCORECARD" aggregate --families families "${main_args[@]}" --out scorecard.json --summary summary.md || status=$? + cat summary.md >> "$GITHUB_STEP_SUMMARY" + exit "$status" + - uses: actions/upload-artifact@v7 + if: ${{ !cancelled() }} + with: + name: scorecard + path: scorecard.json + if-no-files-found: error + + pin: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: cachix/install-nix-action@v31 + - name: Build the instrument + run: echo "SCORECARD=$(nix build --no-link --print-out-paths "$SCORECARD_FLAKE#scorecard")/bin/scorecard" >> "$GITHUB_ENV" + - id: check + name: Compare the pin with rat-stack main + run: | + plan="$("$SCORECARD" pin check)" + echo "$plan" | jq . + echo "tag=$(jq -r ._tag <<<"$plan")" >> "$GITHUB_OUTPUT" + echo "to=$(jq -r '.to // empty' <<<"$plan")" >> "$GITHUB_OUTPUT" + echo "Pin: $(jq -r 'if ._tag == "PinCurrent" then "pin current at \(.commit)" else "rat-stack moved \(.from) -> \(.to)" end' <<<"$plan")" >> "$GITHUB_STEP_SUMMARY" + - id: token + if: steps.check.outputs.tag == 'PinMoved' + name: Pin-bump app token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.SCORECARD_APP_ID }} + private-key: ${{ secrets.SCORECARD_APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + - name: Open or update the pin-bump PR + if: steps.check.outputs.tag == 'PinMoved' + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + TO: ${{ steps.check.outputs.to }} + run: | + nar_hash="$(nix flake prefetch --json "github:joelhooks/rat-stack/$TO" | jq -r .hash)" + "$SCORECARD" pin write --commit "$TO" --nar-hash "$nar_hash" + branch="scorecard/pin-rat-stack" + git config user.name "scorecard-pin[bot]" + git config user.email "scorecard-pin[bot]@users.noreply.github.com" + git switch -c "$branch" + git commit -m "chore(deps): pin rat-stack to ${TO:0:7} for the scorecard" -- evals/ratstack-scorecard/ratstack.pin.json + git push --force "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$branch" + if [ -z "$(gh pr list --head "$branch" --json number --jq '.[0].number')" ]; then + gh pr create --base main --head "$branch" \ + --title "chore(deps): pin rat-stack to ${TO:0:7} for the scorecard" \ + --body "rat-stack \`main\` moved to \`$TO\`. This PR moves \`evals/ratstack-scorecard/ratstack.pin.json\` only; its scorecard run measures the new rat-stack. Kiro merges." + fi diff --git a/evals/ratstack-scorecard/README.md b/evals/ratstack-scorecard/README.md index fb3caae..47aeb61 100644 --- a/evals/ratstack-scorecard/README.md +++ b/evals/ratstack-scorecard/README.md @@ -37,3 +37,14 @@ nix develop --command sh -c 'SANDBOX_PROJECT=$PWD sandbox -- pnpm vitest run' ``` `src/main.ts` and everything it imports use only Deno APIs, `node:` builtins and each other, so `DENO_NO_PACKAGE_JSON=1 deno check src/` type-checks the orchestrator and the decision core without third-party code. The Node scripts in `src/tools/` are the only code that loads npm packages, and they only ever run inside the launcher. + +## In CI + +`.github/workflows/scorecard.yml` runs on every PR to `main`, every push to `main`, daily, and on demand. All jobs run on GitHub-hosted runners (`ubuntu-latest`): the self-hosted fleet excludes public repositories by design. + +- **plan** prints the matrix: one entry per built family, with its timeout and the rat-stack cache key (rat-stack commit, instrument tree hash, nixpkgs rev). +- **measure** runs one family per job. The rat-stack side is restored from the cache when the key matches and measured otherwise; only `main`, the schedule and manual runs save it. The starter side is measured every run. Each job uploads `family-`. +- **aggregate** joins the families, compares them with the latest successful `main` run's `scorecard` artifact (`scorecard latest-main-run`; none yet means a first baseline), writes the Markdown table to the job summary, uploads `scorecard.json`, and fails when the ratchet fails. +- **pin** (not on PRs) compares `ratstack.pin.json` with rat-stack `main`. When rat-stack has moved it opens or updates the `scorecard/pin-rat-stack` PR through the pin-bump GitHub App (`vars.SCORECARD_APP_ID`, `secrets.SCORECARD_APP_PRIVATE_KEY`; contents and pull requests write only). That PR changes only the pin; Kiro merges it. + +`actionlint` is in the dev shell: `nix develop ./evals/ratstack-scorecard --command actionlint .github/workflows/scorecard.yml`. diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index f294a97..8b1a6d3 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -50,7 +50,7 @@ export SCORECARD_PNPM_VERSION=${pkgs.pnpm_12.version} export DENO_NO_PACKAGE_JSON=1 exec deno run --no-config --allow-read --allow-write --allow-env --allow-sys=hostname \ - --allow-run=git,${sandbox}/bin/sandbox \ + --allow-net=api.github.com --allow-run=git,${sandbox}/bin/sandbox \ ${self}/src/main.ts "$@" ''; }; @@ -63,7 +63,7 @@ devShells = forEachSystem (pkgs: let system = pkgs.stdenv.hostPlatform.system; in { default = pkgs.mkShell { - packages = [ pkgs.nodejs_24 pkgs.pnpm_12 pkgs.deno self.packages.${system}.sandbox ]; + packages = [ pkgs.nodejs_24 pkgs.pnpm_12 pkgs.deno pkgs.actionlint self.packages.${system}.sandbox ]; SANDBOX_PNPM_STORE = self.packages.${system}.tools-store; }; }); diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/ratstack-static.json b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/ratstack-static.json new file mode 100644 index 0000000..e5ff4a3 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/ratstack-static.json @@ -0,0 +1,69 @@ +{ + "family": "static", + "wallMs": 3000, + "flags": [], + "definitionHashes": [ + { + "id": "M23", + "hash": "h-m23" + }, + { + "id": "M28", + "hash": "h-m28" + } + ], + "cells": [ + { + "id": "M23", + "side": "ratstack", + "measured": { + "cell": { + "_tag": "Measured", + "runs": [ + 219, + 219, + 219 + ] + }, + "provenance": { + "side": "ratstack", + "commit": "54d356037c994f89698760a4727be71d0005a087", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + } + }, + { + "id": "M28", + "side": "ratstack", + "measured": { + "cell": { + "_tag": "Measured", + "runs": [ + 957, + 957, + 957 + ] + }, + "provenance": { + "side": "ratstack", + "commit": "54d356037c994f89698760a4727be71d0005a087", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + } + } + ] +} diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/starter-static.json b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/starter-static.json new file mode 100644 index 0000000..c0b4cd7 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/starter-static.json @@ -0,0 +1,69 @@ +{ + "family": "static", + "wallMs": 3000, + "flags": [], + "definitionHashes": [ + { + "id": "M23", + "hash": "h-m23" + }, + { + "id": "M28", + "hash": "h-m28" + } + ], + "cells": [ + { + "id": "M23", + "side": "starter", + "measured": { + "cell": { + "_tag": "Measured", + "runs": [ + 219, + 219, + 219 + ] + }, + "provenance": { + "side": "starter", + "commit": "2222222222222222222222222222222222222222", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + } + }, + { + "id": "M28", + "side": "starter", + "measured": { + "cell": { + "_tag": "Measured", + "runs": [ + 392, + 392, + 392 + ] + }, + "provenance": { + "side": "starter", + "commit": "2222222222222222222222222222222222222222", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + } + } + ] +} diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/main/scorecard.json b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/main/scorecard.json new file mode 100644 index 0000000..8744633 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/main/scorecard.json @@ -0,0 +1,151 @@ +{ + "schemaVersion": 1, + "provenance": { + "commit": "3333333333333333333333333333333333333333", + "ratstackCommit": "54d356037c994f89698760a4727be71d0005a087", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "generatedAt": "2026-10-05T21:00:00.000Z" + }, + "rows": [ + { + "definition": { + "id": "M23", + "metric": "M23", + "bin": "fence: debt ledger", + "label": "Suppression directives in tracked source", + "unit": "directives", + "direction": "lower", + "kind": "count", + "runs": 3, + "family": "static" + }, + "definitionHash": "h-m23", + "ratstack": { + "cell": { + "_tag": "Measured", + "runs": [ + 219, + 219, + 219 + ] + }, + "provenance": { + "side": "ratstack", + "commit": "54d356037c994f89698760a4727be71d0005a087", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + }, + "starter": { + "cell": { + "_tag": "Measured", + "runs": [ + 0, + 0, + 0 + ] + }, + "provenance": { + "side": "starter", + "commit": "3333333333333333333333333333333333333333", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + }, + "verdict": { + "_tag": "Beaten" + }, + "flags": [] + }, + { + "definition": { + "id": "M28", + "metric": "M28", + "bin": "install", + "label": "Distinct name@version packages in the side's lockfile", + "unit": "packages", + "direction": "lower", + "kind": "count", + "runs": 3, + "family": "static" + }, + "definitionHash": "h-m28", + "ratstack": { + "cell": { + "_tag": "Measured", + "runs": [ + 957, + 957, + 957 + ] + }, + "provenance": { + "side": "ratstack", + "commit": "54d356037c994f89698760a4727be71d0005a087", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + }, + "starter": { + "cell": { + "_tag": "Measured", + "runs": [ + 392, + 392, + 392 + ] + }, + "provenance": { + "side": "starter", + "commit": "3333333333333333333333333333333333333333", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + }, + "verdict": { + "_tag": "Beaten" + }, + "flags": [] + } + ], + "ratchet": { + "_tag": "FirstBaseline", + "outcomes": [ + { + "_tag": "New", + "id": "M23" + }, + { + "_tag": "New", + "id": "M28" + } + ], + "failures": [] + } +} diff --git a/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts b/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts new file mode 100644 index 0000000..0b9b8a4 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts @@ -0,0 +1,61 @@ +import Ajv from 'ajv' +import { execFile } from 'node:child_process' +import { mkdtemp, readFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, test } from 'vitest' +import schema from '../scorecard.schema.json' with { type: 'json' } + +const instrument = join(import.meta.dirname, '..') +const fixtures = join(import.meta.dirname, '__fixtures__/aggregate') + +interface Exit { + readonly code: number + readonly stderr: string +} + +const aggregate = (args: readonly string[]): Promise => { + const { promise, resolve } = Promise.withResolvers() + execFile( + 'deno', + ['run', '--no-config', '--allow-read', '--allow-write', '--allow-env', 'src/main.ts', 'aggregate', ...args], + { cwd: instrument, env: { ...process.env, DENO_NO_PACKAGE_JSON: '1' } }, + (error, _stdout, stderr) => resolve({ code: error === null ? 0 : Number(error.code), stderr }), + ) + return promise +} + +describe('aggregate through the real CLI (J3)', () => { + test('a row beaten on main and tied on the PR fails the job, names the row, and still writes a valid scorecard', async () => { + const out = await mkdtemp(join(tmpdir(), 'aggregate-')) + const exit = await aggregate([ + '--families', + join(fixtures, 'families'), + '--main', + join(fixtures, 'main/scorecard.json'), + '--out', + join(out, 'scorecard.json'), + '--summary', + join(out, 'summary.md'), + ]) + const written = JSON.parse(await readFile(join(out, 'scorecard.json'), 'utf8')) + const validate = new Ajv({ allErrors: true, strict: true }).compile(schema) + expect(exit.code).toBe(1) + expect(exit.stderr).toContain('M23') + expect(exit.stderr).not.toContain('M28') + expect(validate(written)).toBe(true) + }) + + test('without a main artifact the same families are a passing first baseline', async () => { + const out = await mkdtemp(join(tmpdir(), 'aggregate-')) + const exit = await aggregate([ + '--families', + join(fixtures, 'families'), + '--out', + join(out, 'scorecard.json'), + '--summary', + join(out, 'summary.md'), + ]) + expect(exit.code).toBe(0) + }) +}) diff --git a/evals/ratstack-scorecard/src/families/static.ts b/evals/ratstack-scorecard/src/families/static.ts index 45b7414..3717ed2 100644 --- a/evals/ratstack-scorecard/src/families/static.ts +++ b/evals/ratstack-scorecard/src/families/static.ts @@ -1,6 +1,7 @@ import { join } from 'node:path' -import { arrayAt, numberAt, stringAt } from '../harness/decode.ts' +import { array, decodeJson, number, string, struct } from '../harness/decode.ts' import { + definitionHashesFor, type Instrument, materializeRatstack, materializeStarter, @@ -19,6 +20,12 @@ import { starter } from '../sides/starter.ts' const runs = 3 +const extracted = struct({ files: array(struct({ path: string, comments: array(string), errors: number })) }) + +const lockfileDocuments = struct({ + documents: array(struct({ lockfileVersion: string, packages: array(string), errors: number })), +}) + interface Subject { readonly adapter: SideAdapter readonly root: string @@ -77,15 +84,14 @@ const countDirectives = async ( if (result.code !== 0) { return instrumentError(`extract-comments exited ${result.code}: ${result.stderr.slice(-2000)}`) } - const parsed = JSON.parse(await Deno.readTextFile(output)) - const files = arrayAt(parsed, ['files'], 'extract-comments output') - const comments = files.flatMap((file) => arrayAt(file, ['comments'], 'extract-comments file').map(String)) + const { files } = decodeJson(extracted, await Deno.readTextFile(output), 'extract-comments output') + const comments = files.flatMap((file) => file.comments) const tally = tallyDirectives(comments) totals.push(totalDirectives(tally)) detail = { ...tally, filesCounted: counted.length, - parseErrors: files.reduce((sum, file) => sum + numberAt(file, ['errors'], 'extract-comments file'), 0), + parseErrors: files.reduce((sum, file) => sum + file.errors, 0), ...Object.fromEntries(excluded.map((v) => [`excluded:${v.root}`, `${v.files} files: ${v.reason}`])), } } @@ -105,13 +111,13 @@ const countPackages = async ( const output = join(work, `lockfile-${subject.adapter.side}-${run}.json`) const result = await node(instrument, work, tools, ['src/tools/parse-lockfile.mjs', lockfile, output]) if (result.code !== 0) return instrumentError(`parse-lockfile exited ${result.code}: ${result.stderr.slice(-2000)}`) - const documents = arrayAt(JSON.parse(await Deno.readTextFile(output)), ['documents'], 'parse-lockfile output') - const keys = documents.flatMap((doc) => arrayAt(doc, ['packages'], 'lockfile document').map(String)) + const { documents } = decodeJson(lockfileDocuments, await Deno.readTextFile(output), 'parse-lockfile output') + const keys = documents.flatMap((doc) => doc.packages) counts.push(distinctPackages(keys).length) detail = { lockfile: subject.adapter.lockfile, documents: documents.length, - lockfileVersions: documents.map((doc) => stringAt(doc, ['lockfileVersion'], 'lockfile document')).join(', '), + lockfileVersions: documents.map((doc) => doc.lockfileVersion).join(', '), } } return { cell: { _tag: 'Measured', runs: counts }, detail } @@ -138,5 +144,11 @@ export const measureStatic = async ( }, ) } - return { family: 'static', wallMs: Math.round(performance.now() - started), cells, flags: [] } + return { + family: 'static', + wallMs: Math.round(performance.now() - started), + cells, + flags: [], + definitionHashes: await definitionHashesFor(instrument, 'static'), + } } diff --git a/evals/ratstack-scorecard/src/harness/decode.ts b/evals/ratstack-scorecard/src/harness/decode.ts index e271455..01a2ae8 100644 --- a/evals/ratstack-scorecard/src/harness/decode.ts +++ b/evals/ratstack-scorecard/src/harness/decode.ts @@ -1,41 +1,74 @@ -export const at = (value: unknown, key: string): unknown => - typeof value === 'object' && value !== null && key in value - ? Object.getOwnPropertyDescriptor(value, key)?.value - : undefined +export type Decoder = (value: unknown, path: string) => T -const pathOf = (value: unknown, path: readonly string[]): unknown => path.reduce(at, value) - -const refuse = (what: string, path: readonly string[], expected: string): never => { - throw new Error(`${what}: expected ${expected} at ${path.join('.') || ''}`) +export class DecodeError extends Error { + constructor(readonly path: string, readonly expected: string, readonly found: unknown) { + super(`expected ${expected} at ${path || ''}, found ${JSON.stringify(found)?.slice(0, 80)}`) + } } -export const stringAt = (value: unknown, path: readonly string[], what: string): string => { - const found = pathOf(value, path) - return typeof found === 'string' ? found : refuse(what, path, 'a string') +const fail = (path: string, expected: string, found: unknown): never => { + throw new DecodeError(path, expected, found) } -export const numberAt = (value: unknown, path: readonly string[], what: string): number => { - const found = pathOf(value, path) - return typeof found === 'number' ? found : refuse(what, path, 'a number') -} +const isObject = (value: unknown): value is object => + typeof value === 'object' && value !== null && !Array.isArray(value) -export const arrayAt = (value: unknown, path: readonly string[], what: string): readonly unknown[] => { - const found = pathOf(value, path) - return Array.isArray(found) ? found : refuse(what, path, 'an array') -} +const fieldOf = (value: object, key: string): unknown => Object.getOwnPropertyDescriptor(value, key)?.value + +export const string: Decoder = (value, path) => + typeof value === 'string' ? value : fail(path, 'a string', value) + +export const number: Decoder = (value, path) => + typeof value === 'number' && Number.isFinite(value) ? value : fail(path, 'a finite number', value) + +export const literal = (expected: L): Decoder => (value, path) => + value === expected ? expected : fail(path, JSON.stringify(expected), value) + +export const oneOf = (options: readonly L[]): Decoder => (value, path) => + options.find((option) => option === value) ?? fail(path, `one of ${options.join(', ')}`, value) + +export const array = (item: Decoder): Decoder => (value, path) => + Array.isArray(value) ? value.map((entry, index) => item(entry, `${path}[${index}]`)) : fail(path, 'an array', value) + +export const tuple2 = (first: Decoder, second: Decoder): Decoder => (value, path) => + Array.isArray(value) && value.length === 2 + ? [first(value[0], `${path}[0]`), second(value[1], `${path}[1]`)] + : fail(path, 'a pair', value) + +export const record = (entry: Decoder): Decoder>> => (value, path) => + isObject(value) + ? Object.fromEntries(Object.entries(value).map(([key, item]) => [key, entry(item, `${path}.${key}`)])) + : fail(path, 'an object', value) + +type Fields = Readonly>> +type Decoded = { readonly [K in keyof F]: F[K] extends Decoder ? T : never } + +export const struct = (fields: F): Decoder> => (value, path) => + isObject(value) + ? Object.fromEntries( + Object.entries(fields).map(([key, decode]) => [key, decode(fieldOf(value, key), `${path}.${key}`)]), + ) as Decoded + : fail(path, 'an object', value) + +export const optional = (decode: Decoder): Decoder => (value, path) => + value === undefined ? undefined : decode(value, path) -export const entriesAt = ( - value: unknown, - path: readonly string[], - what: string, -): readonly (readonly [string, unknown])[] => { - const found = pathOf(value, path) - return typeof found === 'object' && found !== null && !Array.isArray(found) - ? Object.entries(found) - : refuse(what, path, 'an object') +export const union = (...options: readonly Decoder[]): Decoder => (value, path) => { + const errors: string[] = [] + for (const option of options) { + try { + return option(value, path) + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)) + } + } + return fail(path, `one of ${options.length} shapes (${errors.join('; ')})`, value) } -export const optionalEntriesAt = (value: unknown, path: readonly string[]): readonly (readonly [string, unknown])[] => { - const found = pathOf(value, path) - return typeof found === 'object' && found !== null && !Array.isArray(found) ? Object.entries(found) : [] +export const decodeJson = (decode: Decoder, text: string, what: string): T => { + try { + return decode(JSON.parse(text), what) + } catch (error) { + throw new Error(`${what}: ${error instanceof Error ? error.message : String(error)}`) + } } diff --git a/evals/ratstack-scorecard/src/harness/instrument.ts b/evals/ratstack-scorecard/src/harness/instrument.ts index 7393b3f..04d9aca 100644 --- a/evals/ratstack-scorecard/src/harness/instrument.ts +++ b/evals/ratstack-scorecard/src/harness/instrument.ts @@ -1,6 +1,7 @@ import { dirname, join } from 'node:path' -import type { CellProvenance, Side } from '../model/cell.ts' -import { optionalEntriesAt, stringAt } from './decode.ts' +import { rowDefinitions } from '../metrics/registry.ts' +import type { CellProvenance, Family, Side } from '../model/cell.ts' +import { decodeJson, type Decoder, record, string, struct } from './decode.ts' import { type Launcher, runSandboxed } from './sandbox.ts' export interface Pin { @@ -43,19 +44,14 @@ export const git = async (cwd: string, args: readonly string[]): Promise => decoder.decode(await git(cwd, args)).trim() -const readJson = async (path: string): Promise => JSON.parse(await Deno.readTextFile(path)) +const readJson = async (decode: Decoder, path: string): Promise => + decodeJson(decode, await Deno.readTextFile(path), path) -const nixpkgsRevOf = (lock: unknown): string => stringAt(lock, ['nodes', 'nixpkgs', 'locked', 'rev'], 'flake.lock') +const flakeLock = struct({ nodes: struct({ nixpkgs: struct({ locked: struct({ rev: string }) }) }) }) -const pinOf = (pin: unknown): Pin => ({ - owner: stringAt(pin, ['owner'], 'ratstack.pin.json'), - repo: stringAt(pin, ['repo'], 'ratstack.pin.json'), - commit: stringAt(pin, ['commit'], 'ratstack.pin.json'), - narHash: stringAt(pin, ['narHash'], 'ratstack.pin.json'), -}) +const pinFile = struct({ owner: string, repo: string, commit: string, narHash: string }) -const dependencyVersions = (manifest: unknown): Record => - Object.fromEntries(optionalEntriesAt(manifest, ['dependencies']).map(([name, version]) => [name, String(version)])) +const manifest = struct({ dependencies: record(string) }) export const loadInstrument = async (checkoutArg: string | undefined): Promise => { const dir = required('SCORECARD_INSTRUMENT') @@ -66,13 +62,13 @@ export const loadInstrument = async (checkoutArg: string | undefined): Promise [ + `src/families/${family}.ts`, + 'src/sides', + 'src/tools', + 'src/harness', + 'flake.lock', + 'pnpm-lock.yaml', +] + +const sha256 = async (text: string): Promise => + [...new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(text)))] + .map((byte) => byte.toString(16).padStart(2, '0')) + .join('') + +export const definitionHashesFor = async ( + instrument: Instrument, + family: Family, +): Promise => { + const trees = await Promise.all( + definitionInputs(family).map(async (path) => + `${path}=${await gitText(instrument.checkout, ['rev-parse', `HEAD:evals/ratstack-scorecard/${path}`])}` + ), + ) + return await Promise.all( + rowDefinitions + .filter((row) => row.family === family) + .map(async (row) => ({ id: row.id, hash: await sha256(JSON.stringify({ row, trees })) })), + ) +} + const copyTracked = async (from: string, to: string, files: readonly string[]): Promise => { for (const file of files) { const target = join(to, file) diff --git a/evals/ratstack-scorecard/src/harness/scorecard-codec.ts b/evals/ratstack-scorecard/src/harness/scorecard-codec.ts new file mode 100644 index 0000000..740fcd7 --- /dev/null +++ b/evals/ratstack-scorecard/src/harness/scorecard-codec.ts @@ -0,0 +1,146 @@ +import type { + Cell, + CellProvenance, + FamilyResult, + Flag, + MeasuredCell, + Ratchet, + Row, + RowDefinition, + RowOutcome, + ScorecardDocument, + Verdict, +} from '../model/cell.ts' +import { + array, + type Decoder, + literal, + number, + oneOf, + optional, + record, + string, + struct, + tuple2, + union, +} from './decode.ts' + +const cellTags = [ + 'Measured', + 'Absent', + 'NoDeployment', + 'Unsupported', + 'Unmeasurable', + 'NoSecret', + 'InstrumentError', +] as const + +const side = oneOf(['ratstack', 'starter']) + +export const cell: Decoder = union( + struct({ _tag: literal('Measured'), runs: array(number) }), + struct({ _tag: literal('Absent'), reason: string }), + struct({ _tag: literal('NoDeployment'), sha: string }), + struct({ + _tag: literal('Unsupported'), + citation: struct({ file: string, lines: tuple2(number, number), text: string }), + check: union<{ readonly _tag: 'Verified' } | { readonly _tag: 'Contradicted'; readonly found: string }>( + struct({ _tag: literal('Verified') }), + struct({ _tag: literal('Contradicted'), found: string }), + ), + }), + struct({ _tag: literal('Unmeasurable'), error: string }), + struct({ _tag: literal('NoSecret'), name: string }), + struct({ _tag: literal('InstrumentError'), error: string }), +) + +const provenanceFields = struct({ + side, + commit: string, + instrumentHash: string, + nixpkgsRev: string, + runner: string, + measuredAt: string, + tools: record(string), + liveCommit: optional(string), + detail: optional(record(union(number, string))), +}) + +export const cellProvenance: Decoder = (value, path) => { + const { liveCommit, detail, ...required } = provenanceFields(value, path) + return { + ...required, + ...(liveCommit === undefined ? {} : { liveCommit }), + ...(detail === undefined ? {} : { detail }), + } +} + +const measuredCell: Decoder = struct({ cell, provenance: cellProvenance }) + +const flag: Decoder = union( + struct({ _tag: literal('LiveDiffersFromPin'), live: string, pin: string }), + struct({ _tag: literal('ScannersDisagree'), primary: number, crossCheck: number }), +) + +const rowDefinition: Decoder = struct({ + id: string, + metric: string, + bin: string, + label: string, + unit: string, + direction: oneOf(['lower', 'higher']), + kind: oneOf(['count', 'measurement']), + runs: number, + family: oneOf(['static', 'cold-path', 'gate-mutation', 'running-stack', 'agent-surfaces', 'networked']), +}) + +const verdict: Decoder = union( + struct({ _tag: oneOf(['Beaten', 'NotBeaten', 'Tie']) }), + struct({ _tag: literal('InstrumentError'), error: string }), +) + +const row: Decoder = struct({ + definition: rowDefinition, + definitionHash: string, + ratstack: measuredCell, + starter: measuredCell, + verdict, + flags: array(flag), +}) + +const cellTag = oneOf(cellTags) + +const outcome: Decoder = union( + struct({ _tag: oneOf(['Held', 'New', 'ReBaselined']), id: string }), + struct({ _tag: literal('Neutral'), id: string, cause: cellTag }), + struct({ _tag: literal('InstrumentError'), id: string, error: string }), + struct({ _tag: literal('LostBeaten'), id: string, ratstack: cellTag, starter: cellTag }), + struct({ _tag: literal('Regressed'), id: string, main: string, pr: string }), +) + +const ratchet: Decoder = union( + struct({ _tag: literal('FirstBaseline'), outcomes: array(outcome), failures: array(outcome) }), + struct({ _tag: literal('Compared'), mainCommit: string, outcomes: array(outcome), failures: array(outcome) }), +) + +export const scorecardDocument: Decoder = struct({ + schemaVersion: literal(1), + provenance: struct({ + commit: string, + ratstackCommit: string, + instrumentHash: string, + nixpkgsRev: string, + runner: string, + generatedAt: string, + }), + rows: array(row), + ratchet, +}) + +export const familyResult: Decoder = struct({ + family: oneOf(['static', 'cold-path', 'gate-mutation', 'running-stack', 'agent-surfaces', 'networked']), + wallMs: number, + cells: array(struct({ id: string, side, measured: measuredCell })), + flags: array(struct({ id: string, flag })), + definitionHashes: array(struct({ id: string, hash: string })), +}) diff --git a/evals/ratstack-scorecard/src/main.ts b/evals/ratstack-scorecard/src/main.ts index 2a1ca1f..862af35 100644 --- a/evals/ratstack-scorecard/src/main.ts +++ b/evals/ratstack-scorecard/src/main.ts @@ -2,27 +2,63 @@ import { join } from 'node:path' import { parseArgs } from 'node:util' import { checkImports } from './check-imports.ts' import { measureStatic } from './families/static.ts' -import { type Instrument, loadInstrument } from './harness/instrument.ts' +import { array, decodeJson, number, string, struct } from './harness/decode.ts' +import { git, type Instrument, loadInstrument } from './harness/instrument.ts' +import { familyResult, scorecardDocument } from './harness/scorecard-codec.ts' import { runJourneys } from './journeys.ts' -import type { FamilyResult, Side } from './model/cell.ts' +import { familyTimeoutMinutes, rowDefinitions } from './metrics/registry.ts' +import { ratstackCacheKey } from './model/cache-key.ts' +import type { Family, FamilyResult, MainBaseline, Side } from './model/cell.ts' +import { planPinBump } from './model/plan-pin-bump.workflow.ts' +import { assembleScorecard } from './model/scorecard-document.ts' +import { renderSummary } from './model/summary-table.ts' const usage = [ 'usage:', + ' scorecard plan', ' scorecard measure --family [--side ratstack|starter] [--out ] [--checkout ]', + ' scorecard aggregate --families [--main ] --out --summary ', + ' scorecard latest-main-run', + ' scorecard pin check', + ' scorecard pin write --commit --nar-hash [--checkout ]', ' scorecard journeys [--checkout ]', ' scorecard check [--checkout ] (import-graph rules, then journeys)', ].join('\n') -const families: Readonly< - Record Promise> -> = { - static: measureStatic, +type FamilyRunner = (instrument: Instrument, work: string, sides: readonly Side[]) => Promise + +const familyRunners: readonly (readonly [Family, FamilyRunner])[] = [['static', measureStatic]] + +const implementedFamilies: readonly Family[] = familyRunners.map(([family]) => family) + +const fail = (message: string): never => { + throw new Error(`${message}\n${usage}`) +} + +const writeOut = async (path: string | undefined, text: string): Promise => { + if (path === undefined) await Deno.stdout.write(new TextEncoder().encode(text)) + else await Deno.writeTextFile(path, text) } const sidesOf = (side: string | undefined): readonly Side[] => { if (side === undefined) return ['ratstack', 'starter'] if (side === 'ratstack' || side === 'starter') return [side] - throw new Error(`unknown side ${side}\n${usage}`) + return fail(`unknown side ${side}`) +} + +const plan = async (): Promise => { + const instrument = await loadInstrument(undefined) + const matrix = implementedFamilies.map((family) => ({ + family, + timeoutMinutes: familyTimeoutMinutes[family], + cacheKey: ratstackCacheKey({ + family, + ratstackCommit: instrument.pin.commit, + instrumentHash: instrument.instrumentHash, + nixpkgsRev: instrument.nixpkgsRev, + }), + })) + await writeOut(undefined, `${JSON.stringify({ include: matrix })}\n`) } const measure = async (args: readonly string[]): Promise => { @@ -36,15 +72,13 @@ const measure = async (args: readonly string[]): Promise => { }, strict: true, }) - const run = families[values.family ?? ''] - if (run === undefined) throw new Error(`unknown family ${values.family}\n${usage}`) + const family = values.family ?? fail('--family is required') + const run = familyRunners.find(([name]) => name === family)?.[1] ?? fail(`unknown or unbuilt family ${family}`) const instrument = await loadInstrument(values.checkout) - const work = await Deno.makeTempDir({ prefix: `scorecard-${values.family}-` }) + const work = await Deno.makeTempDir({ prefix: `scorecard-${family}-` }) try { - const result: FamilyResult = await run(instrument, work, sidesOf(values.side)) - const json = `${JSON.stringify(result, null, 2)}\n` - if (values.out === undefined) await Deno.stdout.write(new TextEncoder().encode(json)) - else await Deno.writeTextFile(values.out, json) + const result = await run(instrument, work, sidesOf(values.side)) + await writeOut(values.out, `${JSON.stringify(result, null, 2)}\n`) } finally { await Deno.remove(work, { recursive: true }) } @@ -67,7 +101,130 @@ const check = async (args: readonly string[]): Promise => { Deno.exit(await runJourneys(instrument, root)) } -const commands: Readonly Promise>> = { measure, journeys, check } +const readFamilyResults = async (dir: string): Promise => { + const results: FamilyResult[] = [] + for await (const entry of Deno.readDir(dir)) { + if (!entry.isFile || !entry.name.endsWith('.json')) continue + const path = join(dir, entry.name) + results.push(decodeJson(familyResult, await Deno.readTextFile(path), path)) + } + return results +} + +const mainBaseline = async (path: string | undefined): Promise => { + if (path === undefined) return { _tag: 'Missing' } + const doc = decodeJson(scorecardDocument, await Deno.readTextFile(path), path) + return { _tag: 'Found', commit: doc.provenance.commit, rows: doc.rows } +} + +const aggregate = async (args: readonly string[]): Promise => { + const { values } = parseArgs({ + args: [...args], + options: { + families: { type: 'string' }, + main: { type: 'string' }, + out: { type: 'string' }, + summary: { type: 'string' }, + }, + strict: true, + }) + const results = await readFamilyResults(values.families ?? fail('--families is required')) + const cells = results.flatMap((result) => result.cells) + const hashes = results.flatMap((result) => result.definitionHashes) + const provenanceOf = (side: Side) => cells.find((cell) => cell.side === side)?.measured.provenance + const starter = provenanceOf('starter') + const ratstack = provenanceOf('ratstack') + const any = starter ?? ratstack ?? fail(`no family results with cells in ${values.families}`) + const doc = assembleScorecard({ + rows: rowDefinitions + .filter((definition) => implementedFamilies.includes(definition.family)) + .map((definition) => ({ + definition, + hash: hashes.find((hash) => hash.id === definition.id)?.hash ?? `no ${definition.family} result`, + })), + cells, + flags: results.flatMap((result) => result.flags), + provenance: { + commit: starter?.commit ?? Deno.env.get('GITHUB_SHA') ?? 'unknown', + ratstackCommit: ratstack?.commit ?? 'unknown', + instrumentHash: any.instrumentHash, + nixpkgsRev: any.nixpkgsRev, + runner: any.runner, + generatedAt: new Date().toISOString(), + }, + main: await mainBaseline(values.main), + }) + await writeOut(values.out ?? fail('--out is required'), `${JSON.stringify(doc, null, 2)}\n`) + await writeOut(values.summary ?? fail('--summary is required'), renderSummary(doc)) + const failures = doc.ratchet.failures + if (failures.length > 0) { + console.error(`scorecard: ${failures.length} failing rows: ${failures.map((failure) => failure.id).join(', ')}`) + Deno.exit(1) + } +} + +const workflowRuns = struct({ workflow_runs: array(struct({ id: number, head_sha: string })) }) + +const latestMainRun = async (): Promise => { + const api = Deno.env.get('GITHUB_API_URL') ?? 'https://api.github.com' + const repository = Deno.env.get('GITHUB_REPOSITORY') ?? fail('GITHUB_REPOSITORY is unset') + const token = Deno.env.get('GITHUB_TOKEN') ?? fail('GITHUB_TOKEN is unset') + const response = await fetch( + `${api}/repos/${repository}/actions/workflows/scorecard.yml/runs?branch=main&event=push&status=success&per_page=1`, + { headers: { authorization: `Bearer ${token}`, accept: 'application/vnd.github+json' } }, + ) + if (response.status === 404) { + await response.body?.cancel() + await writeOut(undefined, '\n') + return + } + if (!response.ok) fail(`listing scorecard runs failed: ${response.status} ${await response.text()}`) + const { workflow_runs } = decodeJson(workflowRuns, await response.text(), 'GitHub workflow runs') + await writeOut(undefined, `${workflow_runs.map((run) => run.id).join('')}\n`) +} + +const pinCommand = async (args: readonly string[]): Promise => { + const [action, ...rest] = args + const { values } = parseArgs({ + args: rest, + options: { 'commit': { type: 'string' }, 'nar-hash': { type: 'string' }, 'checkout': { type: 'string' } }, + strict: true, + }) + const instrument = await loadInstrument(values.checkout) + if (action === 'check') { + const remote = new TextDecoder().decode( + await git(instrument.checkout, [ + 'ls-remote', + `https://github.com/${instrument.pin.owner}/${instrument.pin.repo}`, + 'refs/heads/main', + ]), + ) + const remoteHead = string(remote.split(/\s/u)[0], 'ls-remote refs/heads/main') + await writeOut(undefined, `${JSON.stringify(planPinBump({ pinned: instrument.pin.commit, remoteHead }))}\n`) + return + } + if (action === 'write') { + const pin = { + ...instrument.pin, + commit: values.commit ?? fail('--commit is required'), + narHash: values['nar-hash'] ?? fail('--nar-hash is required'), + } + const path = join(instrument.checkout, 'evals/ratstack-scorecard/ratstack.pin.json') + await Deno.writeTextFile(path, `${JSON.stringify(pin, null, 2)}\n`) + return + } + fail(`unknown pin action ${action}`) +} + +const commands: Readonly Promise>> = { + 'plan': () => plan(), + 'measure': measure, + 'aggregate': aggregate, + 'latest-main-run': () => latestMainRun(), + 'pin': pinCommand, + 'journeys': journeys, + 'check': check, +} const [command, ...rest] = Deno.args const handler = commands[command ?? ''] diff --git a/evals/ratstack-scorecard/src/model/cache-key.property.test.ts b/evals/ratstack-scorecard/src/model/cache-key.property.test.ts new file mode 100644 index 0000000..824088a --- /dev/null +++ b/evals/ratstack-scorecard/src/model/cache-key.property.test.ts @@ -0,0 +1,25 @@ +import { fc, test } from '@fast-check/vitest' +import { describe } from 'vitest' +import { ratstackCacheKey } from './cache-key.ts' +import { sha } from './scorecard.arbitrary.ts' + +const family = fc.constantFrom( + 'static' as const, + 'cold-path' as const, + 'gate-mutation' as const, + 'running-stack' as const, + 'agent-surfaces' as const, + 'networked' as const, +) + +const inputs = fc.record({ family, ratstackCommit: sha, instrumentHash: sha, nixpkgsRev: sha }) + +describe('ratstackCacheKey', () => { + test.prop([inputs, fc.constantFrom('family', 'ratstackCommit', 'instrumentHash', 'nixpkgsRev' as const), inputs])( + 'a key moves exactly when the family, the rat-stack commit, the instrument hash or the nixpkgs rev moves', + (base, field, other) => { + const changed = { ...base, [field]: other[field] } + return (ratstackCacheKey(base) === ratstackCacheKey(changed)) === (base[field] === other[field]) + }, + ) +}) diff --git a/evals/ratstack-scorecard/src/model/cache-key.ts b/evals/ratstack-scorecard/src/model/cache-key.ts new file mode 100644 index 0000000..78249cf --- /dev/null +++ b/evals/ratstack-scorecard/src/model/cache-key.ts @@ -0,0 +1,17 @@ +import type { Family } from './cell.ts' + +export interface RatstackCacheInputs { + readonly family: Family + readonly ratstackCommit: string + readonly instrumentHash: string + readonly nixpkgsRev: string +} + +export const ratstackCacheKey = (inputs: RatstackCacheInputs): string => + [ + 'scorecard-ratstack', + inputs.family, + `rat-stack=${inputs.ratstackCommit}`, + `instrument=${inputs.instrumentHash}`, + `nixpkgs=${inputs.nixpkgsRev}`, + ].join('/') diff --git a/evals/ratstack-scorecard/src/model/cell.ts b/evals/ratstack-scorecard/src/model/cell.ts index 97f086d..7e98f19 100644 --- a/evals/ratstack-scorecard/src/model/cell.ts +++ b/evals/ratstack-scorecard/src/model/cell.ts @@ -107,6 +107,7 @@ export interface FamilyResult { readonly wallMs: number readonly cells: readonly SideCell[] readonly flags: readonly RowFlag[] + readonly definitionHashes: readonly { readonly id: string; readonly hash: string }[] } export type RowOutcome = diff --git a/evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.property.test.ts b/evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.property.test.ts new file mode 100644 index 0000000..a3c472f --- /dev/null +++ b/evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.property.test.ts @@ -0,0 +1,23 @@ +import { fc, test } from '@fast-check/vitest' +import { describe } from 'vitest' +import { planPinBump } from './plan-pin-bump.workflow.ts' +import { sha } from './scorecard.arbitrary.ts' + +const otherThan = (commit: string, index: number): string => { + const at = index % commit.length + const replacement = commit[at] === 'a' ? 'b' : 'a' + return `${commit.slice(0, at)}${replacement}${commit.slice(at + 1)}` +} + +describe('planPinBump', () => { + test.prop([sha])( + 'a remote head equal to the pin plans nothing', + (commit) => planPinBump({ pinned: commit, remoteHead: commit })._tag === 'PinCurrent', + ) + + test.prop([sha, fc.nat()])('any other remote head plans a bump to exactly that commit', (pinned, index) => { + const remoteHead = otherThan(pinned, index) + const plan = planPinBump({ pinned, remoteHead }) + return plan._tag === 'PinMoved' && plan.to === remoteHead && plan.from === pinned + }) +}) diff --git a/evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.ts b/evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.ts new file mode 100644 index 0000000..393fc15 --- /dev/null +++ b/evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.ts @@ -0,0 +1,16 @@ +import { firstRule } from './dispatch.ts' + +export interface PlanPinBumpInput { + readonly pinned: string + readonly remoteHead: string +} + +export type PinPlan = + | { readonly _tag: 'PinCurrent'; readonly commit: string } + | { readonly _tag: 'PinMoved'; readonly from: string; readonly to: string } + +export const planPinBump = (input: PlanPinBumpInput): PinPlan => + firstRule( + [[input.remoteHead === input.pinned, () => ({ _tag: 'PinCurrent', commit: input.pinned })]], + () => ({ _tag: 'PinMoved', from: input.pinned, to: input.remoteHead }), + ) From 32901d8d256975a06f066921d492b92d56ba3cf0 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Oct 2026 05:10:12 +0000 Subject: [PATCH 2/6] fix(repo): hash only what defines a metric and produce J3 through the driver Review fix #9 (part): a row's definition hash covers its registry entry and its family's measurement code only; harness, sides, tools, lockfiles and nixpkgs move the cache key instead. J3 becomes a two-phase journey: the host driver runs the real scorecard aggregate CLI and records it; the sandboxed test decodes the record. Restack repair: assemble keys cells by row and side (DuplicateCell refused at decode, #18), the codec decodes ratstackSupport, and the journey manifest decodes through the shared combinators --- evals/ratstack-scorecard/flake.nix | 2 +- .../aggregate/main/scorecard.json | 10 +- .../journeys/aggregate.journey.test.ts | 59 ++------- .../ratstack-scorecard/journeys/manifest.json | 24 ++++ .../ratstack-scorecard/src/harness/decode.ts | 3 + .../src/harness/instrument.ts | 17 +-- .../src/harness/scorecard-codec.ts | 24 ++++ evals/ratstack-scorecard/src/journeys.ts | 112 ++++++++++++++---- evals/ratstack-scorecard/src/main.ts | 4 +- 9 files changed, 161 insertions(+), 94 deletions(-) diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index 8b1a6d3..2b5be58 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -50,7 +50,7 @@ export SCORECARD_PNPM_VERSION=${pkgs.pnpm_12.version} export DENO_NO_PACKAGE_JSON=1 exec deno run --no-config --allow-read --allow-write --allow-env --allow-sys=hostname \ - --allow-net=api.github.com --allow-run=git,${sandbox}/bin/sandbox \ + --allow-net=api.github.com --allow-run=git,${pkgs.deno}/bin/deno,${sandbox}/bin/sandbox \ ${self}/src/main.ts "$@" ''; }; diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/main/scorecard.json b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/main/scorecard.json index 8744633..de5200b 100644 --- a/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/main/scorecard.json +++ b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/main/scorecard.json @@ -19,7 +19,10 @@ "direction": "lower", "kind": "count", "runs": 3, - "family": "static" + "family": "static", + "ratstackSupport": { + "_tag": "Required" + } }, "definitionHash": "h-m23", "ratstack": { @@ -81,7 +84,10 @@ "direction": "lower", "kind": "count", "runs": 3, - "family": "static" + "family": "static", + "ratstackSupport": { + "_tag": "Required" + } }, "definitionHash": "h-m28", "ratstack": { diff --git a/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts b/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts index 0b9b8a4..a653693 100644 --- a/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts +++ b/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts @@ -1,61 +1,22 @@ import Ajv from 'ajv' -import { execFile } from 'node:child_process' -import { mkdtemp, readFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' import { describe, expect, test } from 'vitest' import schema from '../scorecard.schema.json' with { type: 'json' } +import { launcherRun } from './launcher-run.ts' -const instrument = join(import.meta.dirname, '..') -const fixtures = join(import.meta.dirname, '__fixtures__/aggregate') - -interface Exit { - readonly code: number - readonly stderr: string -} - -const aggregate = (args: readonly string[]): Promise => { - const { promise, resolve } = Promise.withResolvers() - execFile( - 'deno', - ['run', '--no-config', '--allow-read', '--allow-write', '--allow-env', 'src/main.ts', 'aggregate', ...args], - { cwd: instrument, env: { ...process.env, DENO_NO_PACKAGE_JSON: '1' } }, - (error, _stdout, stderr) => resolve({ code: error === null ? 0 : Number(error.code), stderr }), - ) - return promise -} +const validate = new Ajv({ allErrors: true, strict: true }).compile(schema) describe('aggregate through the real CLI (J3)', () => { test('a row beaten on main and tied on the PR fails the job, names the row, and still writes a valid scorecard', async () => { - const out = await mkdtemp(join(tmpdir(), 'aggregate-')) - const exit = await aggregate([ - '--families', - join(fixtures, 'families'), - '--main', - join(fixtures, 'main/scorecard.json'), - '--out', - join(out, 'scorecard.json'), - '--summary', - join(out, 'summary.md'), - ]) - const written = JSON.parse(await readFile(join(out, 'scorecard.json'), 'utf8')) - const validate = new Ajv({ allErrors: true, strict: true }).compile(schema) - expect(exit.code).toBe(1) - expect(exit.stderr).toContain('M23') - expect(exit.stderr).not.toContain('M28') - expect(validate(written)).toBe(true) + const run = await launcherRun('aggregate-regressed') + expect(run.code).toBe(1) + expect(run.stderr).toContain('M23') + expect(run.stderr).not.toContain('M28') + expect(validate(JSON.parse(run.files['scorecard.json'] ?? 'null'))).toBe(true) }) test('without a main artifact the same families are a passing first baseline', async () => { - const out = await mkdtemp(join(tmpdir(), 'aggregate-')) - const exit = await aggregate([ - '--families', - join(fixtures, 'families'), - '--out', - join(out, 'scorecard.json'), - '--summary', - join(out, 'summary.md'), - ]) - expect(exit.code).toBe(0) + const run = await launcherRun('aggregate-first-baseline') + expect(run.code).toBe(0) + expect(validate(JSON.parse(run.files['scorecard.json'] ?? 'null'))).toBe(true) }) }) diff --git a/evals/ratstack-scorecard/journeys/manifest.json b/evals/ratstack-scorecard/journeys/manifest.json index 5f8ed77..e8f13d2 100644 --- a/evals/ratstack-scorecard/journeys/manifest.json +++ b/evals/ratstack-scorecard/journeys/manifest.json @@ -29,6 +29,30 @@ "pnpm-lock.yaml", "flake.lock" ] + }, + { + "id": "aggregate-regressed", + "produce": { + "kind": "aggregate", + "families": "journeys/__fixtures__/aggregate/families", + "main": "journeys/__fixtures__/aggregate/main/scorecard.json" + }, + "inputs": [ + "journeys/__fixtures__/aggregate", + "src" + ] + }, + { + "id": "aggregate-first-baseline", + "produce": { + "kind": "aggregate", + "families": "journeys/__fixtures__/aggregate/families", + "main": null + }, + "inputs": [ + "journeys/__fixtures__/aggregate", + "src" + ] } ] } diff --git a/evals/ratstack-scorecard/src/harness/decode.ts b/evals/ratstack-scorecard/src/harness/decode.ts index 01a2ae8..7a277e6 100644 --- a/evals/ratstack-scorecard/src/harness/decode.ts +++ b/evals/ratstack-scorecard/src/harness/decode.ts @@ -53,6 +53,9 @@ export const struct = (fields: F): Decoder> => (val export const optional = (decode: Decoder): Decoder => (value, path) => value === undefined ? undefined : decode(value, path) +export const nullable = (decode: Decoder): Decoder => (value, path) => + value === null ? null : decode(value, path) + export const union = (...options: readonly Decoder[]): Decoder => (value, path) => { const errors: string[] = [] for (const option of options) { diff --git a/evals/ratstack-scorecard/src/harness/instrument.ts b/evals/ratstack-scorecard/src/harness/instrument.ts index 04d9aca..5e311cf 100644 --- a/evals/ratstack-scorecard/src/harness/instrument.ts +++ b/evals/ratstack-scorecard/src/harness/instrument.ts @@ -91,15 +91,6 @@ export const provenanceFor = ( detail, }) -const definitionInputs = (family: Family): readonly string[] => [ - `src/families/${family}.ts`, - 'src/sides', - 'src/tools', - 'src/harness', - 'flake.lock', - 'pnpm-lock.yaml', -] - const sha256 = async (text: string): Promise => [...new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(text)))] .map((byte) => byte.toString(16).padStart(2, '0')) @@ -109,15 +100,11 @@ export const definitionHashesFor = async ( instrument: Instrument, family: Family, ): Promise => { - const trees = await Promise.all( - definitionInputs(family).map(async (path) => - `${path}=${await gitText(instrument.checkout, ['rev-parse', `HEAD:evals/ratstack-scorecard/${path}`])}` - ), - ) + const measurement = await Deno.readTextFile(join(instrument.dir, 'src/families', `${family}.ts`)) return await Promise.all( rowDefinitions .filter((row) => row.family === family) - .map(async (row) => ({ id: row.id, hash: await sha256(JSON.stringify({ row, trees })) })), + .map(async (row) => ({ id: row.id, hash: await sha256(JSON.stringify({ row, measurement })) })), ) } diff --git a/evals/ratstack-scorecard/src/harness/scorecard-codec.ts b/evals/ratstack-scorecard/src/harness/scorecard-codec.ts index 740fcd7..01bcd3f 100644 --- a/evals/ratstack-scorecard/src/harness/scorecard-codec.ts +++ b/evals/ratstack-scorecard/src/harness/scorecard-codec.ts @@ -5,12 +5,16 @@ import type { Flag, MeasuredCell, Ratchet, + RatstackSupport, Row, RowDefinition, RowOutcome, ScorecardDocument, + Side, + SideCell, Verdict, } from '../model/cell.ts' +import type { CellsByRow } from '../model/scorecard-document.ts' import { array, type Decoder, @@ -92,6 +96,10 @@ const rowDefinition: Decoder = struct({ kind: oneOf(['count', 'measurement']), runs: number, family: oneOf(['static', 'cold-path', 'gate-mutation', 'running-stack', 'agent-surfaces', 'networked']), + ratstackSupport: union( + struct({ _tag: literal('Required') }), + struct({ _tag: literal('MayBeUnsupported'), bar: number }), + ), }) const verdict: Decoder = union( @@ -144,3 +152,19 @@ export const familyResult: Decoder = struct({ flags: array(struct({ id: string, flag })), definitionHashes: array(struct({ id: string, hash: string })), }) + +export class DuplicateCell extends Error { + constructor(readonly id: string, readonly side: Side) { + super(`two family results carry a ${side} cell for ${id}; each (row, side) is measured once`) + } +} + +export const cellsByRow = (cells: readonly SideCell[]): CellsByRow => { + const keyed: Record>> = {} + for (const { id, side, measured } of cells) { + const row = keyed[id] ?? {} + if (row[side] !== undefined) throw new DuplicateCell(id, side) + keyed[id] = { ...row, [side]: measured } + } + return keyed +} diff --git a/evals/ratstack-scorecard/src/journeys.ts b/evals/ratstack-scorecard/src/journeys.ts index 475e108..9813b64 100644 --- a/evals/ratstack-scorecard/src/journeys.ts +++ b/evals/ratstack-scorecard/src/journeys.ts @@ -1,6 +1,6 @@ import { join, relative } from 'node:path' import { measureStatic } from './families/static.ts' -import { arrayAt, at, stringAt } from './harness/decode.ts' +import { array, decodeJson, type Decoder, literal, nullable, string, struct, union } from './harness/decode.ts' import { git, type Instrument, walkFiles } from './harness/instrument.ts' import { runSandboxed } from './harness/sandbox.ts' @@ -13,7 +13,7 @@ interface MeasureStatic { interface JourneyEntry { readonly id: string - readonly produce: MeasureStatic + readonly produce: MeasureStatic | Aggregate readonly inputs: readonly string[] } @@ -29,20 +29,21 @@ export interface LauncherRecord { readonly wallMs: number } -const journeyEntry = (value: unknown): JourneyEntry => { - const produce = at(value, 'produce') - const failingTool = at(produce, 'failingTool') - return { - id: stringAt(value, ['id'], 'journey'), - produce: { - kind: 'measure-static', - ratstackRepo: stringAt(produce, ['ratstackRepo'], 'journey produce'), - starterRepo: stringAt(produce, ['starterRepo'], 'journey produce'), - failingTool: failingTool === null ? null : stringAt(produce, ['failingTool'], 'journey produce'), - }, - inputs: arrayAt(value, ['inputs'], 'journey').map((input) => stringAt({ input }, ['input'], 'journey input')), - } -} +const journeyEntry: Decoder = struct({ + id: string, + produce: union( + struct({ + kind: literal('measure-static'), + ratstackRepo: string, + starterRepo: string, + failingTool: nullable(string), + }), + struct({ kind: literal('aggregate'), families: string, main: nullable(string) }), + ), + inputs: array(string), +}) + +const manifestOf = struct({ journeys: array(journeyEntry) }) const encoder = new TextEncoder() @@ -91,17 +92,69 @@ const overlayWithFailingTool = async (instrument: Instrument, work: string, tool return dir } -const produce = async (instrument: Instrument, root: string, entry: JourneyEntry): Promise => { +interface Aggregate { + readonly kind: 'aggregate' + readonly families: string + readonly main: string | null +} + +const produceAggregate = async (root: string, entry: JourneyEntry, spec: Aggregate): Promise => { + const out = await Deno.makeTempDir({ prefix: `journey-${entry.id}-` }) + const started = performance.now() + try { + const argv = [ + 'aggregate', + '--families', + join(root, spec.families), + ...(spec.main === null ? [] : ['--main', join(root, spec.main)]), + '--out', + join(out, 'scorecard.json'), + '--summary', + join(out, 'summary.md'), + ] + const run = await new Deno.Command(Deno.execPath(), { + args: ['run', '--no-config', '--allow-read', '--allow-write', '--allow-env', join(root, 'src/main.ts'), ...argv], + env: { DENO_NO_PACKAGE_JSON: '1' }, + stdout: 'piped', + stderr: 'piped', + }).output() + const files: Record = {} + for (const name of ['scorecard.json', 'summary.md']) { + const text = await Deno.readTextFile(join(out, name)).catch(() => undefined) + if (text !== undefined) files[name] = text + } + return { + id: entry.id, + inputHash: '', + argv: ['scorecard', ...argv.map((arg) => arg.startsWith(out) ? arg.slice(out.length + 1) : arg)], + code: run.code, + stdout: new TextDecoder().decode(run.stdout), + stderr: new TextDecoder().decode(run.stderr), + files, + egressLog: null, + wallMs: Math.round(performance.now() - started), + } + } finally { + await Deno.remove(out, { recursive: true }) + } +} + +const produceStatic = async ( + instrument: Instrument, + root: string, + entry: JourneyEntry, + spec: MeasureStatic, +): Promise => { const work = await Deno.makeTempDir({ prefix: `journey-${entry.id}-` }) const started = performance.now() try { - const ratstackSrc = await gitRepoFrom(join(root, entry.produce.ratstackRepo), join(work, 'sources/ratstack')) - const checkout = await gitRepoFrom(join(root, entry.produce.starterRepo), join(work, 'sources/starter-checkout')) + const ratstackSrc = await gitRepoFrom(join(root, spec.ratstackRepo), join(work, 'sources/ratstack')) + const checkout = await gitRepoFrom(join(root, spec.starterRepo), join(work, 'sources/starter-checkout')) const project = join(work, 'project') await Deno.mkdir(project) - const dir = entry.produce.failingTool === null + const dir = spec.failingTool === null ? instrument.dir - : await overlayWithFailingTool(instrument, work, entry.produce.failingTool) + : await overlayWithFailingTool(instrument, work, spec.failingTool) const subject: Instrument = { ...instrument, dir, @@ -112,7 +165,7 @@ const produce = async (instrument: Instrument, root: string, entry: JourneyEntry const result = await measureStatic(subject, project, ['ratstack', 'starter']) return { id: entry.id, - inputHash: await inputHashOf(root, entry.inputs, instrument.launcher.executable), + inputHash: '', argv: ['scorecard', 'measure', '--family', 'static'], code: 0, stdout: '', @@ -126,10 +179,19 @@ const produce = async (instrument: Instrument, root: string, entry: JourneyEntry } } +const produce = async (instrument: Instrument, root: string, entry: JourneyEntry): Promise => { + const record = entry.produce.kind === 'aggregate' + ? await produceAggregate(root, entry, entry.produce) + : await produceStatic(instrument, root, entry, entry.produce) + return { ...record, inputHash: await inputHashOf(root, entry.inputs, instrument.launcher.executable) } +} + export const runJourneys = async (instrument: Instrument, root: string): Promise => { - const manifest = arrayAt(JSON.parse(await Deno.readTextFile(join(root, 'journeys/manifest.json'))), [ - 'journeys', - ], 'journeys/manifest.json').map(journeyEntry) + const { journeys: manifest } = decodeJson( + manifestOf, + await Deno.readTextFile(join(root, 'journeys/manifest.json')), + 'journeys/manifest.json', + ) const records = join(root, 'journeys/__records__') await Deno.remove(records, { recursive: true }).catch(() => undefined) await Deno.mkdir(records, { recursive: true }) diff --git a/evals/ratstack-scorecard/src/main.ts b/evals/ratstack-scorecard/src/main.ts index 862af35..fc0a3b5 100644 --- a/evals/ratstack-scorecard/src/main.ts +++ b/evals/ratstack-scorecard/src/main.ts @@ -4,7 +4,7 @@ import { checkImports } from './check-imports.ts' import { measureStatic } from './families/static.ts' import { array, decodeJson, number, string, struct } from './harness/decode.ts' import { git, type Instrument, loadInstrument } from './harness/instrument.ts' -import { familyResult, scorecardDocument } from './harness/scorecard-codec.ts' +import { cellsByRow, familyResult, scorecardDocument } from './harness/scorecard-codec.ts' import { runJourneys } from './journeys.ts' import { familyTimeoutMinutes, rowDefinitions } from './metrics/registry.ts' import { ratstackCacheKey } from './model/cache-key.ts' @@ -142,7 +142,7 @@ const aggregate = async (args: readonly string[]): Promise => { definition, hash: hashes.find((hash) => hash.id === definition.id)?.hash ?? `no ${definition.family} result`, })), - cells, + cells: cellsByRow(cells), flags: results.flatMap((result) => result.flags), provenance: { commit: starter?.commit ?? Deno.env.get('GITHUB_SHA') ?? 'unknown', From 59a5058f83ce4902054536de502c7d01a49ce0c1 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Oct 2026 05:49:05 +0000 Subject: [PATCH 3/6] fix(repo): judge in a sandboxed Effect decide step behind a zero-dependency driver Review fixes #2, #5, #6, #8 and the rest of #9 under Kiro's rulings. The host driver only starts launcher invocations; decide/ (Effect Schema, effect/http) decodes, judges, ratchets and calls GitHub with a 15 s timeout, three transient retries and a typed GithubApiError. The rat-stack cache never saves or restores an instrument error. pin check decodes main's SHA as 40 hex; the bot-owned pin branch is pushed with a lease and a credential helper. Grading jobs build the instrument from the base tree, re-baselined rows list both hashes, and an informational job previews a PR's own instrument --- .github/workflows/scorecard.yml | 140 ++++++++-- evals/ratstack-scorecard/README.md | 25 +- evals/ratstack-scorecard/decide/main.ts | 214 +++++++++++++++ evals/ratstack-scorecard/decide/schema.ts | 146 ++++++++++ evals/ratstack-scorecard/flake.nix | 4 +- .../journeys/aggregate.journey.test.ts | 2 +- .../ratstack-scorecard/journeys/manifest.json | 6 +- evals/ratstack-scorecard/package.json | 7 +- evals/ratstack-scorecard/pnpm-lock.yaml | 249 +++++++++++++++++- .../ratstack-scorecard/scorecard.schema.json | 13 +- .../src/harness/scorecard-codec.ts | 170 ------------ evals/ratstack-scorecard/src/journeys.ts | 26 +- evals/ratstack-scorecard/src/main.ts | 134 ++++------ evals/ratstack-scorecard/src/model/cell.ts | 2 +- ...ompare-with-main.workflow.property.test.ts | 4 +- .../src/model/compare-with-main.workflow.ts | 5 +- .../src/model/summary-table.ts | 26 +- evals/ratstack-scorecard/tsconfig.json | 18 ++ 18 files changed, 881 insertions(+), 310 deletions(-) create mode 100644 evals/ratstack-scorecard/decide/main.ts create mode 100644 evals/ratstack-scorecard/decide/schema.ts delete mode 100644 evals/ratstack-scorecard/src/harness/scorecard-codec.ts create mode 100644 evals/ratstack-scorecard/tsconfig.json diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 815f23e..b33e449 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -18,7 +18,7 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: - SCORECARD_FLAKE: ./evals/ratstack-scorecard + INSTRUMENT_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }} jobs: plan: @@ -28,11 +28,15 @@ jobs: matrix: ${{ steps.plan.outputs.matrix }} steps: - uses: actions/checkout@v7 + with: + ref: ${{ env.INSTRUMENT_REF }} + path: instrument - uses: cachix/install-nix-action@v31 - id: plan name: Families and rat-stack cache keys + working-directory: instrument run: | - scorecard="$(nix build --no-link --print-out-paths "$SCORECARD_FLAKE#scorecard")" + scorecard="$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#scorecard)" matrix="$("$scorecard/bin/scorecard" plan)" echo "$matrix" | jq . echo "matrix=$matrix" >> "$GITHUB_OUTPUT" @@ -47,24 +51,46 @@ jobs: timeout-minutes: ${{ matrix.timeoutMinutes }} steps: - uses: actions/checkout@v7 + - uses: actions/checkout@v7 + with: + ref: ${{ env.INSTRUMENT_REF }} + path: .scorecard-instrument - uses: cachix/install-nix-action@v31 - name: Allow the launcher's unprivileged user namespaces run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - - name: Build the instrument - run: echo "SCORECARD=$(nix build --no-link --print-out-paths "$SCORECARD_FLAKE#scorecard")/bin/scorecard" >> "$GITHUB_ENV" + - name: Build main's instrument + run: echo "SCORECARD=$(nix build --no-link --print-out-paths ./.scorecard-instrument/evals/ratstack-scorecard#scorecard)/bin/scorecard" >> "$GITHUB_ENV" - id: ratstack-cache name: Restore the rat-stack side uses: actions/cache/restore@v6 with: path: scorecard-out/ratstack-${{ matrix.family }}.json key: ${{ matrix.cacheKey }} + - id: cached + name: Refuse a cached rat-stack side that holds an instrument error + if: steps.ratstack-cache.outputs.cache-hit == 'true' + run: | + if "$SCORECARD" cache-check --file "scorecard-out/ratstack-${{ matrix.family }}.json"; then + echo "usable=true" >> "$GITHUB_OUTPUT" + else + rm -f "scorecard-out/ratstack-${{ matrix.family }}.json" + echo "::warning::cached rat-stack ${{ matrix.family }} side refused; measuring it again" + fi - name: Measure the rat-stack side - if: steps.ratstack-cache.outputs.cache-hit != 'true' + if: steps.cached.outputs.usable != 'true' run: | mkdir -p scorecard-out - "$SCORECARD" measure --family "${{ matrix.family }}" --side ratstack --out "scorecard-out/ratstack-${{ matrix.family }}.json" + "$SCORECARD" measure --family "${{ matrix.family }}" --side ratstack --checkout "$GITHUB_WORKSPACE" \ + --out "scorecard-out/ratstack-${{ matrix.family }}.json" + - id: savable + name: Check the fresh rat-stack side before caching it + if: steps.cached.outputs.usable != 'true' && github.event_name != 'pull_request' + run: | + if "$SCORECARD" cache-check --file "scorecard-out/ratstack-${{ matrix.family }}.json"; then + echo "save=true" >> "$GITHUB_OUTPUT" + fi - name: Save the rat-stack side for later runs - if: steps.ratstack-cache.outputs.cache-hit != 'true' && github.event_name != 'pull_request' + if: steps.savable.outputs.save == 'true' uses: actions/cache/save@v6 with: path: scorecard-out/ratstack-${{ matrix.family }}.json @@ -72,7 +98,8 @@ jobs: - name: Measure the starter side run: | mkdir -p scorecard-out - "$SCORECARD" measure --family "${{ matrix.family }}" --side starter --out "scorecard-out/starter-${{ matrix.family }}.json" + "$SCORECARD" measure --family "${{ matrix.family }}" --side starter --checkout "$GITHUB_WORKSPACE" \ + --out "scorecard-out/starter-${{ matrix.family }}.json" - uses: actions/upload-artifact@v7 with: name: family-${{ matrix.family }} @@ -86,9 +113,15 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v7 + - uses: actions/checkout@v7 + with: + ref: ${{ env.INSTRUMENT_REF }} + path: .scorecard-instrument - uses: cachix/install-nix-action@v31 - - name: Build the instrument - run: echo "SCORECARD=$(nix build --no-link --print-out-paths "$SCORECARD_FLAKE#scorecard")/bin/scorecard" >> "$GITHUB_ENV" + - name: Allow the launcher's unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Build main's instrument + run: echo "SCORECARD=$(nix build --no-link --print-out-paths ./.scorecard-instrument/evals/ratstack-scorecard#scorecard)/bin/scorecard" >> "$GITHUB_ENV" - uses: actions/download-artifact@v8 with: pattern: family-* @@ -113,7 +146,7 @@ jobs: if [ -f main/scorecard.json ]; then main_args=(--main main/scorecard.json); fi status=0 "$SCORECARD" aggregate --families families "${main_args[@]}" --out scorecard.json --summary summary.md || status=$? - cat summary.md >> "$GITHUB_STEP_SUMMARY" + if [ -f summary.md ]; then cat summary.md >> "$GITHUB_STEP_SUMMARY"; fi exit "$status" - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} @@ -122,17 +155,73 @@ jobs: path: scorecard.json if-no-files-found: error + instrument-preview: + name: instrument preview (informational) + needs: aggregate + if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.aggregate.result != 'skipped' }} + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - id: changed + name: Does this PR edit the instrument + run: | + if git diff --quiet "${{ github.event.pull_request.base.sha }}" HEAD -- evals/ratstack-scorecard; then + echo "edits=false" >> "$GITHUB_OUTPUT" + echo "This PR leaves the instrument alone; nothing to preview." >> "$GITHUB_STEP_SUMMARY" + else + echo "edits=true" >> "$GITHUB_OUTPUT" + fi + - uses: cachix/install-nix-action@v31 + if: steps.changed.outputs.edits == 'true' + - name: Allow the launcher's unprivileged user namespaces + if: steps.changed.outputs.edits == 'true' + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - uses: actions/download-artifact@v8 + if: steps.changed.outputs.edits == 'true' + with: + name: scorecard + path: graded + - name: Measure with this PR's instrument and list what it would re-baseline + if: steps.changed.outputs.edits == 'true' + run: | + scorecard="$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#scorecard)/bin/scorecard" + mkdir -p preview + for family in $("$scorecard" plan | jq -r '.include[].family'); do + "$scorecard" measure --family "$family" --out "preview/$family.json" + done + status=0 + "$scorecard" aggregate --families preview --main graded/scorecard.json \ + --out preview-scorecard.json --summary preview-summary.md || status=$? + { + echo "## Instrument preview (does not gate)" + echo + echo "This PR's own instrument, compared with the scorecard main's instrument produced for it. Re-baselined rows are listed under definition drift." + echo + cat preview-summary.md + } >> "$GITHUB_STEP_SUMMARY" + exit "$status" + pin: if: github.event_name != 'pull_request' runs-on: ubuntu-latest timeout-minutes: 15 steps: - uses: actions/checkout@v7 + with: + persist-credentials: false - uses: cachix/install-nix-action@v31 + - name: Allow the launcher's unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Build the instrument - run: echo "SCORECARD=$(nix build --no-link --print-out-paths "$SCORECARD_FLAKE#scorecard")/bin/scorecard" >> "$GITHUB_ENV" + run: echo "SCORECARD=$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#scorecard)/bin/scorecard" >> "$GITHUB_ENV" - id: check name: Compare the pin with rat-stack main + env: + GITHUB_TOKEN: ${{ github.token }} run: | plan="$("$SCORECARD" pin check)" echo "$plan" | jq . @@ -148,22 +237,37 @@ jobs: private-key: ${{ secrets.SCORECARD_APP_PRIVATE_KEY }} permission-contents: write permission-pull-requests: write + # scorecard/pin-rat-stack is bot-owned: the job only ever replaces a tip the bot + # itself wrote, through a lease pinned to that tip, so a human commit there is + # never overwritten. The App token reaches git through a credential helper that + # reads it from the environment, never through argv or a URL. - name: Open or update the pin-bump PR if: steps.check.outputs.tag == 'PinMoved' env: GH_TOKEN: ${{ steps.token.outputs.token }} TO: ${{ steps.check.outputs.to }} + BRANCH: scorecard/pin-rat-stack + BOT_EMAIL: scorecard-pin[bot]@users.noreply.github.com run: | + git config credential.helper "!f() { echo username=x-access-token; echo \"password=\${GH_TOKEN}\"; }; f" nar_hash="$(nix flake prefetch --json "github:joelhooks/rat-stack/$TO" | jq -r .hash)" "$SCORECARD" pin write --commit "$TO" --nar-hash "$nar_hash" - branch="scorecard/pin-rat-stack" + lease="$(git ls-remote origin "refs/heads/$BRANCH" | cut -f1)" + if [ -n "$lease" ]; then + git fetch --quiet origin "$lease" + author="$(git log -1 --format=%ae "$lease")" + if [ "$author" != "$BOT_EMAIL" ]; then + echo "::error::$BRANCH tip $lease was written by $author, not the pin bot; refusing to replace it" + exit 1 + fi + fi git config user.name "scorecard-pin[bot]" - git config user.email "scorecard-pin[bot]@users.noreply.github.com" - git switch -c "$branch" + git config user.email "$BOT_EMAIL" + git switch -c "$BRANCH" git commit -m "chore(deps): pin rat-stack to ${TO:0:7} for the scorecard" -- evals/ratstack-scorecard/ratstack.pin.json - git push --force "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$branch" - if [ -z "$(gh pr list --head "$branch" --json number --jq '.[0].number')" ]; then - gh pr create --base main --head "$branch" \ + git push --force-with-lease="refs/heads/$BRANCH:$lease" origin "HEAD:refs/heads/$BRANCH" + if [ -z "$(gh pr list --head "$BRANCH" --json number --jq '.[0].number')" ]; then + gh pr create --base main --head "$BRANCH" \ --title "chore(deps): pin rat-stack to ${TO:0:7} for the scorecard" \ --body "rat-stack \`main\` moved to \`$TO\`. This PR moves \`evals/ratstack-scorecard/ratstack.pin.json\` only; its scorecard run measures the new rat-stack. Kiro merges." fi diff --git a/evals/ratstack-scorecard/README.md b/evals/ratstack-scorecard/README.md index 47aeb61..8107402 100644 --- a/evals/ratstack-scorecard/README.md +++ b/evals/ratstack-scorecard/README.md @@ -28,23 +28,32 @@ scorecard=$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#sco $scorecard/bin/scorecard measure --family static --out static.json ``` -The instrument's own tests run inside the launcher against the same offline store: +The instrument's own checks run with one command: `scorecard check` (also `pnpm scorecard:check` at the repo root, part of `check:ci`). It first refuses any third-party import in the host driver's graph and any import of the journey fixture from `src/`, then runs `scorecard journeys`. + +The program is split in two: + +- **The host driver** (`src/main.ts` and what it imports) uses only Deno APIs, `node:` builtins and its own files, so `DENO_NO_PACKAGE_JSON=1 deno check src/` type-checks it without third-party code. It starts launcher invocations, mounts their inputs read-only, collects output files and exit codes, and writes the pin file. It decodes and judges nothing. Its only network access is through the launcher. +- **The decide step** (`decide/`, Effect 4 with Schema and `effect/http`) runs as its own launcher invocation: `aggregate` (decode the family results and `main`'s scorecard, judge, ratchet, render), `cache-check`, `latest-main-run` and `pin-check`. Only the last two may reach `api.github.com`; GitHub calls time out after 15 s, retry transient failures three times, and fail as a typed `GithubApiError`. Pass or fail is the step's exit code. `pnpm exec tsc` type-checks `decide/` and `journeys/` inside the launcher. + +Journeys run in two phases, because the launcher cannot nest. `scorecard journeys` produces every journey declared in `journeys/manifest.json` through the real launcher work and records it to `journeys/__records__/.json`, then runs the one vitest project inside the launcher. A test is a journey because it imports `journeys/launcher-run.ts`; a missing or stale record fails it red. ```sh -cd evals/ratstack-scorecard -nix develop --command sh -c 'SANDBOX_PROJECT=$PWD sandbox --pnpm-store "$SANDBOX_PNPM_STORE" -- pnpm install --frozen-lockfile' -nix develop --command sh -c 'SANDBOX_PROJECT=$PWD sandbox -- pnpm vitest run' +scorecard=$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#scorecard) +$scorecard/bin/scorecard check ``` -`src/main.ts` and everything it imports use only Deno APIs, `node:` builtins and each other, so `DENO_NO_PACKAGE_JSON=1 deno check src/` type-checks the orchestrator and the decision core without third-party code. The Node scripts in `src/tools/` are the only code that loads npm packages, and they only ever run inside the launcher. +The Node scripts in `src/tools/` and everything in `decide/` load npm packages, and they only ever run inside the launcher. ## In CI `.github/workflows/scorecard.yml` runs on every PR to `main`, every push to `main`, daily, and on demand. All jobs run on GitHub-hosted runners (`ubuntu-latest`): the self-hosted fleet excludes public repositories by design. +On a pull request every grading job builds the instrument from the base commit's tree, and only the measured starter comes from the PR, so a PR that edits `evals/ratstack-scorecard/**` cannot grade itself. A definition hash covers only a row's registry entry and its family's measurement code; harness, sides, tools, lockfiles and nixpkgs move the rat-stack cache key instead. + - **plan** prints the matrix: one entry per built family, with its timeout and the rat-stack cache key (rat-stack commit, instrument tree hash, nixpkgs rev). -- **measure** runs one family per job. The rat-stack side is restored from the cache when the key matches and measured otherwise; only `main`, the schedule and manual runs save it. The starter side is measured every run. Each job uploads `family-`. -- **aggregate** joins the families, compares them with the latest successful `main` run's `scorecard` artifact (`scorecard latest-main-run`; none yet means a first baseline), writes the Markdown table to the job summary, uploads `scorecard.json`, and fails when the ratchet fails. -- **pin** (not on PRs) compares `ratstack.pin.json` with rat-stack `main`. When rat-stack has moved it opens or updates the `scorecard/pin-rat-stack` PR through the pin-bump GitHub App (`vars.SCORECARD_APP_ID`, `secrets.SCORECARD_APP_PRIVATE_KEY`; contents and pull requests write only). That PR changes only the pin; Kiro merges it. +- **measure** runs one family per job. The rat-stack side is restored from the cache when the key matches and `scorecard cache-check` accepts it; an entry holding a rat-stack instrument error is refused and measured again. Only `main`, the schedule and manual runs save it, and only when the fresh result holds no instrument error. The starter side is measured every run. Each job uploads `family-`. +- **aggregate** joins the families, compares them with the latest successful `main` run's `scorecard` artifact (`scorecard latest-main-run`; none yet means a first baseline), writes the Markdown table to the job summary, uploads `scorecard.json`, and fails when the ratchet fails. Every re-baselined row is listed under definition drift with both hashes. +- **instrument preview** (PRs that edit the instrument, never gates) builds the PR's own instrument, measures every family and lists the rows it would re-baseline against the graded scorecard. +- **pin** (not on PRs) compares `ratstack.pin.json` with rat-stack `main`, decoded as a 40-hex SHA. When rat-stack has moved it opens or updates the `scorecard/pin-rat-stack` PR through the pin-bump GitHub App (`vars.SCORECARD_APP_ID`, `secrets.SCORECARD_APP_PRIVATE_KEY`; contents and pull requests write only). The branch is bot-owned: the job refuses to replace a tip another author wrote and pushes with a lease pinned to the tip it read. The token reaches git through a credential helper that reads it from the environment. That PR changes only the pin; Kiro merges it. `actionlint` is in the dev shell: `nix develop ./evals/ratstack-scorecard --command actionlint .github/workflows/scorecard.yml`. diff --git a/evals/ratstack-scorecard/decide/main.ts b/evals/ratstack-scorecard/decide/main.ts new file mode 100644 index 0000000..3d424b9 --- /dev/null +++ b/evals/ratstack-scorecard/decide/main.ts @@ -0,0 +1,214 @@ +import { Cause, Duration, Effect, Schedule, Schema } from 'effect' +import { FetchHttpClient, HttpClient, HttpClientRequest } from 'effect/http' +import { readdir, readFile, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { parseArgs } from 'node:util' +import { rowDefinitions } from '../src/metrics/registry.ts' +import type { Family, FamilyResult, MainBaseline, MeasuredCell, Side, SideCell } from '../src/model/cell.ts' +import { planPinBump } from '../src/model/plan-pin-bump.workflow.ts' +import { assembleScorecard, type CellsByRow } from '../src/model/scorecard-document.ts' +import { renderSummary } from '../src/model/summary-table.ts' +import { FamilyResultSchema, GitRefSchema, ScorecardDocumentSchema, WorkflowRunsSchema } from './schema.ts' + +class MalformedInput extends Schema.TaggedError()('MalformedInput', { + what: Schema.String, + issue: Schema.String, +}) {} + +class DuplicateCell extends Schema.TaggedError()('DuplicateCell', { + id: Schema.String, + side: Schema.String, +}) {} + +class GithubApiError extends Schema.TaggedError()('GithubApiError', { + request: Schema.String, + cause: Schema.String, +}) {} + +class RatchetFailed extends Schema.TaggedError()('RatchetFailed', { + rows: Schema.Array(Schema.String), +}) {} + +class CacheUnusable extends Schema.TaggedError()('CacheUnusable', { + reason: Schema.String, +}) {} + +const readText = (path: string) => + Effect.tryPromise({ + try: () => readFile(path, 'utf8'), + catch: (error) => new MalformedInput({ what: path, issue: String(error) }), + }) + +const writeText = (path: string, text: string) => Effect.promise(() => writeFile(path, text)) + +const decodeFile = (schema: Schema.Codec, path: string) => + readText(path).pipe( + Effect.flatMap((text) => Schema.decodeUnknownEffect(Schema.fromJsonString(schema))(text)), + Effect.mapError((error) => new MalformedInput({ what: path, issue: String(error) })), + ) + +type Keyed = Readonly>>>> + +const noCells: Keyed = {} + +const keyCells = (cells: readonly SideCell[]): Effect.Effect => + Effect.reduce(cells, () => noCells, (keyed: Keyed, cell: SideCell) => + keyed[cell.id]?.[cell.side] === undefined + ? Effect.succeed({ ...keyed, [cell.id]: { ...keyed[cell.id], [cell.side]: cell.measured } }) + : Effect.fail(new DuplicateCell({ id: cell.id, side: cell.side }))) + +const familyResults = (dir: string) => + Effect.promise(() => readdir(dir)).pipe( + Effect.flatMap((names) => + Effect.forEach( + names.filter((name) => name.endsWith('.json')).sort(), + (name) => decodeFile(FamilyResultSchema, join(dir, name)), + ) + ), + ) + +const mainBaseline = (path: string | undefined): Effect.Effect => + path === undefined + ? Effect.succeed({ _tag: 'Missing' }) + : decodeFile(ScorecardDocumentSchema, path).pipe( + Effect.map((doc) => ({ _tag: 'Found', commit: doc.provenance.commit, rows: doc.rows })), + ) + +const aggregate = (args: readonly string[]) => + Effect.gen(function*() { + const { values } = parseArgs({ + args: [...args], + options: { + families: { type: 'string' }, + implemented: { type: 'string' }, + main: { type: 'string' }, + commit: { type: 'string' }, + out: { type: 'string' }, + summary: { type: 'string' }, + }, + strict: true, + }) + const results: readonly FamilyResult[] = yield* familyResults(yield* required(values.families, '--families')) + const implemented: readonly string[] = (yield* required(values.implemented, '--implemented')).split(',') + const cells = results.flatMap((result) => result.cells) + const hashes = results.flatMap((result) => result.definitionHashes) + const provenanceOf = (side: Side) => cells.find((cell) => cell.side === side)?.measured.provenance + const any = provenanceOf('starter') ?? provenanceOf('ratstack') + if (any === undefined) return yield* new MalformedInput({ what: values.families ?? '', issue: 'no cells' }) + const doc = assembleScorecard({ + rows: rowDefinitions + .filter((definition) => implemented.includes(definition.family)) + .map((definition) => ({ + definition, + hash: hashes.find((hash) => hash.id === definition.id)?.hash ?? `no ${definition.family} result`, + })), + cells: yield* keyCells(cells), + flags: results.flatMap((result) => result.flags), + provenance: { + commit: provenanceOf('starter')?.commit ?? (yield* required(values.commit, '--commit')), + ratstackCommit: provenanceOf('ratstack')?.commit ?? 'unknown', + instrumentHash: any.instrumentHash, + nixpkgsRev: any.nixpkgsRev, + runner: any.runner, + generatedAt: new Date().toISOString(), + }, + main: yield* mainBaseline(values.main), + }) + yield* writeText(yield* required(values.out, '--out'), `${JSON.stringify(doc, null, 2)}\n`) + yield* writeText(yield* required(values.summary, '--summary'), renderSummary(doc)) + if (doc.ratchet.failures.length > 0) { + return yield* new RatchetFailed({ rows: doc.ratchet.failures.map((failure) => failure.id) }) + } + }) + +const cacheCheck = (args: readonly string[]) => + Effect.gen(function*() { + const { values } = parseArgs({ args: [...args], options: { file: { type: 'string' } }, strict: true }) + const result = yield* decodeFile(FamilyResultSchema, yield* required(values.file, '--file')).pipe( + Effect.mapError((error) => new CacheUnusable({ reason: `${error.what}: ${error.issue}` })), + ) + const broken = result.cells.filter((cell) => + cell.side === 'ratstack' && cell.measured.cell._tag === 'InstrumentError' + ) + if (broken.length > 0) { + return yield* new CacheUnusable({ + reason: `rat-stack instrument errors on ${broken.map((c) => c.id).join(', ')}`, + }) + } + }) + +const github = (path: string) => + Effect.gen(function*() { + const client = (yield* HttpClient.HttpClient).pipe( + HttpClient.filterStatusOk, + HttpClient.retryTransient({ schedule: Schedule.exponential(Duration.seconds(1)), times: 3 }), + ) + const token = process.env['GITHUB_TOKEN'] + const request = HttpClientRequest.get(`https://api.github.com${path}`).pipe( + HttpClientRequest.acceptJson, + (r) => token === undefined || token === '' ? r : HttpClientRequest.bearerToken(r, token), + ) + return yield* client.execute(request).pipe( + Effect.flatMap((response) => response.json), + Effect.timeout(Duration.seconds(15)), + ) + }).pipe( + Effect.mapError((error) => new GithubApiError({ request: `GET ${path}`, cause: String(error) })), + Effect.provide(FetchHttpClient.layer), + ) + +const decodeBody = (schema: Schema.Codec, what: string) => (body: unknown) => + Schema.decodeUnknownEffect(schema)(body).pipe( + Effect.mapError((error) => new GithubApiError({ request: what, cause: String(error) })), + ) + +const latestMainRun = (args: readonly string[]) => + Effect.gen(function*() { + const { values } = parseArgs({ args: [...args], options: { repository: { type: 'string' } }, strict: true }) + const repository = yield* required(values.repository, '--repository') + const path = + `/repos/${repository}/actions/workflows/scorecard.yml/runs?branch=main&event=push&status=success&per_page=1` + const runs = yield* github(path).pipe(Effect.flatMap(decodeBody(WorkflowRunsSchema, `GET ${path}`))) + yield* Effect.sync(() => process.stdout.write(`${runs.workflow_runs.map((run) => run.id).join('')}\n`)) + }) + +const pinCheck = (args: readonly string[]) => + Effect.gen(function*() { + const { values } = parseArgs({ + args: [...args], + options: { owner: { type: 'string' }, repo: { type: 'string' }, pinned: { type: 'string' } }, + strict: true, + }) + const path = `/repos/${yield* required(values.owner, '--owner')}/${yield* required( + values.repo, + '--repo', + )}/git/ref/heads/main` + const ref = yield* github(path).pipe(Effect.flatMap(decodeBody(GitRefSchema, `GET ${path}`))) + const plan = planPinBump({ pinned: yield* required(values.pinned, '--pinned'), remoteHead: ref.object.sha }) + yield* Effect.sync(() => process.stdout.write(`${JSON.stringify(plan)}\n`)) + }) + +const required = (value: string | undefined, flag: string) => + value === undefined + ? Effect.fail(new MalformedInput({ what: flag, issue: 'is required' })) + : Effect.succeed(value) + +const commands: Readonly Effect.Effect>> = { + 'aggregate': aggregate, + 'cache-check': cacheCheck, + 'latest-main-run': latestMainRun, + 'pin-check': pinCheck, +} + +const [command = '', ...rest] = process.argv.slice(2) +const run = commands[command] +if (run === undefined) { + process.stderr.write(`decide: unknown command ${command}; one of ${Object.keys(commands).join(', ')}\n`) + process.exit(2) +} +const exit = await Effect.runPromiseExit(run(rest)) +if (exit._tag === 'Failure') { + const error = Cause.squash(exit.cause) + process.stderr.write(`decide ${command}: ${String(error)} ${JSON.stringify(error)}\n`) + process.exit(1) +} diff --git a/evals/ratstack-scorecard/decide/schema.ts b/evals/ratstack-scorecard/decide/schema.ts new file mode 100644 index 0000000..8b9437c --- /dev/null +++ b/evals/ratstack-scorecard/decide/schema.ts @@ -0,0 +1,146 @@ +import { Schema } from 'effect' +import type { + Cell, + CellProvenance, + FamilyResult, + Flag, + MeasuredCell, + Ratchet, + Row, + RowDefinition, + RowOutcome, + ScorecardDocument, + Verdict, +} from '../src/model/cell.ts' + +const side = Schema.Literals(['ratstack', 'starter']) +const family = Schema.Literals(['static', 'cold-path', 'gate-mutation', 'running-stack', 'agent-surfaces', 'networked']) +const cellTag = Schema.Literals([ + 'Measured', + 'Absent', + 'NoDeployment', + 'Unsupported', + 'Unmeasurable', + 'NoSecret', + 'InstrumentError', +]) + +export const Sha = Schema.String.check(Schema.isPattern(/^[0-9a-f]{40}$/u)) + +const tagged = (tag: T, fields: F) => + Schema.Struct({ _tag: Schema.Literal(tag), ...fields }) + +export const CellSchema: Schema.Codec = Schema.Union([ + tagged('Measured', { runs: Schema.Array(Schema.Finite) }), + tagged('Absent', { reason: Schema.String }), + tagged('NoDeployment', { sha: Schema.String }), + tagged('Unsupported', { + citation: Schema.Struct({ + file: Schema.String, + lines: Schema.Tuple([Schema.Finite, Schema.Finite]), + text: Schema.String, + }), + check: Schema.Union([tagged('Verified', {}), tagged('Contradicted', { found: Schema.String })]), + }), + tagged('Unmeasurable', { error: Schema.String }), + tagged('NoSecret', { name: Schema.String }), + tagged('InstrumentError', { error: Schema.String }), +]) + +const CellProvenanceSchema: Schema.Codec = Schema.Struct({ + side, + commit: Schema.String, + instrumentHash: Schema.String, + nixpkgsRev: Schema.String, + runner: Schema.String, + measuredAt: Schema.String, + tools: Schema.Record(Schema.String, Schema.String), + liveCommit: Schema.optionalKey(Schema.String), + detail: Schema.optionalKey(Schema.Record(Schema.String, Schema.Union([Schema.Finite, Schema.String]))), +}) + +const MeasuredCellSchema: Schema.Codec = Schema.Struct({ + cell: CellSchema, + provenance: CellProvenanceSchema, +}) + +const FlagSchema: Schema.Codec = Schema.Union([ + tagged('LiveDiffersFromPin', { live: Schema.String, pin: Schema.String }), + tagged('ScannersDisagree', { primary: Schema.Finite, crossCheck: Schema.Finite }), +]) + +const RowDefinitionSchema: Schema.Codec = Schema.Struct({ + id: Schema.String, + metric: Schema.String, + bin: Schema.String, + label: Schema.String, + unit: Schema.String, + direction: Schema.Literals(['lower', 'higher']), + kind: Schema.Literals(['count', 'measurement']), + runs: Schema.Finite, + family, + ratstackSupport: Schema.Union([tagged('Required', {}), tagged('MayBeUnsupported', { bar: Schema.Finite })]), +}) + +const VerdictSchema: Schema.Codec = Schema.Union([ + tagged('Beaten', {}), + tagged('NotBeaten', {}), + tagged('Tie', {}), + tagged('InstrumentError', { error: Schema.String }), +]) + +const RowSchema: Schema.Codec = Schema.Struct({ + definition: RowDefinitionSchema, + definitionHash: Schema.String, + ratstack: MeasuredCellSchema, + starter: MeasuredCellSchema, + verdict: VerdictSchema, + flags: Schema.Array(FlagSchema), +}) + +const OutcomeSchema: Schema.Codec = Schema.Union([ + tagged('Held', { id: Schema.String }), + tagged('New', { id: Schema.String }), + tagged('ReBaselined', { id: Schema.String, mainHash: Schema.String, prHash: Schema.String }), + tagged('Neutral', { id: Schema.String, cause: cellTag }), + tagged('InstrumentError', { id: Schema.String, error: Schema.String }), + tagged('LostBeaten', { id: Schema.String, ratstack: cellTag, starter: cellTag }), + tagged('Regressed', { id: Schema.String, main: Schema.String, pr: Schema.String }), +]) + +const RatchetSchema: Schema.Codec = Schema.Union([ + tagged('FirstBaseline', { outcomes: Schema.Array(OutcomeSchema), failures: Schema.Array(OutcomeSchema) }), + tagged('Compared', { + mainCommit: Schema.String, + outcomes: Schema.Array(OutcomeSchema), + failures: Schema.Array(OutcomeSchema), + }), +]) + +export const ScorecardDocumentSchema: Schema.Codec = Schema.Struct({ + schemaVersion: Schema.Literal(1), + provenance: Schema.Struct({ + commit: Schema.String, + ratstackCommit: Schema.String, + instrumentHash: Schema.String, + nixpkgsRev: Schema.String, + runner: Schema.String, + generatedAt: Schema.String, + }), + rows: Schema.Array(RowSchema), + ratchet: RatchetSchema, +}) + +export const FamilyResultSchema: Schema.Codec = Schema.Struct({ + family, + wallMs: Schema.Finite, + cells: Schema.Array(Schema.Struct({ id: Schema.String, side, measured: MeasuredCellSchema })), + flags: Schema.Array(Schema.Struct({ id: Schema.String, flag: FlagSchema })), + definitionHashes: Schema.Array(Schema.Struct({ id: Schema.String, hash: Schema.String })), +}) + +export const WorkflowRunsSchema = Schema.Struct({ + workflow_runs: Schema.Array(Schema.Struct({ id: Schema.Finite, head_sha: Sha })), +}) + +export const GitRefSchema = Schema.Struct({ object: Schema.Struct({ sha: Sha }) }) diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index 2b5be58..e9e02e1 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -30,7 +30,7 @@ src = self; pname = "ratstack-scorecard"; pnpm = pkgs.pnpm_12; - hash = toolsStoreHash.${system}; + hash = "sha256-9/W9Q1CNzoBbRBhd+qdu7TfR8xk/7mjoiEOkOLwSRpU="; }).pnpm-store; ratstack-src = pkgs.fetchFromGitHub { inherit (pin) owner repo; @@ -50,7 +50,7 @@ export SCORECARD_PNPM_VERSION=${pkgs.pnpm_12.version} export DENO_NO_PACKAGE_JSON=1 exec deno run --no-config --allow-read --allow-write --allow-env --allow-sys=hostname \ - --allow-net=api.github.com --allow-run=git,${pkgs.deno}/bin/deno,${sandbox}/bin/sandbox \ + --allow-run=git,${pkgs.deno}/bin/deno,${sandbox}/bin/sandbox \ ${self}/src/main.ts "$@" ''; }; diff --git a/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts b/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts index a653693..1931381 100644 --- a/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts +++ b/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts @@ -1,4 +1,4 @@ -import Ajv from 'ajv' +import { Ajv } from 'ajv' import { describe, expect, test } from 'vitest' import schema from '../scorecard.schema.json' with { type: 'json' } import { launcherRun } from './launcher-run.ts' diff --git a/evals/ratstack-scorecard/journeys/manifest.json b/evals/ratstack-scorecard/journeys/manifest.json index e8f13d2..e8a38a1 100644 --- a/evals/ratstack-scorecard/journeys/manifest.json +++ b/evals/ratstack-scorecard/journeys/manifest.json @@ -39,7 +39,8 @@ }, "inputs": [ "journeys/__fixtures__/aggregate", - "src" + "src", + "decide" ] }, { @@ -51,7 +52,8 @@ }, "inputs": [ "journeys/__fixtures__/aggregate", - "src" + "src", + "decide" ] } ] diff --git a/evals/ratstack-scorecard/package.json b/evals/ratstack-scorecard/package.json index 0ff3560..f18aa21 100644 --- a/evals/ratstack-scorecard/package.json +++ b/evals/ratstack-scorecard/package.json @@ -11,6 +11,11 @@ "ajv": "8.20.0", "fast-check": "4.10.2", "vitest": "5.0.3", - "effect": "4.0.1" + "effect": "4.0.1", + "typescript": "7.0.2", + "@types/node": "24.19.1" + }, + "scripts": { + "typecheck": "tsc -p tsconfig.json" } } diff --git a/evals/ratstack-scorecard/pnpm-lock.yaml b/evals/ratstack-scorecard/pnpm-lock.yaml index 3ee8937..4fa8b94 100644 --- a/evals/ratstack-scorecard/pnpm-lock.yaml +++ b/evals/ratstack-scorecard/pnpm-lock.yaml @@ -17,7 +17,10 @@ importers: devDependencies: '@fast-check/vitest': specifier: 0.5.0 - version: 0.5.0(vitest@5.0.3(vite@8.3.2(yaml@2.9.1))) + version: 0.5.0(vitest@5.0.3(@types/node@24.19.1)(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1))) + '@types/node': + specifier: 24.19.1 + version: 24.19.1 ajv: specifier: 8.20.0 version: 8.20.0 @@ -27,9 +30,12 @@ importers: fast-check: specifier: 4.10.2 version: 4.10.2 + typescript: + specifier: 7.0.2 + version: 7.0.2 vitest: specifier: 5.0.3 - version: 5.0.3(vite@8.3.2(yaml@2.9.1)) + version: 5.0.3(@types/node@24.19.1)(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1)) packages: @@ -281,6 +287,129 @@ packages: '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + '@types/node@24.19.1': + resolution: {integrity: sha512-aS3/DG0oM05K0RIXXP+hKjinGG5IgSSVGzswZxW3O0sS3pH4/fycXundUC9XsszgKCk4gHXylTEK6hyFxVxnoQ==} + + '@typescript/typescript-aix-ppc64@7.0.2': + resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} + engines: {node: '>=16.20.0'} + cpu: [ppc64] + os: [aix] + + '@typescript/typescript-darwin-arm64@7.0.2': + resolution: {integrity: sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [darwin] + + '@typescript/typescript-darwin-x64@7.0.2': + resolution: {integrity: sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [darwin] + + '@typescript/typescript-freebsd-arm64@7.0.2': + resolution: {integrity: sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [freebsd] + + '@typescript/typescript-freebsd-x64@7.0.2': + resolution: {integrity: sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [freebsd] + + '@typescript/typescript-linux-arm64@7.0.2': + resolution: {integrity: sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [linux] + + '@typescript/typescript-linux-arm@7.0.2': + resolution: {integrity: sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==} + engines: {node: '>=16.20.0'} + cpu: [arm] + os: [linux] + + '@typescript/typescript-linux-loong64@7.0.2': + resolution: {integrity: sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==} + engines: {node: '>=16.20.0'} + cpu: [loong64] + os: [linux] + + '@typescript/typescript-linux-mips64el@7.0.2': + resolution: {integrity: sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==} + engines: {node: '>=16.20.0'} + cpu: [mips64el] + os: [linux] + + '@typescript/typescript-linux-ppc64@7.0.2': + resolution: {integrity: sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==} + engines: {node: '>=16.20.0'} + cpu: [ppc64] + os: [linux] + + '@typescript/typescript-linux-riscv64@7.0.2': + resolution: {integrity: sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==} + engines: {node: '>=16.20.0'} + cpu: [riscv64] + os: [linux] + + '@typescript/typescript-linux-s390x@7.0.2': + resolution: {integrity: sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==} + engines: {node: '>=16.20.0'} + cpu: [s390x] + os: [linux] + + '@typescript/typescript-linux-x64@7.0.2': + resolution: {integrity: sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [linux] + + '@typescript/typescript-netbsd-arm64@7.0.2': + resolution: {integrity: sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [netbsd] + + '@typescript/typescript-netbsd-x64@7.0.2': + resolution: {integrity: sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [netbsd] + + '@typescript/typescript-openbsd-arm64@7.0.2': + resolution: {integrity: sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [openbsd] + + '@typescript/typescript-openbsd-x64@7.0.2': + resolution: {integrity: sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [openbsd] + + '@typescript/typescript-sunos-x64@7.0.2': + resolution: {integrity: sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [sunos] + + '@typescript/typescript-win32-arm64@7.0.2': + resolution: {integrity: sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [win32] + + '@typescript/typescript-win32-x64@7.0.2': + resolution: {integrity: sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [win32] + '@vitest/mocker@5.0.3': resolution: {integrity: sha512-T8sWAIbkSyAjkwTcaEc3Iu0o9A27X1/kdXrizhZkGuSKScRQtRzclfAMpOTcGdXCsqxeWlpGy3XjqaW8CpLORg==} peerDependencies: @@ -482,6 +611,14 @@ packages: resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} engines: {node: '>=12.0.0'} + typescript@7.0.2: + resolution: {integrity: sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==} + engines: {node: '>=16.20.0'} + hasBin: true + + undici-types@7.24.6: + resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + vite@8.3.2: resolution: {integrity: sha512-SQr1x6W5vVSbROg7vsyXIaxK9b0G7zsT68acdWWRmnBUsgDieLCRG+Rep9WdZgcposvv/GSnr4GUUBqB3vXq6w==} engines: {node: ^20.19.0 || >=22.12.0} @@ -578,10 +715,10 @@ packages: snapshots: - '@fast-check/vitest@0.5.0(vitest@5.0.3(vite@8.3.2(yaml@2.9.1)))': + '@fast-check/vitest@0.5.0(vitest@5.0.3(@types/node@24.19.1)(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1)))': dependencies: fast-check: 4.10.2 - vitest: 5.0.3(vite@8.3.2(yaml@2.9.1)) + vitest: 5.0.3(@types/node@24.19.1)(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1)) '@jridgewell/resolve-uri@3.1.2': {} @@ -707,14 +844,78 @@ snapshots: '@types/estree@1.0.9': {} - '@vitest/mocker@5.0.3(vite@8.3.2(yaml@2.9.1))': + '@types/node@24.19.1': + dependencies: + undici-types: 7.24.6 + + '@typescript/typescript-aix-ppc64@7.0.2': + optional: true + + '@typescript/typescript-darwin-arm64@7.0.2': + optional: true + + '@typescript/typescript-darwin-x64@7.0.2': + optional: true + + '@typescript/typescript-freebsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-freebsd-x64@7.0.2': + optional: true + + '@typescript/typescript-linux-arm64@7.0.2': + optional: true + + '@typescript/typescript-linux-arm@7.0.2': + optional: true + + '@typescript/typescript-linux-loong64@7.0.2': + optional: true + + '@typescript/typescript-linux-mips64el@7.0.2': + optional: true + + '@typescript/typescript-linux-ppc64@7.0.2': + optional: true + + '@typescript/typescript-linux-riscv64@7.0.2': + optional: true + + '@typescript/typescript-linux-s390x@7.0.2': + optional: true + + '@typescript/typescript-linux-x64@7.0.2': + optional: true + + '@typescript/typescript-netbsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-netbsd-x64@7.0.2': + optional: true + + '@typescript/typescript-openbsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-openbsd-x64@7.0.2': + optional: true + + '@typescript/typescript-sunos-x64@7.0.2': + optional: true + + '@typescript/typescript-win32-arm64@7.0.2': + optional: true + + '@typescript/typescript-win32-x64@7.0.2': + optional: true + + '@vitest/mocker@5.0.3(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1))': dependencies: '@jridgewell/trace-mapping': 0.3.31 '@vitest/spy': 5.0.3 estree-walker: 3.0.3 magic-string: 1.4.2 optionalDependencies: - vite: 8.3.2(yaml@2.9.1) + vite: 8.3.2(@types/node@24.19.1)(yaml@2.9.1) '@vitest/spy@5.0.3': {} @@ -887,7 +1088,32 @@ snapshots: fdir: 6.5.0(picomatch@4.0.7) picomatch: 4.0.7 - vite@8.3.2(yaml@2.9.1): + typescript@7.0.2: + optionalDependencies: + '@typescript/typescript-aix-ppc64': 7.0.2 + '@typescript/typescript-darwin-arm64': 7.0.2 + '@typescript/typescript-darwin-x64': 7.0.2 + '@typescript/typescript-freebsd-arm64': 7.0.2 + '@typescript/typescript-freebsd-x64': 7.0.2 + '@typescript/typescript-linux-arm': 7.0.2 + '@typescript/typescript-linux-arm64': 7.0.2 + '@typescript/typescript-linux-loong64': 7.0.2 + '@typescript/typescript-linux-mips64el': 7.0.2 + '@typescript/typescript-linux-ppc64': 7.0.2 + '@typescript/typescript-linux-riscv64': 7.0.2 + '@typescript/typescript-linux-s390x': 7.0.2 + '@typescript/typescript-linux-x64': 7.0.2 + '@typescript/typescript-netbsd-arm64': 7.0.2 + '@typescript/typescript-netbsd-x64': 7.0.2 + '@typescript/typescript-openbsd-arm64': 7.0.2 + '@typescript/typescript-openbsd-x64': 7.0.2 + '@typescript/typescript-sunos-x64': 7.0.2 + '@typescript/typescript-win32-arm64': 7.0.2 + '@typescript/typescript-win32-x64': 7.0.2 + + undici-types@7.24.6: {} + + vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1): dependencies: lightningcss: 1.33.0 picomatch: 4.0.7 @@ -895,13 +1121,14 @@ snapshots: rolldown: 1.2.12 tinyglobby: 0.2.17 optionalDependencies: + '@types/node': 24.19.1 fsevents: 2.3.3 yaml: 2.9.1 - vitest@5.0.3(vite@8.3.2(yaml@2.9.1)): + vitest@5.0.3(@types/node@24.19.1)(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1)): dependencies: '@types/chai': 5.2.3 - '@vitest/mocker': 5.0.3(vite@8.3.2(yaml@2.9.1)) + '@vitest/mocker': 5.0.3(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1)) chai: 6.3.0 es-module-lexer: 2.3.2 expect-type: 1.4.0 @@ -912,8 +1139,10 @@ snapshots: tinybench: 6.2.0 tinyexec: 1.3.1 tinyglobby: 0.2.17 - vite: 8.3.2(yaml@2.9.1) + vite: 8.3.2(@types/node@24.19.1)(yaml@2.9.1) why-is-node-running: 3.2.1 + optionalDependencies: + '@types/node': 24.19.1 transitivePeerDependencies: - msw diff --git a/evals/ratstack-scorecard/scorecard.schema.json b/evals/ratstack-scorecard/scorecard.schema.json index 62f1d3b..c9ba370 100644 --- a/evals/ratstack-scorecard/scorecard.schema.json +++ b/evals/ratstack-scorecard/scorecard.schema.json @@ -248,7 +248,18 @@ "type": "object", "additionalProperties": false, "required": ["_tag", "id"], - "properties": { "_tag": { "enum": ["Held", "New", "ReBaselined"] }, "id": { "type": "string" } } + "properties": { "_tag": { "enum": ["Held", "New"] }, "id": { "type": "string" } } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["_tag", "id", "mainHash", "prHash"], + "properties": { + "_tag": { "const": "ReBaselined" }, + "id": { "type": "string" }, + "mainHash": { "type": "string" }, + "prHash": { "type": "string" } + } }, { "type": "object", diff --git a/evals/ratstack-scorecard/src/harness/scorecard-codec.ts b/evals/ratstack-scorecard/src/harness/scorecard-codec.ts deleted file mode 100644 index 01bcd3f..0000000 --- a/evals/ratstack-scorecard/src/harness/scorecard-codec.ts +++ /dev/null @@ -1,170 +0,0 @@ -import type { - Cell, - CellProvenance, - FamilyResult, - Flag, - MeasuredCell, - Ratchet, - RatstackSupport, - Row, - RowDefinition, - RowOutcome, - ScorecardDocument, - Side, - SideCell, - Verdict, -} from '../model/cell.ts' -import type { CellsByRow } from '../model/scorecard-document.ts' -import { - array, - type Decoder, - literal, - number, - oneOf, - optional, - record, - string, - struct, - tuple2, - union, -} from './decode.ts' - -const cellTags = [ - 'Measured', - 'Absent', - 'NoDeployment', - 'Unsupported', - 'Unmeasurable', - 'NoSecret', - 'InstrumentError', -] as const - -const side = oneOf(['ratstack', 'starter']) - -export const cell: Decoder = union( - struct({ _tag: literal('Measured'), runs: array(number) }), - struct({ _tag: literal('Absent'), reason: string }), - struct({ _tag: literal('NoDeployment'), sha: string }), - struct({ - _tag: literal('Unsupported'), - citation: struct({ file: string, lines: tuple2(number, number), text: string }), - check: union<{ readonly _tag: 'Verified' } | { readonly _tag: 'Contradicted'; readonly found: string }>( - struct({ _tag: literal('Verified') }), - struct({ _tag: literal('Contradicted'), found: string }), - ), - }), - struct({ _tag: literal('Unmeasurable'), error: string }), - struct({ _tag: literal('NoSecret'), name: string }), - struct({ _tag: literal('InstrumentError'), error: string }), -) - -const provenanceFields = struct({ - side, - commit: string, - instrumentHash: string, - nixpkgsRev: string, - runner: string, - measuredAt: string, - tools: record(string), - liveCommit: optional(string), - detail: optional(record(union(number, string))), -}) - -export const cellProvenance: Decoder = (value, path) => { - const { liveCommit, detail, ...required } = provenanceFields(value, path) - return { - ...required, - ...(liveCommit === undefined ? {} : { liveCommit }), - ...(detail === undefined ? {} : { detail }), - } -} - -const measuredCell: Decoder = struct({ cell, provenance: cellProvenance }) - -const flag: Decoder = union( - struct({ _tag: literal('LiveDiffersFromPin'), live: string, pin: string }), - struct({ _tag: literal('ScannersDisagree'), primary: number, crossCheck: number }), -) - -const rowDefinition: Decoder = struct({ - id: string, - metric: string, - bin: string, - label: string, - unit: string, - direction: oneOf(['lower', 'higher']), - kind: oneOf(['count', 'measurement']), - runs: number, - family: oneOf(['static', 'cold-path', 'gate-mutation', 'running-stack', 'agent-surfaces', 'networked']), - ratstackSupport: union( - struct({ _tag: literal('Required') }), - struct({ _tag: literal('MayBeUnsupported'), bar: number }), - ), -}) - -const verdict: Decoder = union( - struct({ _tag: oneOf(['Beaten', 'NotBeaten', 'Tie']) }), - struct({ _tag: literal('InstrumentError'), error: string }), -) - -const row: Decoder = struct({ - definition: rowDefinition, - definitionHash: string, - ratstack: measuredCell, - starter: measuredCell, - verdict, - flags: array(flag), -}) - -const cellTag = oneOf(cellTags) - -const outcome: Decoder = union( - struct({ _tag: oneOf(['Held', 'New', 'ReBaselined']), id: string }), - struct({ _tag: literal('Neutral'), id: string, cause: cellTag }), - struct({ _tag: literal('InstrumentError'), id: string, error: string }), - struct({ _tag: literal('LostBeaten'), id: string, ratstack: cellTag, starter: cellTag }), - struct({ _tag: literal('Regressed'), id: string, main: string, pr: string }), -) - -const ratchet: Decoder = union( - struct({ _tag: literal('FirstBaseline'), outcomes: array(outcome), failures: array(outcome) }), - struct({ _tag: literal('Compared'), mainCommit: string, outcomes: array(outcome), failures: array(outcome) }), -) - -export const scorecardDocument: Decoder = struct({ - schemaVersion: literal(1), - provenance: struct({ - commit: string, - ratstackCommit: string, - instrumentHash: string, - nixpkgsRev: string, - runner: string, - generatedAt: string, - }), - rows: array(row), - ratchet, -}) - -export const familyResult: Decoder = struct({ - family: oneOf(['static', 'cold-path', 'gate-mutation', 'running-stack', 'agent-surfaces', 'networked']), - wallMs: number, - cells: array(struct({ id: string, side, measured: measuredCell })), - flags: array(struct({ id: string, flag })), - definitionHashes: array(struct({ id: string, hash: string })), -}) - -export class DuplicateCell extends Error { - constructor(readonly id: string, readonly side: Side) { - super(`two family results carry a ${side} cell for ${id}; each (row, side) is measured once`) - } -} - -export const cellsByRow = (cells: readonly SideCell[]): CellsByRow => { - const keyed: Record>> = {} - for (const { id, side, measured } of cells) { - const row = keyed[id] ?? {} - if (row[side] !== undefined) throw new DuplicateCell(id, side) - keyed[id] = { ...row, [side]: measured } - } - return keyed -} diff --git a/evals/ratstack-scorecard/src/journeys.ts b/evals/ratstack-scorecard/src/journeys.ts index 9813b64..cb86944 100644 --- a/evals/ratstack-scorecard/src/journeys.ts +++ b/evals/ratstack-scorecard/src/journeys.ts @@ -98,8 +98,14 @@ interface Aggregate { readonly main: string | null } -const produceAggregate = async (root: string, entry: JourneyEntry, spec: Aggregate): Promise => { - const out = await Deno.makeTempDir({ prefix: `journey-${entry.id}-` }) +const produceAggregate = async ( + instrument: Instrument, + root: string, + entry: JourneyEntry, + spec: Aggregate, +): Promise => { + await Deno.mkdir(join(root, '.cache'), { recursive: true }) + const out = await Deno.makeTempDir({ dir: join(root, '.cache'), prefix: `journey-${entry.id}-` }) const started = performance.now() try { const argv = [ @@ -113,8 +119,18 @@ const produceAggregate = async (root: string, entry: JourneyEntry, spec: Aggrega join(out, 'summary.md'), ] const run = await new Deno.Command(Deno.execPath(), { - args: ['run', '--no-config', '--allow-read', '--allow-write', '--allow-env', join(root, 'src/main.ts'), ...argv], - env: { DENO_NO_PACKAGE_JSON: '1' }, + args: [ + 'run', + '--no-config', + '--allow-read', + '--allow-write', + '--allow-env', + '--allow-sys=hostname', + `--allow-run=git,${instrument.launcher.executable}`, + join(root, 'src/main.ts'), + ...argv, + ], + env: { ...Deno.env.toObject(), DENO_NO_PACKAGE_JSON: '1' }, stdout: 'piped', stderr: 'piped', }).output() @@ -181,7 +197,7 @@ const produceStatic = async ( const produce = async (instrument: Instrument, root: string, entry: JourneyEntry): Promise => { const record = entry.produce.kind === 'aggregate' - ? await produceAggregate(root, entry, entry.produce) + ? await produceAggregate(instrument, root, entry, entry.produce) : await produceStatic(instrument, root, entry, entry.produce) return { ...record, inputHash: await inputHashOf(root, entry.inputs, instrument.launcher.executable) } } diff --git a/evals/ratstack-scorecard/src/main.ts b/evals/ratstack-scorecard/src/main.ts index fc0a3b5..1f8e339 100644 --- a/evals/ratstack-scorecard/src/main.ts +++ b/evals/ratstack-scorecard/src/main.ts @@ -2,22 +2,19 @@ import { join } from 'node:path' import { parseArgs } from 'node:util' import { checkImports } from './check-imports.ts' import { measureStatic } from './families/static.ts' -import { array, decodeJson, number, string, struct } from './harness/decode.ts' -import { git, type Instrument, loadInstrument } from './harness/instrument.ts' -import { cellsByRow, familyResult, scorecardDocument } from './harness/scorecard-codec.ts' +import { type Instrument, loadInstrument } from './harness/instrument.ts' +import { runSandboxed } from './harness/sandbox.ts' import { runJourneys } from './journeys.ts' -import { familyTimeoutMinutes, rowDefinitions } from './metrics/registry.ts' +import { familyTimeoutMinutes } from './metrics/registry.ts' import { ratstackCacheKey } from './model/cache-key.ts' -import type { Family, FamilyResult, MainBaseline, Side } from './model/cell.ts' -import { planPinBump } from './model/plan-pin-bump.workflow.ts' -import { assembleScorecard } from './model/scorecard-document.ts' -import { renderSummary } from './model/summary-table.ts' +import type { Family, FamilyResult, Side } from './model/cell.ts' const usage = [ 'usage:', ' scorecard plan', ' scorecard measure --family [--side ratstack|starter] [--out ] [--checkout ]', ' scorecard aggregate --families [--main ] --out --summary ', + ' scorecard cache-check --file ', ' scorecard latest-main-run', ' scorecard pin check', ' scorecard pin write --commit --nar-hash [--checkout ]', @@ -101,86 +98,52 @@ const check = async (args: readonly string[]): Promise => { Deno.exit(await runJourneys(instrument, root)) } -const readFamilyResults = async (dir: string): Promise => { - const results: FamilyResult[] = [] - for await (const entry of Deno.readDir(dir)) { - if (!entry.isFile || !entry.name.endsWith('.json')) continue - const path = join(dir, entry.name) - results.push(decodeJson(familyResult, await Deno.readTextFile(path), path)) - } - return results -} - -const mainBaseline = async (path: string | undefined): Promise => { - if (path === undefined) return { _tag: 'Missing' } - const doc = decodeJson(scorecardDocument, await Deno.readTextFile(path), path) - return { _tag: 'Found', commit: doc.provenance.commit, rows: doc.rows } +const decide = async ( + instrument: Instrument, + args: readonly string[], + options: { readonly github: boolean }, +): Promise => { + const root = join(instrument.checkout, 'evals/ratstack-scorecard') + const install = await runSandboxed(instrument.launcher, { + project: instrument.checkout, + cwd: root, + command: ['pnpm', 'install', '--frozen-lockfile'], + pnpmStore: instrument.toolsStore, + deadlineMs: 10 * 60_000, + }) + if (install.code !== 0) fail(`installing the decide step's dependencies failed:\n${install.stderr}`) + const token = Deno.env.get('GITHUB_TOKEN') + const result = await runSandboxed(instrument.launcher, { + project: instrument.checkout, + cwd: Deno.cwd(), + command: ['node', join(root, 'decide/main.ts'), ...args], + ...(options.github ? { allowHosts: ['api.github.com'] } : {}), + ...(options.github && token !== undefined ? { env: { GITHUB_TOKEN: token } } : {}), + deadlineMs: 5 * 60_000, + }) + await Deno.stdout.write(new TextEncoder().encode(result.stdout)) + await Deno.stderr.write(new TextEncoder().encode(result.stderr)) + return result.code } const aggregate = async (args: readonly string[]): Promise => { - const { values } = parseArgs({ - args: [...args], - options: { - families: { type: 'string' }, - main: { type: 'string' }, - out: { type: 'string' }, - summary: { type: 'string' }, - }, - strict: true, - }) - const results = await readFamilyResults(values.families ?? fail('--families is required')) - const cells = results.flatMap((result) => result.cells) - const hashes = results.flatMap((result) => result.definitionHashes) - const provenanceOf = (side: Side) => cells.find((cell) => cell.side === side)?.measured.provenance - const starter = provenanceOf('starter') - const ratstack = provenanceOf('ratstack') - const any = starter ?? ratstack ?? fail(`no family results with cells in ${values.families}`) - const doc = assembleScorecard({ - rows: rowDefinitions - .filter((definition) => implementedFamilies.includes(definition.family)) - .map((definition) => ({ - definition, - hash: hashes.find((hash) => hash.id === definition.id)?.hash ?? `no ${definition.family} result`, - })), - cells: cellsByRow(cells), - flags: results.flatMap((result) => result.flags), - provenance: { - commit: starter?.commit ?? Deno.env.get('GITHUB_SHA') ?? 'unknown', - ratstackCommit: ratstack?.commit ?? 'unknown', - instrumentHash: any.instrumentHash, - nixpkgsRev: any.nixpkgsRev, - runner: any.runner, - generatedAt: new Date().toISOString(), - }, - main: await mainBaseline(values.main), - }) - await writeOut(values.out ?? fail('--out is required'), `${JSON.stringify(doc, null, 2)}\n`) - await writeOut(values.summary ?? fail('--summary is required'), renderSummary(doc)) - const failures = doc.ratchet.failures - if (failures.length > 0) { - console.error(`scorecard: ${failures.length} failing rows: ${failures.map((failure) => failure.id).join(', ')}`) - Deno.exit(1) - } + const instrument = await loadInstrument(undefined) + const commit = ['--commit', Deno.env.get('GITHUB_SHA') ?? instrument.starterCommit] + Deno.exit( + await decide(instrument, ['aggregate', '--implemented', implementedFamilies.join(','), ...commit, ...args], { + github: false, + }), + ) } -const workflowRuns = struct({ workflow_runs: array(struct({ id: number, head_sha: string })) }) +const cacheCheck = async (args: readonly string[]): Promise => + Deno.exit(await decide(await loadInstrument(undefined), ['cache-check', ...args], { github: false })) const latestMainRun = async (): Promise => { - const api = Deno.env.get('GITHUB_API_URL') ?? 'https://api.github.com' const repository = Deno.env.get('GITHUB_REPOSITORY') ?? fail('GITHUB_REPOSITORY is unset') - const token = Deno.env.get('GITHUB_TOKEN') ?? fail('GITHUB_TOKEN is unset') - const response = await fetch( - `${api}/repos/${repository}/actions/workflows/scorecard.yml/runs?branch=main&event=push&status=success&per_page=1`, - { headers: { authorization: `Bearer ${token}`, accept: 'application/vnd.github+json' } }, + Deno.exit( + await decide(await loadInstrument(undefined), ['latest-main-run', '--repository', repository], { github: true }), ) - if (response.status === 404) { - await response.body?.cancel() - await writeOut(undefined, '\n') - return - } - if (!response.ok) fail(`listing scorecard runs failed: ${response.status} ${await response.text()}`) - const { workflow_runs } = decodeJson(workflowRuns, await response.text(), 'GitHub workflow runs') - await writeOut(undefined, `${workflow_runs.map((run) => run.id).join('')}\n`) } const pinCommand = async (args: readonly string[]): Promise => { @@ -192,16 +155,10 @@ const pinCommand = async (args: readonly string[]): Promise => { }) const instrument = await loadInstrument(values.checkout) if (action === 'check') { - const remote = new TextDecoder().decode( - await git(instrument.checkout, [ - 'ls-remote', - `https://github.com/${instrument.pin.owner}/${instrument.pin.repo}`, - 'refs/heads/main', - ]), + const { owner, repo, commit } = instrument.pin + Deno.exit( + await decide(instrument, ['pin-check', '--owner', owner, '--repo', repo, '--pinned', commit], { github: true }), ) - const remoteHead = string(remote.split(/\s/u)[0], 'ls-remote refs/heads/main') - await writeOut(undefined, `${JSON.stringify(planPinBump({ pinned: instrument.pin.commit, remoteHead }))}\n`) - return } if (action === 'write') { const pin = { @@ -220,6 +177,7 @@ const commands: Readonly Promise plan(), 'measure': measure, 'aggregate': aggregate, + 'cache-check': cacheCheck, 'latest-main-run': () => latestMainRun(), 'pin': pinCommand, 'journeys': journeys, diff --git a/evals/ratstack-scorecard/src/model/cell.ts b/evals/ratstack-scorecard/src/model/cell.ts index 7e98f19..3e2eb7e 100644 --- a/evals/ratstack-scorecard/src/model/cell.ts +++ b/evals/ratstack-scorecard/src/model/cell.ts @@ -113,7 +113,7 @@ export interface FamilyResult { export type RowOutcome = | { readonly _tag: 'Held'; readonly id: string } | { readonly _tag: 'New'; readonly id: string } - | { readonly _tag: 'ReBaselined'; readonly id: string } + | { readonly _tag: 'ReBaselined'; readonly id: string; readonly mainHash: string; readonly prHash: string } | { readonly _tag: 'Neutral'; readonly id: string; readonly cause: CellTag } | { readonly _tag: 'InstrumentError'; readonly id: string; readonly error: string } | { readonly _tag: 'LostBeaten'; readonly id: string; readonly ratstack: CellTag; readonly starter: CellTag } diff --git a/evals/ratstack-scorecard/src/model/compare-with-main.workflow.property.test.ts b/evals/ratstack-scorecard/src/model/compare-with-main.workflow.property.test.ts index e96db22..19cd01b 100644 --- a/evals/ratstack-scorecard/src/model/compare-with-main.workflow.property.test.ts +++ b/evals/ratstack-scorecard/src/model/compare-with-main.workflow.property.test.ts @@ -48,7 +48,9 @@ describe('compareWithMain', () => { rows: [pr], main: { _tag: 'Found', commit, rows: [{ ...main, verdict: { _tag: 'Beaten' } }] }, }) - return ratchet.failures.length === 0 && ratchet.outcomes[0]!._tag === 'ReBaselined' + return ratchet.failures.length === 0 && + JSON.stringify(ratchet.outcomes) === + JSON.stringify([{ _tag: 'ReBaselined', id: main.definition.id, mainHash: main.definitionHash, prHash: hash }]) }) test.prop([ diff --git a/evals/ratstack-scorecard/src/model/compare-with-main.workflow.ts b/evals/ratstack-scorecard/src/model/compare-with-main.workflow.ts index f313ae2..b81eaf9 100644 --- a/evals/ratstack-scorecard/src/model/compare-with-main.workflow.ts +++ b/evals/ratstack-scorecard/src/model/compare-with-main.workflow.ts @@ -61,7 +61,10 @@ const starterAgainstMain = (main: Row, pr: Row): RowOutcome => const compareRow = (main: Row, pr: Row): RowOutcome => firstRule([ - [main.definitionHash !== pr.definitionHash, () => ({ _tag: 'ReBaselined', id: pr.definition.id })], + [ + main.definitionHash !== pr.definitionHash, + () => ({ _tag: 'ReBaselined', id: pr.definition.id, mainHash: main.definitionHash, prHash: pr.definitionHash }), + ], [ beatenWeight(main.verdict) > beatenWeight(pr.verdict), () => ({ diff --git a/evals/ratstack-scorecard/src/model/summary-table.ts b/evals/ratstack-scorecard/src/model/summary-table.ts index 5c159cd..3d6b439 100644 --- a/evals/ratstack-scorecard/src/model/summary-table.ts +++ b/evals/ratstack-scorecard/src/model/summary-table.ts @@ -44,7 +44,7 @@ const showOutcome = (outcome: RowOutcome): string => matchOutcome(outcome, { Held: () => 'held', New: () => 'new', - ReBaselined: () => 're-baselined', + ReBaselined: (o) => `re-baselined (definition \`${short(o.mainHash)}\` → \`${short(o.prHash)}\`)`, Neutral: (o) => `neutral (${o.cause})`, InstrumentError: (o) => `**failed**: ${o.error}`, LostBeaten: (o) => `**failed**: no longer beaten (rat-stack ${o.ratstack}, starter ${o.starter})`, @@ -57,6 +57,29 @@ const outcomesOf = (doc: ScorecardDocument): readonly RowOutcome[] => const failuresOf = (doc: ScorecardDocument): readonly RowOutcome[] => matchRatchet(doc.ratchet, { FirstBaseline: (r) => r.failures, Compared: (r) => r.failures }) +const rebaselinedOf = (outcome: RowOutcome): readonly { id: string; mainHash: string; prHash: string }[] => + matchOutcome(outcome, { + Held: () => [], + New: () => [], + ReBaselined: (o) => [o], + Neutral: () => [], + InstrumentError: () => [], + LostBeaten: () => [], + Regressed: () => [], + }) + +const drift = (outcomes: readonly RowOutcome[]): readonly string[] => { + const rebaselined = outcomes.flatMap(rebaselinedOf) + return [ + ...rebaselined.slice(0, 1).flatMap(() => [ + `**Definition drift:** ${rebaselined.length} rows re-baselined; their ratchet starts over from this run.`, + '', + ]), + ...rebaselined.map((o) => `- ${o.id}: \`${o.mainHash}\` on \`main\`, \`${o.prHash}\` here`), + ...rebaselined.slice(0, 1).map(() => ''), + ] +} + const ratchetLine = (doc: ScorecardDocument): string => matchRatchet(doc.ratchet, { FirstBaseline: (r) => `first baseline (no \`main\` artifact), ${r.failures.length} failing rows`, @@ -91,6 +114,7 @@ export const renderSummary = (doc: ScorecardDocument): string => '', ...failuresOf(doc).map((failure) => `- ${failure.id}: ${cellText(showOutcome(failure))}`), '', + ...drift(outcomesOf(doc)), `Beaten: ${doc.rows.filter((row) => row.verdict._tag === 'Beaten').length} of ${doc.rows.length} rows.`, '', '| Row | Bin | Metric | rat-stack | starter | Verdict | Ratchet | Flags |', diff --git a/evals/ratstack-scorecard/tsconfig.json b/evals/ratstack-scorecard/tsconfig.json new file mode 100644 index 0000000..9483e66 --- /dev/null +++ b/evals/ratstack-scorecard/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "strict": true, + "exactOptionalPropertyTypes": true, + "noUncheckedIndexedAccess": true, + "target": "es2024", + "module": "nodenext", + "moduleResolution": "nodenext", + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "resolveJsonModule": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["decide/**/*.ts", "journeys/**/*.ts"] +} From a3387656c713fa61a345e3b4bbacabfb2704f088 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Oct 2026 06:06:58 +0000 Subject: [PATCH 4/6] ci(ci): call one stable scorecard entrypoint from the base instrument, bootstrapping only without one Kiro's bootstrap ruling: the workflow builds the grading instrument and runs scorecard ci, which measures, caches, fetches main's scorecard, judges and publishes inside the instrument. ci/instrument-ref.sh, run from the base tree when the base has it, grades a PR with its base's instrument and uses the PR head only when the base has no scorecard workflow, announcing BOOTSTRAP in the summary and a PR comment. The instrument-ref journey proves on real git history that editing or deleting the instrument or the workflow on a base that has one is still graded by the base. instrumentHash now hashes the grading instrument's own files --- .github/workflows/scorecard.yml | 189 +++++----------- evals/ratstack-scorecard/README.md | 11 +- evals/ratstack-scorecard/ci/instrument-ref.sh | 26 +++ evals/ratstack-scorecard/decide/main.ts | 115 ++++++++-- evals/ratstack-scorecard/decide/schema.ts | 4 + evals/ratstack-scorecard/flake.nix | 4 +- .../journeys/instrument-ref.test.ts | 42 ++++ .../ratstack-scorecard/journeys/manifest.json | 10 + .../ratstack-scorecard/src/harness/decide.ts | 49 ++++ .../src/harness/instrument.ts | 24 +- evals/ratstack-scorecard/src/journeys.ts | 97 +++++++- evals/ratstack-scorecard/src/main.ts | 209 +++++++++++++----- .../src/metrics/registry.ts | 11 +- 13 files changed, 550 insertions(+), 241 deletions(-) create mode 100755 evals/ratstack-scorecard/ci/instrument-ref.sh create mode 100644 evals/ratstack-scorecard/journeys/instrument-ref.test.ts create mode 100644 evals/ratstack-scorecard/src/harness/decide.ts diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index b33e449..f04a5aa 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -17,151 +17,87 @@ concurrency: group: scorecard-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} -env: - INSTRUMENT_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }} - jobs: - plan: + scorecard: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 360 + permissions: + contents: read + actions: read + pull-requests: write outputs: - matrix: ${{ steps.plan.outputs.matrix }} - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ env.INSTRUMENT_REF }} - path: instrument - - uses: cachix/install-nix-action@v31 - - id: plan - name: Families and rat-stack cache keys - working-directory: instrument - run: | - scorecard="$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#scorecard)" - matrix="$("$scorecard/bin/scorecard" plan)" - echo "$matrix" | jq . - echo "matrix=$matrix" >> "$GITHUB_OUTPUT" - - measure: - name: measure (${{ matrix.family }}) - needs: plan - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} - runs-on: ubuntu-latest - timeout-minutes: ${{ matrix.timeoutMinutes }} + mode: ${{ steps.instrument.outputs.mode }} steps: - - uses: actions/checkout@v7 - uses: actions/checkout@v7 with: - ref: ${{ env.INSTRUMENT_REF }} - path: .scorecard-instrument - - uses: cachix/install-nix-action@v31 - - name: Allow the launcher's unprivileged user namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - - name: Build main's instrument - run: echo "SCORECARD=$(nix build --no-link --print-out-paths ./.scorecard-instrument/evals/ratstack-scorecard#scorecard)/bin/scorecard" >> "$GITHUB_ENV" - - id: ratstack-cache - name: Restore the rat-stack side - uses: actions/cache/restore@v6 - with: - path: scorecard-out/ratstack-${{ matrix.family }}.json - key: ${{ matrix.cacheKey }} - - id: cached - name: Refuse a cached rat-stack side that holds an instrument error - if: steps.ratstack-cache.outputs.cache-hit == 'true' + fetch-depth: 0 + - id: instrument + name: Choose the instrument that grades this run + env: + BASE: ${{ github.event.pull_request.base.sha }} run: | - if "$SCORECARD" cache-check --file "scorecard-out/ratstack-${{ matrix.family }}.json"; then - echo "usable=true" >> "$GITHUB_OUTPUT" - else - rm -f "scorecard-out/ratstack-${{ matrix.family }}.json" - echo "::warning::cached rat-stack ${{ matrix.family }} side refused; measuring it again" + if [ "$GITHUB_EVENT_NAME" != pull_request ]; then + printf 'ref=%s\nmode=base\n' "$GITHUB_SHA" >> "$GITHUB_OUTPUT" + exit 0 fi - - name: Measure the rat-stack side - if: steps.cached.outputs.usable != 'true' - run: | - mkdir -p scorecard-out - "$SCORECARD" measure --family "${{ matrix.family }}" --side ratstack --checkout "$GITHUB_WORKSPACE" \ - --out "scorecard-out/ratstack-${{ matrix.family }}.json" - - id: savable - name: Check the fresh rat-stack side before caching it - if: steps.cached.outputs.usable != 'true' && github.event_name != 'pull_request' - run: | - if "$SCORECARD" cache-check --file "scorecard-out/ratstack-${{ matrix.family }}.json"; then - echo "save=true" >> "$GITHUB_OUTPUT" + selector=evals/ratstack-scorecard/ci/instrument-ref.sh + if git cat-file -e "$BASE:$selector" 2>/dev/null; then + git show "$BASE:$selector" | bash -s -- "$BASE" "$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + else + bash "$selector" "$BASE" "$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" fi - - name: Save the rat-stack side for later runs - if: steps.savable.outputs.save == 'true' - uses: actions/cache/save@v6 - with: - path: scorecard-out/ratstack-${{ matrix.family }}.json - key: ${{ matrix.cacheKey }} - - name: Measure the starter side - run: | - mkdir -p scorecard-out - "$SCORECARD" measure --family "${{ matrix.family }}" --side starter --checkout "$GITHUB_WORKSPACE" \ - --out "scorecard-out/starter-${{ matrix.family }}.json" - - uses: actions/upload-artifact@v7 - with: - name: family-${{ matrix.family }} - path: scorecard-out/ - if-no-files-found: error - - aggregate: - needs: [plan, measure] - if: ${{ !cancelled() && needs.plan.result == 'success' }} - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v7 + cat "$GITHUB_OUTPUT" - uses: actions/checkout@v7 with: - ref: ${{ env.INSTRUMENT_REF }} + ref: ${{ steps.instrument.outputs.ref }} path: .scorecard-instrument + persist-credentials: false + - name: "BOOTSTRAP: self-graded, base had no instrument" + if: steps.instrument.outputs.mode == 'bootstrap' + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + run: | + banner="# ⚠️ BOOTSTRAP: SELF-GRADED, BASE HAD NO INSTRUMENT + + The base commit has no \`.github/workflows/scorecard.yml\`, so this run grades the pull request with its own instrument. Every later pull request is graded by its base's instrument." + printf '%s\n\n' "$banner" >> "$GITHUB_STEP_SUMMARY" + gh pr comment "$PR" --body "$banner" - uses: cachix/install-nix-action@v31 - name: Allow the launcher's unprivileged user namespaces run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - - name: Build main's instrument + - name: Build the grading instrument run: echo "SCORECARD=$(nix build --no-link --print-out-paths ./.scorecard-instrument/evals/ratstack-scorecard#scorecard)/bin/scorecard" >> "$GITHUB_ENV" - - uses: actions/download-artifact@v8 + - name: Restore rat-stack results + uses: actions/cache/restore@v6 with: - pattern: family-* - merge-multiple: true - path: families - - id: main - name: Find the latest scorecard on main + path: .scorecard-cache + key: scorecard-ratstack-${{ github.run_id }} + restore-keys: scorecard-ratstack- + - name: Measure, judge, ratchet and publish env: GITHUB_TOKEN: ${{ github.token }} - run: echo "run=$("$SCORECARD" latest-main-run)" >> "$GITHUB_OUTPUT" - - name: Download main's scorecard - if: steps.main.outputs.run != '' - uses: actions/download-artifact@v8 - with: - name: scorecard - run-id: ${{ steps.main.outputs.run }} - github-token: ${{ github.token }} - path: main - - name: Judge, ratchet and publish - run: | - main_args=() - if [ -f main/scorecard.json ]; then main_args=(--main main/scorecard.json); fi - status=0 - "$SCORECARD" aggregate --families families "${main_args[@]}" --out scorecard.json --summary summary.md || status=$? - if [ -f summary.md ]; then cat summary.md >> "$GITHUB_STEP_SUMMARY"; fi - exit "$status" + run: '"$SCORECARD" ci --checkout "$GITHUB_WORKSPACE" --cache .scorecard-cache --out scorecard-out' - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: name: scorecard - path: scorecard.json + path: scorecard-out/scorecard.json if-no-files-found: error + - name: Save rat-stack results + if: ${{ !cancelled() && github.event_name != 'pull_request' }} + uses: actions/cache/save@v6 + with: + path: .scorecard-cache + key: scorecard-ratstack-${{ github.run_id }} instrument-preview: name: instrument preview (informational) - needs: aggregate - if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.aggregate.result != 'skipped' }} + needs: scorecard + if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.scorecard.outputs.mode == 'base' }} continue-on-error: true runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 360 steps: - uses: actions/checkout@v7 with: @@ -185,25 +121,14 @@ jobs: with: name: scorecard path: graded - - name: Measure with this PR's instrument and list what it would re-baseline + - name: Grade with this PR's own instrument against the graded scorecard if: steps.changed.outputs.edits == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} run: | scorecard="$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#scorecard)/bin/scorecard" - mkdir -p preview - for family in $("$scorecard" plan | jq -r '.include[].family'); do - "$scorecard" measure --family "$family" --out "preview/$family.json" - done - status=0 - "$scorecard" aggregate --families preview --main graded/scorecard.json \ - --out preview-scorecard.json --summary preview-summary.md || status=$? - { - echo "## Instrument preview (does not gate)" - echo - echo "This PR's own instrument, compared with the scorecard main's instrument produced for it. Re-baselined rows are listed under definition drift." - echo - cat preview-summary.md - } >> "$GITHUB_STEP_SUMMARY" - exit "$status" + "$scorecard" ci --checkout "$GITHUB_WORKSPACE" --cache .scorecard-cache --out preview \ + --main graded/scorecard.json --title "Instrument preview (does not gate)" pin: if: github.event_name != 'pull_request' diff --git a/evals/ratstack-scorecard/README.md b/evals/ratstack-scorecard/README.md index 8107402..db11064 100644 --- a/evals/ratstack-scorecard/README.md +++ b/evals/ratstack-scorecard/README.md @@ -33,7 +33,7 @@ The instrument's own checks run with one command: `scorecard check` (also `pnpm The program is split in two: - **The host driver** (`src/main.ts` and what it imports) uses only Deno APIs, `node:` builtins and its own files, so `DENO_NO_PACKAGE_JSON=1 deno check src/` type-checks it without third-party code. It starts launcher invocations, mounts their inputs read-only, collects output files and exit codes, and writes the pin file. It decodes and judges nothing. Its only network access is through the launcher. -- **The decide step** (`decide/`, Effect 4 with Schema and `effect/http`) runs as its own launcher invocation: `aggregate` (decode the family results and `main`'s scorecard, judge, ratchet, render), `cache-check`, `latest-main-run` and `pin-check`. Only the last two may reach `api.github.com`; GitHub calls time out after 15 s, retry transient failures three times, and fail as a typed `GithubApiError`. Pass or fail is the step's exit code. `pnpm exec tsc` type-checks `decide/` and `journeys/` inside the launcher. +- **The decide step** (`decide/`, Effect 4 with Schema and `effect/http`) runs as its own launcher invocation from a copy of the grading instrument: `aggregate` (decode the family results and `main`'s scorecard, judge, ratchet, render), `cache-check`, `main-scorecard` and `pin-check`. Only the last two reach the network, and only `api.github.com` and the artifact store (`*.blob.core.windows.net`); GitHub calls time out after 15 s, retry transient failures three times, and fail as a typed `GithubApiError`. Pass or fail is the step's exit code. `pnpm exec tsc` type-checks `decide/` and `journeys/` inside the launcher. Journeys run in two phases, because the launcher cannot nest. `scorecard journeys` produces every journey declared in `journeys/manifest.json` through the real launcher work and records it to `journeys/__records__/.json`, then runs the one vitest project inside the launcher. A test is a journey because it imports `journeys/launcher-run.ts`; a missing or stale record fails it red. @@ -48,12 +48,11 @@ The Node scripts in `src/tools/` and everything in `decide/` load npm packages, `.github/workflows/scorecard.yml` runs on every PR to `main`, every push to `main`, daily, and on demand. All jobs run on GitHub-hosted runners (`ubuntu-latest`): the self-hosted fleet excludes public repositories by design. -On a pull request every grading job builds the instrument from the base commit's tree, and only the measured starter comes from the PR, so a PR that edits `evals/ratstack-scorecard/**` cannot grade itself. A definition hash covers only a row's registry entry and its family's measurement code; harness, sides, tools, lockfiles and nixpkgs move the rat-stack cache key instead. +The workflow is a thin caller: it chooses the grading instrument, builds it, and runs `scorecard ci`, the one stable entrypoint. Measuring every family, the rat-stack cache, finding `main`'s scorecard, judging, the ratchet and the job summary all happen inside the instrument, so a pull request that adds a step never needs its base to know that step; the step starts grading after it merges. -- **plan** prints the matrix: one entry per built family, with its timeout and the rat-stack cache key (rat-stack commit, instrument tree hash, nixpkgs rev). -- **measure** runs one family per job. The rat-stack side is restored from the cache when the key matches and `scorecard cache-check` accepts it; an entry holding a rat-stack instrument error is refused and measured again. Only `main`, the schedule and manual runs save it, and only when the fresh result holds no instrument error. The starter side is measured every run. Each job uploads `family-`. -- **aggregate** joins the families, compares them with the latest successful `main` run's `scorecard` artifact (`scorecard latest-main-run`; none yet means a first baseline), writes the Markdown table to the job summary, uploads `scorecard.json`, and fails when the ratchet fails. Every re-baselined row is listed under definition drift with both hashes. -- **instrument preview** (PRs that edit the instrument, never gates) builds the PR's own instrument, measures every family and lists the rows it would re-baseline against the graded scorecard. +- **Which instrument grades.** Pushes, the schedule and manual runs use their own commit. A pull request is graded by its base commit's instrument, chosen by `ci/instrument-ref.sh` (run from the base tree whenever the base has it). Only when the base tree has no `.github/workflows/scorecard.yml` at all is the PR head's instrument used; the job summary and a PR comment then say **BOOTSTRAP: self-graded, base had no instrument**. A base commit missing from the clone fails the job instead of falling back to bootstrap. The `instrument-ref` journey proves it on real git history: with the workflow on the base, a PR that edits the instrument to flip a verdict, deletes the instrument, or deletes the workflow is still graded by the base. +- **scorecard** builds the chosen instrument and runs `scorecard ci`. The rat-stack side comes from the cache when its key (rat-stack commit, instrument content hash, nixpkgs rev) matches and `cache-check` accepts it; an entry holding a rat-stack instrument error is refused and measured again. Only pushes, the schedule and manual runs save the cache, only results without instrument errors, and stale keys are pruned. The starter side is measured every run. `scorecard ci` compares with the latest successful `main` run's `scorecard` artifact (none yet means a first baseline), writes the table to the job summary, uploads `scorecard.json`, and fails when the ratchet fails. Re-baselined rows are listed under definition drift with both hashes. A definition hash covers only a row's registry entry and its family's measurement code. +- **instrument preview** (PRs graded by their base that edit the instrument; never gates) runs the PR's own `scorecard ci` against the graded scorecard and lists what it would re-baseline. - **pin** (not on PRs) compares `ratstack.pin.json` with rat-stack `main`, decoded as a 40-hex SHA. When rat-stack has moved it opens or updates the `scorecard/pin-rat-stack` PR through the pin-bump GitHub App (`vars.SCORECARD_APP_ID`, `secrets.SCORECARD_APP_PRIVATE_KEY`; contents and pull requests write only). The branch is bot-owned: the job refuses to replace a tip another author wrote and pushes with a lease pinned to the tip it read. The token reaches git through a credential helper that reads it from the environment. That PR changes only the pin; Kiro merges it. `actionlint` is in the dev shell: `nix develop ./evals/ratstack-scorecard --command actionlint .github/workflows/scorecard.yml`. diff --git a/evals/ratstack-scorecard/ci/instrument-ref.sh b/evals/ratstack-scorecard/ci/instrument-ref.sh new file mode 100755 index 0000000..900ab59 --- /dev/null +++ b/evals/ratstack-scorecard/ci/instrument-ref.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "usage: instrument-ref.sh " >&2 + exit 2 +fi +base=$1 +head=$2 + +git cat-file -e "${base}^{commit}" 2>/dev/null || { + echo "base commit ${base} is not in this clone; fetch it before choosing an instrument" >&2 + exit 1 +} +git cat-file -e "${head}^{commit}" 2>/dev/null || { + echo "head commit ${head} is not in this clone" >&2 + exit 1 +} + +if git cat-file -e "${base}:.github/workflows/scorecard.yml" 2>/dev/null; then + echo "ref=${base}" + echo "mode=base" +else + echo "ref=${head}" + echo "mode=bootstrap" +fi diff --git a/evals/ratstack-scorecard/decide/main.ts b/evals/ratstack-scorecard/decide/main.ts index 3d424b9..36954f5 100644 --- a/evals/ratstack-scorecard/decide/main.ts +++ b/evals/ratstack-scorecard/decide/main.ts @@ -1,14 +1,21 @@ import { Cause, Duration, Effect, Schedule, Schema } from 'effect' -import { FetchHttpClient, HttpClient, HttpClientRequest } from 'effect/http' -import { readdir, readFile, writeFile } from 'node:fs/promises' -import { join } from 'node:path' +import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from 'effect/http' +import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' import { parseArgs } from 'node:util' +import { inflateRawSync } from 'node:zlib' import { rowDefinitions } from '../src/metrics/registry.ts' import type { Family, FamilyResult, MainBaseline, MeasuredCell, Side, SideCell } from '../src/model/cell.ts' import { planPinBump } from '../src/model/plan-pin-bump.workflow.ts' import { assembleScorecard, type CellsByRow } from '../src/model/scorecard-document.ts' import { renderSummary } from '../src/model/summary-table.ts' -import { FamilyResultSchema, GitRefSchema, ScorecardDocumentSchema, WorkflowRunsSchema } from './schema.ts' +import { + ArtifactsSchema, + FamilyResultSchema, + GitRefSchema, + ScorecardDocumentSchema, + WorkflowRunsSchema, +} from './schema.ts' class MalformedInput extends Schema.TaggedError()('MalformedInput', { what: Schema.String, @@ -39,7 +46,11 @@ const readText = (path: string) => catch: (error) => new MalformedInput({ what: path, issue: String(error) }), }) -const writeText = (path: string, text: string) => Effect.promise(() => writeFile(path, text)) +const writeText = (path: string, text: string) => + Effect.promise(async () => { + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, text) + }) const decodeFile = (schema: Schema.Codec, path: string) => readText(path).pipe( @@ -137,39 +148,105 @@ const cacheCheck = (args: readonly string[]) => } }) -const github = (path: string) => +const githubGet = (path: string) => Effect.gen(function*() { const client = (yield* HttpClient.HttpClient).pipe( - HttpClient.filterStatusOk, HttpClient.retryTransient({ schedule: Schedule.exponential(Duration.seconds(1)), times: 3 }), ) const token = process.env['GITHUB_TOKEN'] - const request = HttpClientRequest.get(`https://api.github.com${path}`).pipe( - HttpClientRequest.acceptJson, + const request = HttpClientRequest.get(path.startsWith('https://') ? path : `https://api.github.com${path}`).pipe( (r) => token === undefined || token === '' ? r : HttpClientRequest.bearerToken(r, token), ) - return yield* client.execute(request).pipe( - Effect.flatMap((response) => response.json), - Effect.timeout(Duration.seconds(15)), - ) + return yield* client.execute(request).pipe(Effect.timeout(Duration.seconds(15))) }).pipe( Effect.mapError((error) => new GithubApiError({ request: `GET ${path}`, cause: String(error) })), Effect.provide(FetchHttpClient.layer), ) +const ok = (path: string) => +( + response: HttpClientResponse.HttpClientResponse, +): Effect.Effect => + response.status >= 200 && response.status < 300 + ? Effect.succeed(response) + : Effect.fail(new GithubApiError({ request: `GET ${path}`, cause: `status ${response.status}` })) + +const github = (path: string) => + githubGet(path).pipe( + Effect.flatMap(ok(path)), + Effect.flatMap((response) => response.json), + Effect.mapError((error) => new GithubApiError({ request: `GET ${path}`, cause: String(error) })), + ) + const decodeBody = (schema: Schema.Codec, what: string) => (body: unknown) => Schema.decodeUnknownEffect(schema)(body).pipe( Effect.mapError((error) => new GithubApiError({ request: what, cause: String(error) })), ) -const latestMainRun = (args: readonly string[]) => +const zipEntry = (zip: Buffer, name: string) => + Effect.try({ + try: () => { + const end = zip.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06])) + const count = zip.readUInt16LE(end + 10) + let entry = zip.readUInt32LE(end + 16) + for (let index = 0; index < count; index++) { + const nameLength = zip.readUInt16LE(entry + 28) + const extraLength = zip.readUInt16LE(entry + 30) + const commentLength = zip.readUInt16LE(entry + 32) + if (zip.toString('utf8', entry + 46, entry + 46 + nameLength) === name) { + const method = zip.readUInt16LE(entry + 10) + const size = zip.readUInt32LE(entry + 20) + const local = zip.readUInt32LE(entry + 42) + const start = local + 30 + zip.readUInt16LE(local + 26) + zip.readUInt16LE(local + 28) + const data = zip.subarray(start, start + size) + return (method === 8 ? inflateRawSync(data) : data).toString('utf8') + } + entry += 46 + nameLength + extraLength + commentLength + } + throw new Error(`no ${name} in the archive`) + }, + catch: (error) => new GithubApiError({ request: `unzip ${name}`, cause: String(error) }), + }) + +const mainScorecard = (args: readonly string[]) => Effect.gen(function*() { - const { values } = parseArgs({ args: [...args], options: { repository: { type: 'string' } }, strict: true }) + const { values } = parseArgs({ + args: [...args], + options: { repository: { type: 'string' }, out: { type: 'string' } }, + strict: true, + }) const repository = yield* required(values.repository, '--repository') - const path = + const out = yield* required(values.out, '--out') + const runsPath = `/repos/${repository}/actions/workflows/scorecard.yml/runs?branch=main&event=push&status=success&per_page=1` - const runs = yield* github(path).pipe(Effect.flatMap(decodeBody(WorkflowRunsSchema, `GET ${path}`))) - yield* Effect.sync(() => process.stdout.write(`${runs.workflow_runs.map((run) => run.id).join('')}\n`)) + const listing = yield* githubGet(runsPath) + if (listing.status === 404) return yield* Effect.log('main has no scorecard workflow yet: first baseline') + const runs = yield* ok(runsPath)(listing).pipe( + Effect.flatMap((response) => response.json), + Effect.mapError((error) => new GithubApiError({ request: `GET ${runsPath}`, cause: String(error) })), + Effect.flatMap(decodeBody(WorkflowRunsSchema, `GET ${runsPath}`)), + ) + const run = runs.workflow_runs[0] + if (run === undefined) return yield* Effect.log('no successful scorecard run on main: first baseline') + const artifactsPath = `/repos/${repository}/actions/runs/${run.id}/artifacts?name=scorecard` + const artifacts = yield* github(artifactsPath).pipe(Effect.flatMap(decodeBody(ArtifactsSchema, artifactsPath))) + const artifact = artifacts.artifacts[0] + if (artifact === undefined) { + return yield* new GithubApiError({ request: artifactsPath, cause: `run ${run.id} has no scorecard artifact` }) + } + const zip = yield* githubGet(artifact.archive_download_url).pipe( + Effect.flatMap(ok(artifact.archive_download_url)), + Effect.flatMap((response) => response.arrayBuffer), + Effect.mapError((error) => new GithubApiError({ request: 'artifact download', cause: String(error) })), + ) + const text = yield* zipEntry(Buffer.from(zip), 'scorecard.json') + yield* Schema.decodeUnknownEffect(Schema.fromJsonString(ScorecardDocumentSchema))(text).pipe( + Effect.mapError((error) => + new MalformedInput({ what: `main's scorecard (run ${run.id})`, issue: String(error) }) + ), + ) + yield* writeText(out, text) + yield* Effect.log(`main's scorecard from run ${run.id} (${run.head_sha})`) }) const pinCheck = (args: readonly string[]) => @@ -196,7 +273,7 @@ const required = (value: string | undefined, flag: string) => const commands: Readonly Effect.Effect>> = { 'aggregate': aggregate, 'cache-check': cacheCheck, - 'latest-main-run': latestMainRun, + 'main-scorecard': mainScorecard, 'pin-check': pinCheck, } diff --git a/evals/ratstack-scorecard/decide/schema.ts b/evals/ratstack-scorecard/decide/schema.ts index 8b9437c..11e5210 100644 --- a/evals/ratstack-scorecard/decide/schema.ts +++ b/evals/ratstack-scorecard/decide/schema.ts @@ -144,3 +144,7 @@ export const WorkflowRunsSchema = Schema.Struct({ }) export const GitRefSchema = Schema.Struct({ object: Schema.Struct({ sha: Sha }) }) + +export const ArtifactsSchema = Schema.Struct({ + artifacts: Schema.Array(Schema.Struct({ id: Schema.Finite, archive_download_url: Schema.String })), +}) diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index e9e02e1..955962f 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -39,7 +39,7 @@ }; scorecard = pkgs.writeShellApplication { name = "scorecard"; - runtimeInputs = [ pkgs.deno pkgs.git ]; + runtimeInputs = [ pkgs.deno pkgs.git pkgs.bash ]; text = '' export SCORECARD_INSTRUMENT=${self} export SCORECARD_SANDBOX=${sandbox}/bin/sandbox @@ -50,7 +50,7 @@ export SCORECARD_PNPM_VERSION=${pkgs.pnpm_12.version} export DENO_NO_PACKAGE_JSON=1 exec deno run --no-config --allow-read --allow-write --allow-env --allow-sys=hostname \ - --allow-run=git,${pkgs.deno}/bin/deno,${sandbox}/bin/sandbox \ + --allow-run=git,bash,${pkgs.deno}/bin/deno,${sandbox}/bin/sandbox \ ${self}/src/main.ts "$@" ''; }; diff --git a/evals/ratstack-scorecard/journeys/instrument-ref.test.ts b/evals/ratstack-scorecard/journeys/instrument-ref.test.ts new file mode 100644 index 0000000..99a1ee7 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/instrument-ref.test.ts @@ -0,0 +1,42 @@ +import { Effect, Schema } from 'effect' +import { describe, expect, test } from 'vitest' +import { launcherRun } from './launcher-run.ts' + +const Selection = Schema.Struct({ + code: Schema.Number, + stdout: Schema.String, + stderr: Schema.String, + base: Schema.String, + head: Schema.String, + graded: Schema.NullOr(Schema.String), +}) + +const selection = async (name: string) => { + const run = await launcherRun('instrument-ref') + return Effect.runPromise(Schema.decodeUnknownEffect(Schema.fromJsonString(Selection))(run.files[`${name}.json`])) +} + +describe('which instrument grades a pull request (bootstrap ruling)', () => { + test('a base without the scorecard workflow is the only case that grades with the PR head, and says BOOTSTRAP', async () => { + const s = await selection('bootstrap') + expect(s.code).toBe(0) + expect(s.stdout).toBe(`ref=${s.head}\nmode=bootstrap\n`) + }) + + test.each(['edits-instrument', 'deletes-instrument', 'deletes-workflow'])( + 'when the base has the workflow, a PR that %s is still graded by the base instrument, so the base verdict (red) stands', + async (name) => { + const s = await selection(name) + expect(s.code).toBe(0) + expect(s.stdout).toBe(`ref=${s.base}\nmode=base\n`) + expect(s.graded).toBe('red') + }, + ) + + test('a base commit missing from the clone fails instead of falling back to bootstrap', async () => { + const s = await selection('base-missing') + expect(s.code).toBe(1) + expect(s.stdout).toBe('') + expect(s.stderr).toContain('is not in this clone') + }) +}) diff --git a/evals/ratstack-scorecard/journeys/manifest.json b/evals/ratstack-scorecard/journeys/manifest.json index e8a38a1..69e59ee 100644 --- a/evals/ratstack-scorecard/journeys/manifest.json +++ b/evals/ratstack-scorecard/journeys/manifest.json @@ -55,6 +55,16 @@ "src", "decide" ] + }, + { + "id": "instrument-ref", + "produce": { + "kind": "instrument-ref" + }, + "inputs": [ + "ci/instrument-ref.sh", + "src/journeys.ts" + ] } ] } diff --git a/evals/ratstack-scorecard/src/harness/decide.ts b/evals/ratstack-scorecard/src/harness/decide.ts new file mode 100644 index 0000000..a45b4e8 --- /dev/null +++ b/evals/ratstack-scorecard/src/harness/decide.ts @@ -0,0 +1,49 @@ +import { dirname, join } from 'node:path' +import { type Instrument, walkFiles } from './instrument.ts' +import { runSandboxed } from './sandbox.ts' + +const generated = ['node_modules/', '.cache/', 'journeys/__records__/'] + +const githubHosts = ['api.github.com', '*.blob.core.windows.net'] + +export interface DecideWork { + readonly instrument: Instrument + readonly work: string +} + +export const prepareDecide = async (instrument: Instrument, work: string): Promise => { + const copy = join(work, 'instrument') + for (const file of await walkFiles(instrument.dir)) { + if (generated.some((prefix) => file.startsWith(prefix))) continue + await Deno.mkdir(dirname(join(copy, file)), { recursive: true }) + await Deno.writeFile(join(copy, file), await Deno.readFile(join(instrument.dir, file))) + } + const install = await runSandboxed(instrument.launcher, { + project: work, + cwd: copy, + command: ['pnpm', 'install', '--frozen-lockfile'], + pnpmStore: instrument.toolsStore, + deadlineMs: 10 * 60_000, + }) + if (install.code !== 0) throw new Error(`installing the decide step's dependencies failed:\n${install.stderr}`) + return { instrument, work } +} + +export const decide = async ( + { instrument, work }: DecideWork, + args: readonly string[], + options: { readonly github: boolean }, +): Promise => { + const token = Deno.env.get('GITHUB_TOKEN') + const result = await runSandboxed(instrument.launcher, { + project: work, + cwd: work, + command: ['node', 'instrument/decide/main.ts', ...args], + ...(options.github ? { allowHosts: githubHosts } : {}), + ...(options.github && token !== undefined ? { env: { GITHUB_TOKEN: token } } : {}), + deadlineMs: 5 * 60_000, + }) + await Deno.stdout.write(new TextEncoder().encode(result.stdout)) + await Deno.stderr.write(new TextEncoder().encode(result.stderr)) + return result.code +} diff --git a/evals/ratstack-scorecard/src/harness/instrument.ts b/evals/ratstack-scorecard/src/harness/instrument.ts index 5e311cf..eaad545 100644 --- a/evals/ratstack-scorecard/src/harness/instrument.ts +++ b/evals/ratstack-scorecard/src/harness/instrument.ts @@ -53,6 +53,23 @@ const pinFile = struct({ owner: string, repo: string, commit: string, narHash: s const manifest = struct({ dependencies: record(string) }) +const sha256Bytes = async (bytes: BufferSource): Promise => + [...new Uint8Array(await crypto.subtle.digest('SHA-256', bytes))] + .map((byte) => byte.toString(16).padStart(2, '0')) + .join('') + +const sha256 = (text: string): Promise => sha256Bytes(new TextEncoder().encode(text)) + +const generated = ['node_modules/', '.cache/', 'journeys/__records__/'] + +const instrumentTreeHash = async (dir: string): Promise => { + const files = (await walkFiles(dir)).filter((file) => !generated.some((prefix) => file.startsWith(prefix))).sort() + const parts = await Promise.all( + files.map(async (file) => `${file}\0${await sha256Bytes(await Deno.readFile(join(dir, file)))}`), + ) + return sha256(parts.join('\n')) +} + export const loadInstrument = async (checkoutArg: string | undefined): Promise => { const dir = required('SCORECARD_INSTRUMENT') const checkout = await gitText(checkoutArg ?? Deno.cwd(), ['rev-parse', '--show-toplevel']) @@ -63,7 +80,7 @@ export const loadInstrument = async (checkoutArg: string | undefined): Promise => - [...new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(text)))] - .map((byte) => byte.toString(16).padStart(2, '0')) - .join('') - export const definitionHashesFor = async ( instrument: Instrument, family: Family, diff --git a/evals/ratstack-scorecard/src/journeys.ts b/evals/ratstack-scorecard/src/journeys.ts index cb86944..55f8184 100644 --- a/evals/ratstack-scorecard/src/journeys.ts +++ b/evals/ratstack-scorecard/src/journeys.ts @@ -13,10 +13,14 @@ interface MeasureStatic { interface JourneyEntry { readonly id: string - readonly produce: MeasureStatic | Aggregate + readonly produce: MeasureStatic | Aggregate | InstrumentRef readonly inputs: readonly string[] } +interface InstrumentRef { + readonly kind: 'instrument-ref' +} + export interface LauncherRecord { readonly id: string readonly inputHash: string @@ -31,7 +35,7 @@ export interface LauncherRecord { const journeyEntry: Decoder = struct({ id: string, - produce: union( + produce: union( struct({ kind: literal('measure-static'), ratstackRepo: string, @@ -39,6 +43,7 @@ const journeyEntry: Decoder = struct({ failingTool: nullable(string), }), struct({ kind: literal('aggregate'), families: string, main: nullable(string) }), + struct({ kind: literal('instrument-ref') }), ), inputs: array(string), }) @@ -195,10 +200,92 @@ const produceStatic = async ( } } +const identity = ['-c', 'user.name=journey', '-c', 'user.email=journey@invalid', '-c', 'commit.gpgsign=false'] + +const commitTree = async (repo: string, files: Readonly>, message: string) => { + for (const [path, content] of Object.entries(files)) { + if (content === null) await Deno.remove(join(repo, path), { recursive: true }) + else { + await Deno.mkdir(join(repo, path, '..'), { recursive: true }) + await Deno.writeTextFile(join(repo, path), content) + } + } + await git(repo, ['add', '-A']) + await git(repo, [...identity, 'commit', '-q', '--allow-empty', '-m', message]) + return new TextDecoder().decode(await git(repo, ['rev-parse', 'HEAD'])).trim() +} + +const produceInstrumentRef = async (root: string, entry: JourneyEntry): Promise => { + const repo = await Deno.makeTempDir({ prefix: `journey-${entry.id}-` }) + const started = performance.now() + try { + await git(repo, ['init', '-q', '-b', 'main']) + const workflow = '.github/workflows/scorecard.yml' + const verdict = 'evals/ratstack-scorecard/verdict' + const baseWithout = await commitTree(repo, { 'README.md': 'starter\n' }, 'base without the scorecard') + const introduce = await commitTree(repo, { [workflow]: 'name: Scorecard\n', [verdict]: 'green\n' }, 'add scorecard') + const baseWith = await commitTree(repo, { [verdict]: 'red\n' }, 'main instrument grades the row red') + const editsInstrument = await commitTree(repo, { [verdict]: 'green\n' }, 'PR edits the instrument to green') + await git(repo, ['checkout', '-q', baseWith]) + const deletesInstrument = await commitTree(repo, { 'evals/ratstack-scorecard': null }, 'PR deletes the instrument') + await git(repo, ['checkout', '-q', baseWith]) + const deletesWorkflow = await commitTree( + repo, + { [workflow]: null, [verdict]: 'green\n' }, + 'PR deletes the workflow', + ) + const cases: Readonly> = { + 'bootstrap': [baseWithout, introduce], + 'edits-instrument': [baseWith, editsInstrument], + 'deletes-instrument': [baseWith, deletesInstrument], + 'deletes-workflow': [baseWith, deletesWorkflow], + 'base-missing': ['0'.repeat(40), editsInstrument], + } + const files: Record = {} + for (const [name, [base, head]] of Object.entries(cases)) { + const run = await new Deno.Command('bash', { + args: [join(root, 'ci/instrument-ref.sh'), base, head], + cwd: repo, + stdout: 'piped', + stderr: 'piped', + }).output() + const stdout = new TextDecoder().decode(run.stdout) + const ref = /^ref=(\S+)$/mu.exec(stdout)?.[1] + const show = ref === undefined + ? undefined + : await new Deno.Command('git', { args: ['show', `${ref}:${verdict}`], cwd: repo, stdout: 'piped' }).output() + files[`${name}.json`] = JSON.stringify({ + code: run.code, + stdout, + stderr: new TextDecoder().decode(run.stderr), + base, + head, + graded: show?.success === true ? new TextDecoder().decode(show.stdout).trim() : null, + }) + } + return { + id: entry.id, + inputHash: '', + argv: ['bash', 'ci/instrument-ref.sh', '', ''], + code: 0, + stdout: '', + stderr: '', + files, + egressLog: null, + wallMs: Math.round(performance.now() - started), + } + } finally { + await Deno.remove(repo, { recursive: true }) + } +} + const produce = async (instrument: Instrument, root: string, entry: JourneyEntry): Promise => { - const record = entry.produce.kind === 'aggregate' - ? await produceAggregate(instrument, root, entry, entry.produce) - : await produceStatic(instrument, root, entry, entry.produce) + const spec = entry.produce + const record = spec.kind === 'aggregate' + ? await produceAggregate(instrument, root, entry, spec) + : spec.kind === 'instrument-ref' + ? await produceInstrumentRef(root, entry) + : await produceStatic(instrument, root, entry, spec) return { ...record, inputHash: await inputHashOf(root, entry.inputs, instrument.launcher.executable) } } diff --git a/evals/ratstack-scorecard/src/main.ts b/evals/ratstack-scorecard/src/main.ts index 1f8e339..92e201b 100644 --- a/evals/ratstack-scorecard/src/main.ts +++ b/evals/ratstack-scorecard/src/main.ts @@ -2,20 +2,17 @@ import { join } from 'node:path' import { parseArgs } from 'node:util' import { checkImports } from './check-imports.ts' import { measureStatic } from './families/static.ts' -import { type Instrument, loadInstrument } from './harness/instrument.ts' -import { runSandboxed } from './harness/sandbox.ts' +import { decide, type DecideWork, prepareDecide } from './harness/decide.ts' +import { type Instrument, loadInstrument, walkFiles } from './harness/instrument.ts' import { runJourneys } from './journeys.ts' -import { familyTimeoutMinutes } from './metrics/registry.ts' import { ratstackCacheKey } from './model/cache-key.ts' import type { Family, FamilyResult, Side } from './model/cell.ts' const usage = [ 'usage:', - ' scorecard plan', + ' scorecard ci [--checkout ] --cache --out [--main ] [--title ]', ' scorecard measure --family [--side ratstack|starter] [--out ] [--checkout ]', ' scorecard aggregate --families [--main ] --out --summary ', - ' scorecard cache-check --file ', - ' scorecard latest-main-run', ' scorecard pin check', ' scorecard pin write --commit --nar-hash [--checkout ]', ' scorecard journeys [--checkout ]', @@ -43,21 +40,6 @@ const sidesOf = (side: string | undefined): readonly Side[] => { return fail(`unknown side ${side}`) } -const plan = async (): Promise => { - const instrument = await loadInstrument(undefined) - const matrix = implementedFamilies.map((family) => ({ - family, - timeoutMinutes: familyTimeoutMinutes[family], - cacheKey: ratstackCacheKey({ - family, - ratstackCommit: instrument.pin.commit, - instrumentHash: instrument.instrumentHash, - nixpkgsRev: instrument.nixpkgsRev, - }), - })) - await writeOut(undefined, `${JSON.stringify({ include: matrix })}\n`) -} - const measure = async (args: readonly string[]): Promise => { const { values } = parseArgs({ args: [...args], @@ -98,52 +80,156 @@ const check = async (args: readonly string[]): Promise => { Deno.exit(await runJourneys(instrument, root)) } -const decide = async ( - instrument: Instrument, - args: readonly string[], - options: { readonly github: boolean }, -): Promise => { - const root = join(instrument.checkout, 'evals/ratstack-scorecard') - const install = await runSandboxed(instrument.launcher, { - project: instrument.checkout, - cwd: root, - command: ['pnpm', 'install', '--frozen-lockfile'], - pnpmStore: instrument.toolsStore, - deadlineMs: 10 * 60_000, - }) - if (install.code !== 0) fail(`installing the decide step's dependencies failed:\n${install.stderr}`) - const token = Deno.env.get('GITHUB_TOKEN') - const result = await runSandboxed(instrument.launcher, { - project: instrument.checkout, - cwd: Deno.cwd(), - command: ['node', join(root, 'decide/main.ts'), ...args], - ...(options.github ? { allowHosts: ['api.github.com'] } : {}), - ...(options.github && token !== undefined ? { env: { GITHUB_TOKEN: token } } : {}), - deadlineMs: 5 * 60_000, - }) - await Deno.stdout.write(new TextEncoder().encode(result.stdout)) - await Deno.stderr.write(new TextEncoder().encode(result.stderr)) - return result.code +const exists = (path: string): Promise => Deno.stat(path).then(() => true, () => false) + +const copyInto = async (from: string, to: string): Promise => { + await Deno.mkdir(join(to, '..'), { recursive: true }) + await Deno.copyFile(from, to) } +const withDecide = async (instrument: Instrument, run: (work: DecideWork) => Promise): Promise => { + const work = await Deno.makeTempDir({ prefix: 'scorecard-decide-' }) + try { + return await run(await prepareDecide(instrument, work)) + } finally { + await Deno.remove(work, { recursive: true }) + } +} + +const aggregateIn = (work: DecideWork, instrument: Instrument, main: boolean): Promise => + decide(work, [ + 'aggregate', + '--families', + 'families', + '--implemented', + implementedFamilies.join(','), + '--commit', + Deno.env.get('GITHUB_SHA') ?? instrument.starterCommit, + ...(main ? ['--main', 'main/scorecard.json'] : []), + '--out', + 'out/scorecard.json', + '--summary', + 'out/summary.md', + ], { github: false }) + const aggregate = async (args: readonly string[]): Promise => { + const { values } = parseArgs({ + args: [...args], + options: { + families: { type: 'string' }, + main: { type: 'string' }, + out: { type: 'string' }, + summary: { type: 'string' }, + }, + strict: true, + }) + const families = values.families ?? fail('--families is required') + const out = values.out ?? fail('--out is required') + const summary = values.summary ?? fail('--summary is required') const instrument = await loadInstrument(undefined) - const commit = ['--commit', Deno.env.get('GITHUB_SHA') ?? instrument.starterCommit] Deno.exit( - await decide(instrument, ['aggregate', '--implemented', implementedFamilies.join(','), ...commit, ...args], { - github: false, + await withDecide(instrument, async (work) => { + for (const file of await walkFiles(families)) { + await copyInto(join(families, file), join(work.work, 'families', file)) + } + if (values.main !== undefined) await copyInto(values.main, join(work.work, 'main/scorecard.json')) + const code = await aggregateIn(work, instrument, values.main !== undefined) + if (await exists(join(work.work, 'out/scorecard.json'))) { + await copyInto(join(work.work, 'out/scorecard.json'), out) + } + if (await exists(join(work.work, 'out/summary.md'))) await copyInto(join(work.work, 'out/summary.md'), summary) + return code }), ) } -const cacheCheck = async (args: readonly string[]): Promise => - Deno.exit(await decide(await loadInstrument(undefined), ['cache-check', ...args], { github: false })) +const ci = async (args: readonly string[]): Promise => { + const { values } = parseArgs({ + args: [...args], + options: { + checkout: { type: 'string' }, + cache: { type: 'string' }, + out: { type: 'string' }, + main: { type: 'string' }, + title: { type: 'string' }, + }, + strict: true, + }) + const cache = values.cache ?? fail('--cache is required') + const out = values.out ?? fail('--out is required') + const pullRequest = Deno.env.get('GITHUB_EVENT_NAME') === 'pull_request' + const instrument = await loadInstrument(values.checkout) + await Deno.mkdir(cache, { recursive: true }) + const code = await withDecide(instrument, async (work) => { + const families = join(work.work, 'families') + await Deno.mkdir(families, { recursive: true }) + const keys: string[] = [] + for (const [family, run] of familyRunners) { + const key = ratstackCacheKey({ + family, + ratstackCommit: instrument.pin.commit, + instrumentHash: instrument.instrumentHash, + nixpkgsRev: instrument.nixpkgsRev, + }).replaceAll('/', '__') + keys.push(`${key}.json`) + const cached = join(cache, `${key}.json`) + const ratstackFile = join(families, `ratstack-${family}.json`) + const usable = await exists(cached) && + (await copyInto(cached, ratstackFile), + (await decide(work, ['cache-check', '--file', `families/ratstack-${family}.json`], { github: false })) === 0) + if (!usable) { + if (await exists(cached)) console.error(`ci: cached rat-stack ${family} side refused; measuring it again`) + await measureInto(instrument, run, family, ['ratstack'], ratstackFile) + const savable = !pullRequest && + (await decide(work, ['cache-check', '--file', `families/ratstack-${family}.json`], { github: false })) === 0 + if (savable) await copyInto(ratstackFile, cached) + } + await measureInto(instrument, run, family, ['starter'], join(families, `starter-${family}.json`)) + } + if (!pullRequest) { + for (const file of await walkFiles(cache)) if (!keys.includes(file)) await Deno.remove(join(cache, file)) + } + const main = values.main !== undefined + ? (await copyInto(values.main, join(work.work, 'main/scorecard.json')), true) + : (await Deno.mkdir(join(work.work, 'main'), { recursive: true }), + (await decide(work, [ + 'main-scorecard', + '--repository', + Deno.env.get('GITHUB_REPOSITORY') ?? fail('GITHUB_REPOSITORY is unset'), + '--out', + 'main/scorecard.json', + ], { github: true })) === 0 || fail("finding main's scorecard failed"), + await exists(join(work.work, 'main/scorecard.json'))) + const verdict = await aggregateIn(work, instrument, main) + await Deno.mkdir(out, { recursive: true }) + for (const file of ['scorecard.json', 'summary.md']) { + if (await exists(join(work.work, 'out', file))) await copyInto(join(work.work, 'out', file), join(out, file)) + } + return verdict + }) + const summaryFile = Deno.env.get('GITHUB_STEP_SUMMARY') + if (summaryFile !== undefined && await exists(join(out, 'summary.md'))) { + const title = values.title === undefined ? '' : `## ${values.title}\n\n` + await Deno.writeTextFile(summaryFile, `${title}${await Deno.readTextFile(join(out, 'summary.md'))}\n`, { + append: true, + }) + } + Deno.exit(code) +} -const latestMainRun = async (): Promise => { - const repository = Deno.env.get('GITHUB_REPOSITORY') ?? fail('GITHUB_REPOSITORY is unset') - Deno.exit( - await decide(await loadInstrument(undefined), ['latest-main-run', '--repository', repository], { github: true }), - ) +const measureInto = async ( + instrument: Instrument, + run: FamilyRunner, + family: Family, + sides: readonly Side[], + file: string, +): Promise => { + const work = await Deno.makeTempDir({ prefix: `scorecard-${family}-` }) + try { + await Deno.writeTextFile(file, `${JSON.stringify(await run(instrument, work, sides), null, 2)}\n`) + } finally { + await Deno.remove(work, { recursive: true }) + } } const pinCommand = async (args: readonly string[]): Promise => { @@ -157,7 +243,10 @@ const pinCommand = async (args: readonly string[]): Promise => { if (action === 'check') { const { owner, repo, commit } = instrument.pin Deno.exit( - await decide(instrument, ['pin-check', '--owner', owner, '--repo', repo, '--pinned', commit], { github: true }), + await withDecide( + instrument, + (work) => decide(work, ['pin-check', '--owner', owner, '--repo', repo, '--pinned', commit], { github: true }), + ), ) } if (action === 'write') { @@ -174,11 +263,9 @@ const pinCommand = async (args: readonly string[]): Promise => { } const commands: Readonly Promise>> = { - 'plan': () => plan(), + 'ci': ci, 'measure': measure, 'aggregate': aggregate, - 'cache-check': cacheCheck, - 'latest-main-run': () => latestMainRun(), 'pin': pinCommand, 'journeys': journeys, 'check': check, diff --git a/evals/ratstack-scorecard/src/metrics/registry.ts b/evals/ratstack-scorecard/src/metrics/registry.ts index d1e364b..5f85495 100644 --- a/evals/ratstack-scorecard/src/metrics/registry.ts +++ b/evals/ratstack-scorecard/src/metrics/registry.ts @@ -1,19 +1,10 @@ -import type { Family, RowDefinition } from '../model/cell.ts' +import type { RowDefinition } from '../model/cell.ts' const count3 = { kind: 'count', runs: 3 } as const const measured5 = { kind: 'measurement', runs: 5 } as const const measured3 = { kind: 'measurement', runs: 3 } as const const required = { ratstackSupport: { _tag: 'Required' } } as const -export const familyTimeoutMinutes: Readonly> = { - 'static': 20, - 'cold-path': 120, - 'gate-mutation': 240, - 'running-stack': 90, - 'agent-surfaces': 90, - 'networked': 60, -} - export const rowDefinitions: readonly RowDefinition[] = [ { id: 'M1.checks', From 7a81ef734dfdf498dd02299e1201d0d27d3d498a Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Oct 2026 06:43:45 +0000 Subject: [PATCH 5/6] fix(repo): refresh the scorecard tools store hashes for every platform With supportedArchitectures and the per-system hash table from the layer below, this layer's lockfile (typescript 7 for the decide step) carries its own linux and darwin hashes; the darwin value is the one the macOS CI leg computed --- evals/ratstack-scorecard/flake.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index 955962f..d41732b 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -15,9 +15,9 @@ forEachSystem = fn: nixpkgs.lib.genAttrs systems (system: fn nixpkgs.legacyPackages.${system}); pin = builtins.fromJSON (builtins.readFile ./ratstack.pin.json); toolsStoreHash = { - x86_64-linux = "sha256-jUyIV2ysOxkIij/Ff5VGuIADTibbY0SsCPGglmIvdn0="; - aarch64-linux = "sha256-jUyIV2ysOxkIij/Ff5VGuIADTibbY0SsCPGglmIvdn0="; - aarch64-darwin = "sha256-XWYBTy3xZbQwJxdGbAK4+R69XNmL3RRyXu6+nY+XPXc="; + x86_64-linux = "sha256-6VGGHRgFjm5/RJ9ZY+39KMiFHsuWX+by37oWW6fQq/w="; + aarch64-linux = "sha256-6VGGHRgFjm5/RJ9ZY+39KMiFHsuWX+by37oWW6fQq/w="; + aarch64-darwin = "sha256-vCwExwVl5WmwbuhB7HxbZD0F7WSE7fS6PTXm4SJcn2U="; }; in { @@ -30,7 +30,7 @@ src = self; pname = "ratstack-scorecard"; pnpm = pkgs.pnpm_12; - hash = "sha256-9/W9Q1CNzoBbRBhd+qdu7TfR8xk/7mjoiEOkOLwSRpU="; + hash = toolsStoreHash.${system}; }).pnpm-store; ratstack-src = pkgs.fetchFromGitHub { inherit (pin) owner repo; From d70e047ad9121272107873209c4738e5c025b19d Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 6 Oct 2026 08:46:30 +0000 Subject: [PATCH 6/6] docs(repo): require a darwin offline install in CI before a launcher change is done Kiro approved this AGENTS.md line on 2026-10-06 after the scorecard's macOS leg exposed a launcher path that only the darwin sandbox exercises (pnpm 12's store lock in /tmp) --- AGENTS.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fab02d3..8a5c0c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,12 +14,13 @@ Starter template for TypeScript / Effect libraries and tools. ## Definition of Done -| ID | Rule | Gate | -| --------- | --------------------------------------------------- | ------------------- | -| `START-1` | Formatting passes dprint with no diffs | `pnpm format:check` | -| `START-2` | Typechecking succeeds workspace-wide with no errors | `pnpm typecheck` | -| `START-3` | All test suites pass | `pnpm test` | -| `START-4` | Full CI validation passes before completion | `pnpm check:ci` | +| ID | Rule | Gate | +| --------- | ---------------------------------------------------------------------------------------------------- | ------------------------- | +| `START-1` | Formatting passes dprint with no diffs | `pnpm format:check` | +| `START-2` | Typechecking succeeds workspace-wide with no errors | `pnpm typecheck` | +| `START-3` | All test suites pass | `pnpm test` | +| `START-4` | Full CI validation passes before completion | `pnpm check:ci` | +| `START-5` | A launcher change counts as done only when its darwin path has run a real offline pnpm install in CI | `check (macos)` job in CI | Workspace roots: `packages/` holds libraries, `apps/` holds publishable applications — both are workspace globs in `pnpm-workspace.yaml`. Turbo declares