diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml new file mode 100644 index 0000000..f04a5aa --- /dev/null +++ b/.github/workflows/scorecard.yml @@ -0,0 +1,198 @@ +name: Scorecard + +on: + pull_request: + branches: [main] + push: + branches: [main] + schedule: + - cron: "17 4 * * *" + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: scorecard-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + scorecard: + runs-on: ubuntu-latest + timeout-minutes: 360 + permissions: + contents: read + actions: read + pull-requests: write + outputs: + mode: ${{ steps.instrument.outputs.mode }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - id: instrument + name: Choose the instrument that grades this run + env: + BASE: ${{ github.event.pull_request.base.sha }} + run: | + if [ "$GITHUB_EVENT_NAME" != pull_request ]; then + printf 'ref=%s\nmode=base\n' "$GITHUB_SHA" >> "$GITHUB_OUTPUT" + exit 0 + fi + selector=evals/ratstack-scorecard/ci/instrument-ref.sh + if git cat-file -e "$BASE:$selector" 2>/dev/null; then + git show "$BASE:$selector" | bash -s -- "$BASE" "$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + else + bash "$selector" "$BASE" "$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + fi + cat "$GITHUB_OUTPUT" + - uses: actions/checkout@v7 + with: + ref: ${{ steps.instrument.outputs.ref }} + path: .scorecard-instrument + persist-credentials: false + - name: "BOOTSTRAP: self-graded, base had no instrument" + if: steps.instrument.outputs.mode == 'bootstrap' + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + run: | + banner="# ⚠️ BOOTSTRAP: SELF-GRADED, BASE HAD NO INSTRUMENT + + The base commit has no \`.github/workflows/scorecard.yml\`, so this run grades the pull request with its own instrument. Every later pull request is graded by its base's instrument." + printf '%s\n\n' "$banner" >> "$GITHUB_STEP_SUMMARY" + gh pr comment "$PR" --body "$banner" + - uses: cachix/install-nix-action@v31 + - name: Allow the launcher's unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Build the grading instrument + run: echo "SCORECARD=$(nix build --no-link --print-out-paths ./.scorecard-instrument/evals/ratstack-scorecard#scorecard)/bin/scorecard" >> "$GITHUB_ENV" + - name: Restore rat-stack results + uses: actions/cache/restore@v6 + with: + path: .scorecard-cache + key: scorecard-ratstack-${{ github.run_id }} + restore-keys: scorecard-ratstack- + - name: Measure, judge, ratchet and publish + env: + GITHUB_TOKEN: ${{ github.token }} + run: '"$SCORECARD" ci --checkout "$GITHUB_WORKSPACE" --cache .scorecard-cache --out scorecard-out' + - uses: actions/upload-artifact@v7 + if: ${{ !cancelled() }} + with: + name: scorecard + path: scorecard-out/scorecard.json + if-no-files-found: error + - name: Save rat-stack results + if: ${{ !cancelled() && github.event_name != 'pull_request' }} + uses: actions/cache/save@v6 + with: + path: .scorecard-cache + key: scorecard-ratstack-${{ github.run_id }} + + instrument-preview: + name: instrument preview (informational) + needs: scorecard + if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.scorecard.outputs.mode == 'base' }} + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - id: changed + name: Does this PR edit the instrument + run: | + if git diff --quiet "${{ github.event.pull_request.base.sha }}" HEAD -- evals/ratstack-scorecard; then + echo "edits=false" >> "$GITHUB_OUTPUT" + echo "This PR leaves the instrument alone; nothing to preview." >> "$GITHUB_STEP_SUMMARY" + else + echo "edits=true" >> "$GITHUB_OUTPUT" + fi + - uses: cachix/install-nix-action@v31 + if: steps.changed.outputs.edits == 'true' + - name: Allow the launcher's unprivileged user namespaces + if: steps.changed.outputs.edits == 'true' + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - uses: actions/download-artifact@v8 + if: steps.changed.outputs.edits == 'true' + with: + name: scorecard + path: graded + - name: Grade with this PR's own instrument against the graded scorecard + if: steps.changed.outputs.edits == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + scorecard="$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#scorecard)/bin/scorecard" + "$scorecard" ci --checkout "$GITHUB_WORKSPACE" --cache .scorecard-cache --out preview \ + --main graded/scorecard.json --title "Instrument preview (does not gate)" + + pin: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: cachix/install-nix-action@v31 + - name: Allow the launcher's unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Build the instrument + run: echo "SCORECARD=$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#scorecard)/bin/scorecard" >> "$GITHUB_ENV" + - id: check + name: Compare the pin with rat-stack main + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + plan="$("$SCORECARD" pin check)" + echo "$plan" | jq . + echo "tag=$(jq -r ._tag <<<"$plan")" >> "$GITHUB_OUTPUT" + echo "to=$(jq -r '.to // empty' <<<"$plan")" >> "$GITHUB_OUTPUT" + echo "Pin: $(jq -r 'if ._tag == "PinCurrent" then "pin current at \(.commit)" else "rat-stack moved \(.from) -> \(.to)" end' <<<"$plan")" >> "$GITHUB_STEP_SUMMARY" + - id: token + if: steps.check.outputs.tag == 'PinMoved' + name: Pin-bump app token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.SCORECARD_APP_ID }} + private-key: ${{ secrets.SCORECARD_APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + # scorecard/pin-rat-stack is bot-owned: the job only ever replaces a tip the bot + # itself wrote, through a lease pinned to that tip, so a human commit there is + # never overwritten. The App token reaches git through a credential helper that + # reads it from the environment, never through argv or a URL. + - name: Open or update the pin-bump PR + if: steps.check.outputs.tag == 'PinMoved' + env: + GH_TOKEN: ${{ steps.token.outputs.token }} + TO: ${{ steps.check.outputs.to }} + BRANCH: scorecard/pin-rat-stack + BOT_EMAIL: scorecard-pin[bot]@users.noreply.github.com + run: | + git config credential.helper "!f() { echo username=x-access-token; echo \"password=\${GH_TOKEN}\"; }; f" + nar_hash="$(nix flake prefetch --json "github:joelhooks/rat-stack/$TO" | jq -r .hash)" + "$SCORECARD" pin write --commit "$TO" --nar-hash "$nar_hash" + lease="$(git ls-remote origin "refs/heads/$BRANCH" | cut -f1)" + if [ -n "$lease" ]; then + git fetch --quiet origin "$lease" + author="$(git log -1 --format=%ae "$lease")" + if [ "$author" != "$BOT_EMAIL" ]; then + echo "::error::$BRANCH tip $lease was written by $author, not the pin bot; refusing to replace it" + exit 1 + fi + fi + git config user.name "scorecard-pin[bot]" + git config user.email "$BOT_EMAIL" + git switch -c "$BRANCH" + git commit -m "chore(deps): pin rat-stack to ${TO:0:7} for the scorecard" -- evals/ratstack-scorecard/ratstack.pin.json + git push --force-with-lease="refs/heads/$BRANCH:$lease" origin "HEAD:refs/heads/$BRANCH" + if [ -z "$(gh pr list --head "$BRANCH" --json number --jq '.[0].number')" ]; then + gh pr create --base main --head "$BRANCH" \ + --title "chore(deps): pin rat-stack to ${TO:0:7} for the scorecard" \ + --body "rat-stack \`main\` moved to \`$TO\`. This PR moves \`evals/ratstack-scorecard/ratstack.pin.json\` only; its scorecard run measures the new rat-stack. Kiro merges." + fi diff --git a/AGENTS.md b/AGENTS.md index fab02d3..8a5c0c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,12 +14,13 @@ Starter template for TypeScript / Effect libraries and tools. ## Definition of Done -| ID | Rule | Gate | -| --------- | --------------------------------------------------- | ------------------- | -| `START-1` | Formatting passes dprint with no diffs | `pnpm format:check` | -| `START-2` | Typechecking succeeds workspace-wide with no errors | `pnpm typecheck` | -| `START-3` | All test suites pass | `pnpm test` | -| `START-4` | Full CI validation passes before completion | `pnpm check:ci` | +| ID | Rule | Gate | +| --------- | ---------------------------------------------------------------------------------------------------- | ------------------------- | +| `START-1` | Formatting passes dprint with no diffs | `pnpm format:check` | +| `START-2` | Typechecking succeeds workspace-wide with no errors | `pnpm typecheck` | +| `START-3` | All test suites pass | `pnpm test` | +| `START-4` | Full CI validation passes before completion | `pnpm check:ci` | +| `START-5` | A launcher change counts as done only when its darwin path has run a real offline pnpm install in CI | `check (macos)` job in CI | Workspace roots: `packages/` holds libraries, `apps/` holds publishable applications — both are workspace globs in `pnpm-workspace.yaml`. Turbo declares diff --git a/evals/ratstack-scorecard/README.md b/evals/ratstack-scorecard/README.md index fb3caae..db11064 100644 --- a/evals/ratstack-scorecard/README.md +++ b/evals/ratstack-scorecard/README.md @@ -28,12 +28,31 @@ scorecard=$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#sco $scorecard/bin/scorecard measure --family static --out static.json ``` -The instrument's own tests run inside the launcher against the same offline store: +The instrument's own checks run with one command: `scorecard check` (also `pnpm scorecard:check` at the repo root, part of `check:ci`). It first refuses any third-party import in the host driver's graph and any import of the journey fixture from `src/`, then runs `scorecard journeys`. + +The program is split in two: + +- **The host driver** (`src/main.ts` and what it imports) uses only Deno APIs, `node:` builtins and its own files, so `DENO_NO_PACKAGE_JSON=1 deno check src/` type-checks it without third-party code. It starts launcher invocations, mounts their inputs read-only, collects output files and exit codes, and writes the pin file. It decodes and judges nothing. Its only network access is through the launcher. +- **The decide step** (`decide/`, Effect 4 with Schema and `effect/http`) runs as its own launcher invocation from a copy of the grading instrument: `aggregate` (decode the family results and `main`'s scorecard, judge, ratchet, render), `cache-check`, `main-scorecard` and `pin-check`. Only the last two reach the network, and only `api.github.com` and the artifact store (`*.blob.core.windows.net`); GitHub calls time out after 15 s, retry transient failures three times, and fail as a typed `GithubApiError`. Pass or fail is the step's exit code. `pnpm exec tsc` type-checks `decide/` and `journeys/` inside the launcher. + +Journeys run in two phases, because the launcher cannot nest. `scorecard journeys` produces every journey declared in `journeys/manifest.json` through the real launcher work and records it to `journeys/__records__/.json`, then runs the one vitest project inside the launcher. A test is a journey because it imports `journeys/launcher-run.ts`; a missing or stale record fails it red. ```sh -cd evals/ratstack-scorecard -nix develop --command sh -c 'SANDBOX_PROJECT=$PWD sandbox --pnpm-store "$SANDBOX_PNPM_STORE" -- pnpm install --frozen-lockfile' -nix develop --command sh -c 'SANDBOX_PROJECT=$PWD sandbox -- pnpm vitest run' +scorecard=$(nix build --no-link --print-out-paths ./evals/ratstack-scorecard#scorecard) +$scorecard/bin/scorecard check ``` -`src/main.ts` and everything it imports use only Deno APIs, `node:` builtins and each other, so `DENO_NO_PACKAGE_JSON=1 deno check src/` type-checks the orchestrator and the decision core without third-party code. The Node scripts in `src/tools/` are the only code that loads npm packages, and they only ever run inside the launcher. +The Node scripts in `src/tools/` and everything in `decide/` load npm packages, and they only ever run inside the launcher. + +## In CI + +`.github/workflows/scorecard.yml` runs on every PR to `main`, every push to `main`, daily, and on demand. All jobs run on GitHub-hosted runners (`ubuntu-latest`): the self-hosted fleet excludes public repositories by design. + +The workflow is a thin caller: it chooses the grading instrument, builds it, and runs `scorecard ci`, the one stable entrypoint. Measuring every family, the rat-stack cache, finding `main`'s scorecard, judging, the ratchet and the job summary all happen inside the instrument, so a pull request that adds a step never needs its base to know that step; the step starts grading after it merges. + +- **Which instrument grades.** Pushes, the schedule and manual runs use their own commit. A pull request is graded by its base commit's instrument, chosen by `ci/instrument-ref.sh` (run from the base tree whenever the base has it). Only when the base tree has no `.github/workflows/scorecard.yml` at all is the PR head's instrument used; the job summary and a PR comment then say **BOOTSTRAP: self-graded, base had no instrument**. A base commit missing from the clone fails the job instead of falling back to bootstrap. The `instrument-ref` journey proves it on real git history: with the workflow on the base, a PR that edits the instrument to flip a verdict, deletes the instrument, or deletes the workflow is still graded by the base. +- **scorecard** builds the chosen instrument and runs `scorecard ci`. The rat-stack side comes from the cache when its key (rat-stack commit, instrument content hash, nixpkgs rev) matches and `cache-check` accepts it; an entry holding a rat-stack instrument error is refused and measured again. Only pushes, the schedule and manual runs save the cache, only results without instrument errors, and stale keys are pruned. The starter side is measured every run. `scorecard ci` compares with the latest successful `main` run's `scorecard` artifact (none yet means a first baseline), writes the table to the job summary, uploads `scorecard.json`, and fails when the ratchet fails. Re-baselined rows are listed under definition drift with both hashes. A definition hash covers only a row's registry entry and its family's measurement code. +- **instrument preview** (PRs graded by their base that edit the instrument; never gates) runs the PR's own `scorecard ci` against the graded scorecard and lists what it would re-baseline. +- **pin** (not on PRs) compares `ratstack.pin.json` with rat-stack `main`, decoded as a 40-hex SHA. When rat-stack has moved it opens or updates the `scorecard/pin-rat-stack` PR through the pin-bump GitHub App (`vars.SCORECARD_APP_ID`, `secrets.SCORECARD_APP_PRIVATE_KEY`; contents and pull requests write only). The branch is bot-owned: the job refuses to replace a tip another author wrote and pushes with a lease pinned to the tip it read. The token reaches git through a credential helper that reads it from the environment. That PR changes only the pin; Kiro merges it. + +`actionlint` is in the dev shell: `nix develop ./evals/ratstack-scorecard --command actionlint .github/workflows/scorecard.yml`. diff --git a/evals/ratstack-scorecard/ci/instrument-ref.sh b/evals/ratstack-scorecard/ci/instrument-ref.sh new file mode 100755 index 0000000..900ab59 --- /dev/null +++ b/evals/ratstack-scorecard/ci/instrument-ref.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "usage: instrument-ref.sh " >&2 + exit 2 +fi +base=$1 +head=$2 + +git cat-file -e "${base}^{commit}" 2>/dev/null || { + echo "base commit ${base} is not in this clone; fetch it before choosing an instrument" >&2 + exit 1 +} +git cat-file -e "${head}^{commit}" 2>/dev/null || { + echo "head commit ${head} is not in this clone" >&2 + exit 1 +} + +if git cat-file -e "${base}:.github/workflows/scorecard.yml" 2>/dev/null; then + echo "ref=${base}" + echo "mode=base" +else + echo "ref=${head}" + echo "mode=bootstrap" +fi diff --git a/evals/ratstack-scorecard/decide/main.ts b/evals/ratstack-scorecard/decide/main.ts new file mode 100644 index 0000000..36954f5 --- /dev/null +++ b/evals/ratstack-scorecard/decide/main.ts @@ -0,0 +1,291 @@ +import { Cause, Duration, Effect, Schedule, Schema } from 'effect' +import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from 'effect/http' +import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { parseArgs } from 'node:util' +import { inflateRawSync } from 'node:zlib' +import { rowDefinitions } from '../src/metrics/registry.ts' +import type { Family, FamilyResult, MainBaseline, MeasuredCell, Side, SideCell } from '../src/model/cell.ts' +import { planPinBump } from '../src/model/plan-pin-bump.workflow.ts' +import { assembleScorecard, type CellsByRow } from '../src/model/scorecard-document.ts' +import { renderSummary } from '../src/model/summary-table.ts' +import { + ArtifactsSchema, + FamilyResultSchema, + GitRefSchema, + ScorecardDocumentSchema, + WorkflowRunsSchema, +} from './schema.ts' + +class MalformedInput extends Schema.TaggedError()('MalformedInput', { + what: Schema.String, + issue: Schema.String, +}) {} + +class DuplicateCell extends Schema.TaggedError()('DuplicateCell', { + id: Schema.String, + side: Schema.String, +}) {} + +class GithubApiError extends Schema.TaggedError()('GithubApiError', { + request: Schema.String, + cause: Schema.String, +}) {} + +class RatchetFailed extends Schema.TaggedError()('RatchetFailed', { + rows: Schema.Array(Schema.String), +}) {} + +class CacheUnusable extends Schema.TaggedError()('CacheUnusable', { + reason: Schema.String, +}) {} + +const readText = (path: string) => + Effect.tryPromise({ + try: () => readFile(path, 'utf8'), + catch: (error) => new MalformedInput({ what: path, issue: String(error) }), + }) + +const writeText = (path: string, text: string) => + Effect.promise(async () => { + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, text) + }) + +const decodeFile = (schema: Schema.Codec, path: string) => + readText(path).pipe( + Effect.flatMap((text) => Schema.decodeUnknownEffect(Schema.fromJsonString(schema))(text)), + Effect.mapError((error) => new MalformedInput({ what: path, issue: String(error) })), + ) + +type Keyed = Readonly>>>> + +const noCells: Keyed = {} + +const keyCells = (cells: readonly SideCell[]): Effect.Effect => + Effect.reduce(cells, () => noCells, (keyed: Keyed, cell: SideCell) => + keyed[cell.id]?.[cell.side] === undefined + ? Effect.succeed({ ...keyed, [cell.id]: { ...keyed[cell.id], [cell.side]: cell.measured } }) + : Effect.fail(new DuplicateCell({ id: cell.id, side: cell.side }))) + +const familyResults = (dir: string) => + Effect.promise(() => readdir(dir)).pipe( + Effect.flatMap((names) => + Effect.forEach( + names.filter((name) => name.endsWith('.json')).sort(), + (name) => decodeFile(FamilyResultSchema, join(dir, name)), + ) + ), + ) + +const mainBaseline = (path: string | undefined): Effect.Effect => + path === undefined + ? Effect.succeed({ _tag: 'Missing' }) + : decodeFile(ScorecardDocumentSchema, path).pipe( + Effect.map((doc) => ({ _tag: 'Found', commit: doc.provenance.commit, rows: doc.rows })), + ) + +const aggregate = (args: readonly string[]) => + Effect.gen(function*() { + const { values } = parseArgs({ + args: [...args], + options: { + families: { type: 'string' }, + implemented: { type: 'string' }, + main: { type: 'string' }, + commit: { type: 'string' }, + out: { type: 'string' }, + summary: { type: 'string' }, + }, + strict: true, + }) + const results: readonly FamilyResult[] = yield* familyResults(yield* required(values.families, '--families')) + const implemented: readonly string[] = (yield* required(values.implemented, '--implemented')).split(',') + const cells = results.flatMap((result) => result.cells) + const hashes = results.flatMap((result) => result.definitionHashes) + const provenanceOf = (side: Side) => cells.find((cell) => cell.side === side)?.measured.provenance + const any = provenanceOf('starter') ?? provenanceOf('ratstack') + if (any === undefined) return yield* new MalformedInput({ what: values.families ?? '', issue: 'no cells' }) + const doc = assembleScorecard({ + rows: rowDefinitions + .filter((definition) => implemented.includes(definition.family)) + .map((definition) => ({ + definition, + hash: hashes.find((hash) => hash.id === definition.id)?.hash ?? `no ${definition.family} result`, + })), + cells: yield* keyCells(cells), + flags: results.flatMap((result) => result.flags), + provenance: { + commit: provenanceOf('starter')?.commit ?? (yield* required(values.commit, '--commit')), + ratstackCommit: provenanceOf('ratstack')?.commit ?? 'unknown', + instrumentHash: any.instrumentHash, + nixpkgsRev: any.nixpkgsRev, + runner: any.runner, + generatedAt: new Date().toISOString(), + }, + main: yield* mainBaseline(values.main), + }) + yield* writeText(yield* required(values.out, '--out'), `${JSON.stringify(doc, null, 2)}\n`) + yield* writeText(yield* required(values.summary, '--summary'), renderSummary(doc)) + if (doc.ratchet.failures.length > 0) { + return yield* new RatchetFailed({ rows: doc.ratchet.failures.map((failure) => failure.id) }) + } + }) + +const cacheCheck = (args: readonly string[]) => + Effect.gen(function*() { + const { values } = parseArgs({ args: [...args], options: { file: { type: 'string' } }, strict: true }) + const result = yield* decodeFile(FamilyResultSchema, yield* required(values.file, '--file')).pipe( + Effect.mapError((error) => new CacheUnusable({ reason: `${error.what}: ${error.issue}` })), + ) + const broken = result.cells.filter((cell) => + cell.side === 'ratstack' && cell.measured.cell._tag === 'InstrumentError' + ) + if (broken.length > 0) { + return yield* new CacheUnusable({ + reason: `rat-stack instrument errors on ${broken.map((c) => c.id).join(', ')}`, + }) + } + }) + +const githubGet = (path: string) => + Effect.gen(function*() { + const client = (yield* HttpClient.HttpClient).pipe( + HttpClient.retryTransient({ schedule: Schedule.exponential(Duration.seconds(1)), times: 3 }), + ) + const token = process.env['GITHUB_TOKEN'] + const request = HttpClientRequest.get(path.startsWith('https://') ? path : `https://api.github.com${path}`).pipe( + (r) => token === undefined || token === '' ? r : HttpClientRequest.bearerToken(r, token), + ) + return yield* client.execute(request).pipe(Effect.timeout(Duration.seconds(15))) + }).pipe( + Effect.mapError((error) => new GithubApiError({ request: `GET ${path}`, cause: String(error) })), + Effect.provide(FetchHttpClient.layer), + ) + +const ok = (path: string) => +( + response: HttpClientResponse.HttpClientResponse, +): Effect.Effect => + response.status >= 200 && response.status < 300 + ? Effect.succeed(response) + : Effect.fail(new GithubApiError({ request: `GET ${path}`, cause: `status ${response.status}` })) + +const github = (path: string) => + githubGet(path).pipe( + Effect.flatMap(ok(path)), + Effect.flatMap((response) => response.json), + Effect.mapError((error) => new GithubApiError({ request: `GET ${path}`, cause: String(error) })), + ) + +const decodeBody = (schema: Schema.Codec, what: string) => (body: unknown) => + Schema.decodeUnknownEffect(schema)(body).pipe( + Effect.mapError((error) => new GithubApiError({ request: what, cause: String(error) })), + ) + +const zipEntry = (zip: Buffer, name: string) => + Effect.try({ + try: () => { + const end = zip.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06])) + const count = zip.readUInt16LE(end + 10) + let entry = zip.readUInt32LE(end + 16) + for (let index = 0; index < count; index++) { + const nameLength = zip.readUInt16LE(entry + 28) + const extraLength = zip.readUInt16LE(entry + 30) + const commentLength = zip.readUInt16LE(entry + 32) + if (zip.toString('utf8', entry + 46, entry + 46 + nameLength) === name) { + const method = zip.readUInt16LE(entry + 10) + const size = zip.readUInt32LE(entry + 20) + const local = zip.readUInt32LE(entry + 42) + const start = local + 30 + zip.readUInt16LE(local + 26) + zip.readUInt16LE(local + 28) + const data = zip.subarray(start, start + size) + return (method === 8 ? inflateRawSync(data) : data).toString('utf8') + } + entry += 46 + nameLength + extraLength + commentLength + } + throw new Error(`no ${name} in the archive`) + }, + catch: (error) => new GithubApiError({ request: `unzip ${name}`, cause: String(error) }), + }) + +const mainScorecard = (args: readonly string[]) => + Effect.gen(function*() { + const { values } = parseArgs({ + args: [...args], + options: { repository: { type: 'string' }, out: { type: 'string' } }, + strict: true, + }) + const repository = yield* required(values.repository, '--repository') + const out = yield* required(values.out, '--out') + const runsPath = + `/repos/${repository}/actions/workflows/scorecard.yml/runs?branch=main&event=push&status=success&per_page=1` + const listing = yield* githubGet(runsPath) + if (listing.status === 404) return yield* Effect.log('main has no scorecard workflow yet: first baseline') + const runs = yield* ok(runsPath)(listing).pipe( + Effect.flatMap((response) => response.json), + Effect.mapError((error) => new GithubApiError({ request: `GET ${runsPath}`, cause: String(error) })), + Effect.flatMap(decodeBody(WorkflowRunsSchema, `GET ${runsPath}`)), + ) + const run = runs.workflow_runs[0] + if (run === undefined) return yield* Effect.log('no successful scorecard run on main: first baseline') + const artifactsPath = `/repos/${repository}/actions/runs/${run.id}/artifacts?name=scorecard` + const artifacts = yield* github(artifactsPath).pipe(Effect.flatMap(decodeBody(ArtifactsSchema, artifactsPath))) + const artifact = artifacts.artifacts[0] + if (artifact === undefined) { + return yield* new GithubApiError({ request: artifactsPath, cause: `run ${run.id} has no scorecard artifact` }) + } + const zip = yield* githubGet(artifact.archive_download_url).pipe( + Effect.flatMap(ok(artifact.archive_download_url)), + Effect.flatMap((response) => response.arrayBuffer), + Effect.mapError((error) => new GithubApiError({ request: 'artifact download', cause: String(error) })), + ) + const text = yield* zipEntry(Buffer.from(zip), 'scorecard.json') + yield* Schema.decodeUnknownEffect(Schema.fromJsonString(ScorecardDocumentSchema))(text).pipe( + Effect.mapError((error) => + new MalformedInput({ what: `main's scorecard (run ${run.id})`, issue: String(error) }) + ), + ) + yield* writeText(out, text) + yield* Effect.log(`main's scorecard from run ${run.id} (${run.head_sha})`) + }) + +const pinCheck = (args: readonly string[]) => + Effect.gen(function*() { + const { values } = parseArgs({ + args: [...args], + options: { owner: { type: 'string' }, repo: { type: 'string' }, pinned: { type: 'string' } }, + strict: true, + }) + const path = `/repos/${yield* required(values.owner, '--owner')}/${yield* required( + values.repo, + '--repo', + )}/git/ref/heads/main` + const ref = yield* github(path).pipe(Effect.flatMap(decodeBody(GitRefSchema, `GET ${path}`))) + const plan = planPinBump({ pinned: yield* required(values.pinned, '--pinned'), remoteHead: ref.object.sha }) + yield* Effect.sync(() => process.stdout.write(`${JSON.stringify(plan)}\n`)) + }) + +const required = (value: string | undefined, flag: string) => + value === undefined + ? Effect.fail(new MalformedInput({ what: flag, issue: 'is required' })) + : Effect.succeed(value) + +const commands: Readonly Effect.Effect>> = { + 'aggregate': aggregate, + 'cache-check': cacheCheck, + 'main-scorecard': mainScorecard, + 'pin-check': pinCheck, +} + +const [command = '', ...rest] = process.argv.slice(2) +const run = commands[command] +if (run === undefined) { + process.stderr.write(`decide: unknown command ${command}; one of ${Object.keys(commands).join(', ')}\n`) + process.exit(2) +} +const exit = await Effect.runPromiseExit(run(rest)) +if (exit._tag === 'Failure') { + const error = Cause.squash(exit.cause) + process.stderr.write(`decide ${command}: ${String(error)} ${JSON.stringify(error)}\n`) + process.exit(1) +} diff --git a/evals/ratstack-scorecard/decide/schema.ts b/evals/ratstack-scorecard/decide/schema.ts new file mode 100644 index 0000000..11e5210 --- /dev/null +++ b/evals/ratstack-scorecard/decide/schema.ts @@ -0,0 +1,150 @@ +import { Schema } from 'effect' +import type { + Cell, + CellProvenance, + FamilyResult, + Flag, + MeasuredCell, + Ratchet, + Row, + RowDefinition, + RowOutcome, + ScorecardDocument, + Verdict, +} from '../src/model/cell.ts' + +const side = Schema.Literals(['ratstack', 'starter']) +const family = Schema.Literals(['static', 'cold-path', 'gate-mutation', 'running-stack', 'agent-surfaces', 'networked']) +const cellTag = Schema.Literals([ + 'Measured', + 'Absent', + 'NoDeployment', + 'Unsupported', + 'Unmeasurable', + 'NoSecret', + 'InstrumentError', +]) + +export const Sha = Schema.String.check(Schema.isPattern(/^[0-9a-f]{40}$/u)) + +const tagged = (tag: T, fields: F) => + Schema.Struct({ _tag: Schema.Literal(tag), ...fields }) + +export const CellSchema: Schema.Codec = Schema.Union([ + tagged('Measured', { runs: Schema.Array(Schema.Finite) }), + tagged('Absent', { reason: Schema.String }), + tagged('NoDeployment', { sha: Schema.String }), + tagged('Unsupported', { + citation: Schema.Struct({ + file: Schema.String, + lines: Schema.Tuple([Schema.Finite, Schema.Finite]), + text: Schema.String, + }), + check: Schema.Union([tagged('Verified', {}), tagged('Contradicted', { found: Schema.String })]), + }), + tagged('Unmeasurable', { error: Schema.String }), + tagged('NoSecret', { name: Schema.String }), + tagged('InstrumentError', { error: Schema.String }), +]) + +const CellProvenanceSchema: Schema.Codec = Schema.Struct({ + side, + commit: Schema.String, + instrumentHash: Schema.String, + nixpkgsRev: Schema.String, + runner: Schema.String, + measuredAt: Schema.String, + tools: Schema.Record(Schema.String, Schema.String), + liveCommit: Schema.optionalKey(Schema.String), + detail: Schema.optionalKey(Schema.Record(Schema.String, Schema.Union([Schema.Finite, Schema.String]))), +}) + +const MeasuredCellSchema: Schema.Codec = Schema.Struct({ + cell: CellSchema, + provenance: CellProvenanceSchema, +}) + +const FlagSchema: Schema.Codec = Schema.Union([ + tagged('LiveDiffersFromPin', { live: Schema.String, pin: Schema.String }), + tagged('ScannersDisagree', { primary: Schema.Finite, crossCheck: Schema.Finite }), +]) + +const RowDefinitionSchema: Schema.Codec = Schema.Struct({ + id: Schema.String, + metric: Schema.String, + bin: Schema.String, + label: Schema.String, + unit: Schema.String, + direction: Schema.Literals(['lower', 'higher']), + kind: Schema.Literals(['count', 'measurement']), + runs: Schema.Finite, + family, + ratstackSupport: Schema.Union([tagged('Required', {}), tagged('MayBeUnsupported', { bar: Schema.Finite })]), +}) + +const VerdictSchema: Schema.Codec = Schema.Union([ + tagged('Beaten', {}), + tagged('NotBeaten', {}), + tagged('Tie', {}), + tagged('InstrumentError', { error: Schema.String }), +]) + +const RowSchema: Schema.Codec = Schema.Struct({ + definition: RowDefinitionSchema, + definitionHash: Schema.String, + ratstack: MeasuredCellSchema, + starter: MeasuredCellSchema, + verdict: VerdictSchema, + flags: Schema.Array(FlagSchema), +}) + +const OutcomeSchema: Schema.Codec = Schema.Union([ + tagged('Held', { id: Schema.String }), + tagged('New', { id: Schema.String }), + tagged('ReBaselined', { id: Schema.String, mainHash: Schema.String, prHash: Schema.String }), + tagged('Neutral', { id: Schema.String, cause: cellTag }), + tagged('InstrumentError', { id: Schema.String, error: Schema.String }), + tagged('LostBeaten', { id: Schema.String, ratstack: cellTag, starter: cellTag }), + tagged('Regressed', { id: Schema.String, main: Schema.String, pr: Schema.String }), +]) + +const RatchetSchema: Schema.Codec = Schema.Union([ + tagged('FirstBaseline', { outcomes: Schema.Array(OutcomeSchema), failures: Schema.Array(OutcomeSchema) }), + tagged('Compared', { + mainCommit: Schema.String, + outcomes: Schema.Array(OutcomeSchema), + failures: Schema.Array(OutcomeSchema), + }), +]) + +export const ScorecardDocumentSchema: Schema.Codec = Schema.Struct({ + schemaVersion: Schema.Literal(1), + provenance: Schema.Struct({ + commit: Schema.String, + ratstackCommit: Schema.String, + instrumentHash: Schema.String, + nixpkgsRev: Schema.String, + runner: Schema.String, + generatedAt: Schema.String, + }), + rows: Schema.Array(RowSchema), + ratchet: RatchetSchema, +}) + +export const FamilyResultSchema: Schema.Codec = Schema.Struct({ + family, + wallMs: Schema.Finite, + cells: Schema.Array(Schema.Struct({ id: Schema.String, side, measured: MeasuredCellSchema })), + flags: Schema.Array(Schema.Struct({ id: Schema.String, flag: FlagSchema })), + definitionHashes: Schema.Array(Schema.Struct({ id: Schema.String, hash: Schema.String })), +}) + +export const WorkflowRunsSchema = Schema.Struct({ + workflow_runs: Schema.Array(Schema.Struct({ id: Schema.Finite, head_sha: Sha })), +}) + +export const GitRefSchema = Schema.Struct({ object: Schema.Struct({ sha: Sha }) }) + +export const ArtifactsSchema = Schema.Struct({ + artifacts: Schema.Array(Schema.Struct({ id: Schema.Finite, archive_download_url: Schema.String })), +}) diff --git a/evals/ratstack-scorecard/flake.nix b/evals/ratstack-scorecard/flake.nix index f294a97..d41732b 100644 --- a/evals/ratstack-scorecard/flake.nix +++ b/evals/ratstack-scorecard/flake.nix @@ -15,9 +15,9 @@ forEachSystem = fn: nixpkgs.lib.genAttrs systems (system: fn nixpkgs.legacyPackages.${system}); pin = builtins.fromJSON (builtins.readFile ./ratstack.pin.json); toolsStoreHash = { - x86_64-linux = "sha256-jUyIV2ysOxkIij/Ff5VGuIADTibbY0SsCPGglmIvdn0="; - aarch64-linux = "sha256-jUyIV2ysOxkIij/Ff5VGuIADTibbY0SsCPGglmIvdn0="; - aarch64-darwin = "sha256-XWYBTy3xZbQwJxdGbAK4+R69XNmL3RRyXu6+nY+XPXc="; + x86_64-linux = "sha256-6VGGHRgFjm5/RJ9ZY+39KMiFHsuWX+by37oWW6fQq/w="; + aarch64-linux = "sha256-6VGGHRgFjm5/RJ9ZY+39KMiFHsuWX+by37oWW6fQq/w="; + aarch64-darwin = "sha256-vCwExwVl5WmwbuhB7HxbZD0F7WSE7fS6PTXm4SJcn2U="; }; in { @@ -39,7 +39,7 @@ }; scorecard = pkgs.writeShellApplication { name = "scorecard"; - runtimeInputs = [ pkgs.deno pkgs.git ]; + runtimeInputs = [ pkgs.deno pkgs.git pkgs.bash ]; text = '' export SCORECARD_INSTRUMENT=${self} export SCORECARD_SANDBOX=${sandbox}/bin/sandbox @@ -50,7 +50,7 @@ export SCORECARD_PNPM_VERSION=${pkgs.pnpm_12.version} export DENO_NO_PACKAGE_JSON=1 exec deno run --no-config --allow-read --allow-write --allow-env --allow-sys=hostname \ - --allow-run=git,${sandbox}/bin/sandbox \ + --allow-run=git,bash,${pkgs.deno}/bin/deno,${sandbox}/bin/sandbox \ ${self}/src/main.ts "$@" ''; }; @@ -63,7 +63,7 @@ devShells = forEachSystem (pkgs: let system = pkgs.stdenv.hostPlatform.system; in { default = pkgs.mkShell { - packages = [ pkgs.nodejs_24 pkgs.pnpm_12 pkgs.deno self.packages.${system}.sandbox ]; + packages = [ pkgs.nodejs_24 pkgs.pnpm_12 pkgs.deno pkgs.actionlint self.packages.${system}.sandbox ]; SANDBOX_PNPM_STORE = self.packages.${system}.tools-store; }; }); diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/ratstack-static.json b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/ratstack-static.json new file mode 100644 index 0000000..e5ff4a3 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/ratstack-static.json @@ -0,0 +1,69 @@ +{ + "family": "static", + "wallMs": 3000, + "flags": [], + "definitionHashes": [ + { + "id": "M23", + "hash": "h-m23" + }, + { + "id": "M28", + "hash": "h-m28" + } + ], + "cells": [ + { + "id": "M23", + "side": "ratstack", + "measured": { + "cell": { + "_tag": "Measured", + "runs": [ + 219, + 219, + 219 + ] + }, + "provenance": { + "side": "ratstack", + "commit": "54d356037c994f89698760a4727be71d0005a087", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + } + }, + { + "id": "M28", + "side": "ratstack", + "measured": { + "cell": { + "_tag": "Measured", + "runs": [ + 957, + 957, + 957 + ] + }, + "provenance": { + "side": "ratstack", + "commit": "54d356037c994f89698760a4727be71d0005a087", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + } + } + ] +} diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/starter-static.json b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/starter-static.json new file mode 100644 index 0000000..c0b4cd7 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/families/starter-static.json @@ -0,0 +1,69 @@ +{ + "family": "static", + "wallMs": 3000, + "flags": [], + "definitionHashes": [ + { + "id": "M23", + "hash": "h-m23" + }, + { + "id": "M28", + "hash": "h-m28" + } + ], + "cells": [ + { + "id": "M23", + "side": "starter", + "measured": { + "cell": { + "_tag": "Measured", + "runs": [ + 219, + 219, + 219 + ] + }, + "provenance": { + "side": "starter", + "commit": "2222222222222222222222222222222222222222", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + } + }, + { + "id": "M28", + "side": "starter", + "measured": { + "cell": { + "_tag": "Measured", + "runs": [ + 392, + 392, + 392 + ] + }, + "provenance": { + "side": "starter", + "commit": "2222222222222222222222222222222222222222", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + } + } + ] +} diff --git a/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/main/scorecard.json b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/main/scorecard.json new file mode 100644 index 0000000..de5200b --- /dev/null +++ b/evals/ratstack-scorecard/journeys/__fixtures__/aggregate/main/scorecard.json @@ -0,0 +1,157 @@ +{ + "schemaVersion": 1, + "provenance": { + "commit": "3333333333333333333333333333333333333333", + "ratstackCommit": "54d356037c994f89698760a4727be71d0005a087", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "generatedAt": "2026-10-05T21:00:00.000Z" + }, + "rows": [ + { + "definition": { + "id": "M23", + "metric": "M23", + "bin": "fence: debt ledger", + "label": "Suppression directives in tracked source", + "unit": "directives", + "direction": "lower", + "kind": "count", + "runs": 3, + "family": "static", + "ratstackSupport": { + "_tag": "Required" + } + }, + "definitionHash": "h-m23", + "ratstack": { + "cell": { + "_tag": "Measured", + "runs": [ + 219, + 219, + 219 + ] + }, + "provenance": { + "side": "ratstack", + "commit": "54d356037c994f89698760a4727be71d0005a087", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + }, + "starter": { + "cell": { + "_tag": "Measured", + "runs": [ + 0, + 0, + 0 + ] + }, + "provenance": { + "side": "starter", + "commit": "3333333333333333333333333333333333333333", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + }, + "verdict": { + "_tag": "Beaten" + }, + "flags": [] + }, + { + "definition": { + "id": "M28", + "metric": "M28", + "bin": "install", + "label": "Distinct name@version packages in the side's lockfile", + "unit": "packages", + "direction": "lower", + "kind": "count", + "runs": 3, + "family": "static", + "ratstackSupport": { + "_tag": "Required" + } + }, + "definitionHash": "h-m28", + "ratstack": { + "cell": { + "_tag": "Measured", + "runs": [ + 957, + 957, + 957 + ] + }, + "provenance": { + "side": "ratstack", + "commit": "54d356037c994f89698760a4727be71d0005a087", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + }, + "starter": { + "cell": { + "_tag": "Measured", + "runs": [ + 392, + 392, + 392 + ] + }, + "provenance": { + "side": "starter", + "commit": "3333333333333333333333333333333333333333", + "instrumentHash": "1111111111111111111111111111111111111111", + "nixpkgsRev": "4975466d324710c576dc11ad614684e6bd8cad8e", + "runner": "GitHub Actions 1", + "measuredAt": "2026-10-05T22:00:00.000Z", + "tools": { + "oxc-parser": "0.152.0", + "yaml": "2.9.1" + } + } + }, + "verdict": { + "_tag": "Beaten" + }, + "flags": [] + } + ], + "ratchet": { + "_tag": "FirstBaseline", + "outcomes": [ + { + "_tag": "New", + "id": "M23" + }, + { + "_tag": "New", + "id": "M28" + } + ], + "failures": [] + } +} diff --git a/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts b/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts new file mode 100644 index 0000000..1931381 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/aggregate.journey.test.ts @@ -0,0 +1,22 @@ +import { Ajv } from 'ajv' +import { describe, expect, test } from 'vitest' +import schema from '../scorecard.schema.json' with { type: 'json' } +import { launcherRun } from './launcher-run.ts' + +const validate = new Ajv({ allErrors: true, strict: true }).compile(schema) + +describe('aggregate through the real CLI (J3)', () => { + test('a row beaten on main and tied on the PR fails the job, names the row, and still writes a valid scorecard', async () => { + const run = await launcherRun('aggregate-regressed') + expect(run.code).toBe(1) + expect(run.stderr).toContain('M23') + expect(run.stderr).not.toContain('M28') + expect(validate(JSON.parse(run.files['scorecard.json'] ?? 'null'))).toBe(true) + }) + + test('without a main artifact the same families are a passing first baseline', async () => { + const run = await launcherRun('aggregate-first-baseline') + expect(run.code).toBe(0) + expect(validate(JSON.parse(run.files['scorecard.json'] ?? 'null'))).toBe(true) + }) +}) diff --git a/evals/ratstack-scorecard/journeys/instrument-ref.test.ts b/evals/ratstack-scorecard/journeys/instrument-ref.test.ts new file mode 100644 index 0000000..99a1ee7 --- /dev/null +++ b/evals/ratstack-scorecard/journeys/instrument-ref.test.ts @@ -0,0 +1,42 @@ +import { Effect, Schema } from 'effect' +import { describe, expect, test } from 'vitest' +import { launcherRun } from './launcher-run.ts' + +const Selection = Schema.Struct({ + code: Schema.Number, + stdout: Schema.String, + stderr: Schema.String, + base: Schema.String, + head: Schema.String, + graded: Schema.NullOr(Schema.String), +}) + +const selection = async (name: string) => { + const run = await launcherRun('instrument-ref') + return Effect.runPromise(Schema.decodeUnknownEffect(Schema.fromJsonString(Selection))(run.files[`${name}.json`])) +} + +describe('which instrument grades a pull request (bootstrap ruling)', () => { + test('a base without the scorecard workflow is the only case that grades with the PR head, and says BOOTSTRAP', async () => { + const s = await selection('bootstrap') + expect(s.code).toBe(0) + expect(s.stdout).toBe(`ref=${s.head}\nmode=bootstrap\n`) + }) + + test.each(['edits-instrument', 'deletes-instrument', 'deletes-workflow'])( + 'when the base has the workflow, a PR that %s is still graded by the base instrument, so the base verdict (red) stands', + async (name) => { + const s = await selection(name) + expect(s.code).toBe(0) + expect(s.stdout).toBe(`ref=${s.base}\nmode=base\n`) + expect(s.graded).toBe('red') + }, + ) + + test('a base commit missing from the clone fails instead of falling back to bootstrap', async () => { + const s = await selection('base-missing') + expect(s.code).toBe(1) + expect(s.stdout).toBe('') + expect(s.stderr).toContain('is not in this clone') + }) +}) diff --git a/evals/ratstack-scorecard/journeys/manifest.json b/evals/ratstack-scorecard/journeys/manifest.json index 5f8ed77..69e59ee 100644 --- a/evals/ratstack-scorecard/journeys/manifest.json +++ b/evals/ratstack-scorecard/journeys/manifest.json @@ -29,6 +29,42 @@ "pnpm-lock.yaml", "flake.lock" ] + }, + { + "id": "aggregate-regressed", + "produce": { + "kind": "aggregate", + "families": "journeys/__fixtures__/aggregate/families", + "main": "journeys/__fixtures__/aggregate/main/scorecard.json" + }, + "inputs": [ + "journeys/__fixtures__/aggregate", + "src", + "decide" + ] + }, + { + "id": "aggregate-first-baseline", + "produce": { + "kind": "aggregate", + "families": "journeys/__fixtures__/aggregate/families", + "main": null + }, + "inputs": [ + "journeys/__fixtures__/aggregate", + "src", + "decide" + ] + }, + { + "id": "instrument-ref", + "produce": { + "kind": "instrument-ref" + }, + "inputs": [ + "ci/instrument-ref.sh", + "src/journeys.ts" + ] } ] } diff --git a/evals/ratstack-scorecard/package.json b/evals/ratstack-scorecard/package.json index 0ff3560..f18aa21 100644 --- a/evals/ratstack-scorecard/package.json +++ b/evals/ratstack-scorecard/package.json @@ -11,6 +11,11 @@ "ajv": "8.20.0", "fast-check": "4.10.2", "vitest": "5.0.3", - "effect": "4.0.1" + "effect": "4.0.1", + "typescript": "7.0.2", + "@types/node": "24.19.1" + }, + "scripts": { + "typecheck": "tsc -p tsconfig.json" } } diff --git a/evals/ratstack-scorecard/pnpm-lock.yaml b/evals/ratstack-scorecard/pnpm-lock.yaml index 3ee8937..4fa8b94 100644 --- a/evals/ratstack-scorecard/pnpm-lock.yaml +++ b/evals/ratstack-scorecard/pnpm-lock.yaml @@ -17,7 +17,10 @@ importers: devDependencies: '@fast-check/vitest': specifier: 0.5.0 - version: 0.5.0(vitest@5.0.3(vite@8.3.2(yaml@2.9.1))) + version: 0.5.0(vitest@5.0.3(@types/node@24.19.1)(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1))) + '@types/node': + specifier: 24.19.1 + version: 24.19.1 ajv: specifier: 8.20.0 version: 8.20.0 @@ -27,9 +30,12 @@ importers: fast-check: specifier: 4.10.2 version: 4.10.2 + typescript: + specifier: 7.0.2 + version: 7.0.2 vitest: specifier: 5.0.3 - version: 5.0.3(vite@8.3.2(yaml@2.9.1)) + version: 5.0.3(@types/node@24.19.1)(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1)) packages: @@ -281,6 +287,129 @@ packages: '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + '@types/node@24.19.1': + resolution: {integrity: sha512-aS3/DG0oM05K0RIXXP+hKjinGG5IgSSVGzswZxW3O0sS3pH4/fycXundUC9XsszgKCk4gHXylTEK6hyFxVxnoQ==} + + '@typescript/typescript-aix-ppc64@7.0.2': + resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} + engines: {node: '>=16.20.0'} + cpu: [ppc64] + os: [aix] + + '@typescript/typescript-darwin-arm64@7.0.2': + resolution: {integrity: sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [darwin] + + '@typescript/typescript-darwin-x64@7.0.2': + resolution: {integrity: sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [darwin] + + '@typescript/typescript-freebsd-arm64@7.0.2': + resolution: {integrity: sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [freebsd] + + '@typescript/typescript-freebsd-x64@7.0.2': + resolution: {integrity: sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [freebsd] + + '@typescript/typescript-linux-arm64@7.0.2': + resolution: {integrity: sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [linux] + + '@typescript/typescript-linux-arm@7.0.2': + resolution: {integrity: sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==} + engines: {node: '>=16.20.0'} + cpu: [arm] + os: [linux] + + '@typescript/typescript-linux-loong64@7.0.2': + resolution: {integrity: sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==} + engines: {node: '>=16.20.0'} + cpu: [loong64] + os: [linux] + + '@typescript/typescript-linux-mips64el@7.0.2': + resolution: {integrity: sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==} + engines: {node: '>=16.20.0'} + cpu: [mips64el] + os: [linux] + + '@typescript/typescript-linux-ppc64@7.0.2': + resolution: {integrity: sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==} + engines: {node: '>=16.20.0'} + cpu: [ppc64] + os: [linux] + + '@typescript/typescript-linux-riscv64@7.0.2': + resolution: {integrity: sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==} + engines: {node: '>=16.20.0'} + cpu: [riscv64] + os: [linux] + + '@typescript/typescript-linux-s390x@7.0.2': + resolution: {integrity: sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==} + engines: {node: '>=16.20.0'} + cpu: [s390x] + os: [linux] + + '@typescript/typescript-linux-x64@7.0.2': + resolution: {integrity: sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [linux] + + '@typescript/typescript-netbsd-arm64@7.0.2': + resolution: {integrity: sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [netbsd] + + '@typescript/typescript-netbsd-x64@7.0.2': + resolution: {integrity: sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [netbsd] + + '@typescript/typescript-openbsd-arm64@7.0.2': + resolution: {integrity: sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [openbsd] + + '@typescript/typescript-openbsd-x64@7.0.2': + resolution: {integrity: sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [openbsd] + + '@typescript/typescript-sunos-x64@7.0.2': + resolution: {integrity: sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [sunos] + + '@typescript/typescript-win32-arm64@7.0.2': + resolution: {integrity: sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==} + engines: {node: '>=16.20.0'} + cpu: [arm64] + os: [win32] + + '@typescript/typescript-win32-x64@7.0.2': + resolution: {integrity: sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==} + engines: {node: '>=16.20.0'} + cpu: [x64] + os: [win32] + '@vitest/mocker@5.0.3': resolution: {integrity: sha512-T8sWAIbkSyAjkwTcaEc3Iu0o9A27X1/kdXrizhZkGuSKScRQtRzclfAMpOTcGdXCsqxeWlpGy3XjqaW8CpLORg==} peerDependencies: @@ -482,6 +611,14 @@ packages: resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} engines: {node: '>=12.0.0'} + typescript@7.0.2: + resolution: {integrity: sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==} + engines: {node: '>=16.20.0'} + hasBin: true + + undici-types@7.24.6: + resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + vite@8.3.2: resolution: {integrity: sha512-SQr1x6W5vVSbROg7vsyXIaxK9b0G7zsT68acdWWRmnBUsgDieLCRG+Rep9WdZgcposvv/GSnr4GUUBqB3vXq6w==} engines: {node: ^20.19.0 || >=22.12.0} @@ -578,10 +715,10 @@ packages: snapshots: - '@fast-check/vitest@0.5.0(vitest@5.0.3(vite@8.3.2(yaml@2.9.1)))': + '@fast-check/vitest@0.5.0(vitest@5.0.3(@types/node@24.19.1)(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1)))': dependencies: fast-check: 4.10.2 - vitest: 5.0.3(vite@8.3.2(yaml@2.9.1)) + vitest: 5.0.3(@types/node@24.19.1)(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1)) '@jridgewell/resolve-uri@3.1.2': {} @@ -707,14 +844,78 @@ snapshots: '@types/estree@1.0.9': {} - '@vitest/mocker@5.0.3(vite@8.3.2(yaml@2.9.1))': + '@types/node@24.19.1': + dependencies: + undici-types: 7.24.6 + + '@typescript/typescript-aix-ppc64@7.0.2': + optional: true + + '@typescript/typescript-darwin-arm64@7.0.2': + optional: true + + '@typescript/typescript-darwin-x64@7.0.2': + optional: true + + '@typescript/typescript-freebsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-freebsd-x64@7.0.2': + optional: true + + '@typescript/typescript-linux-arm64@7.0.2': + optional: true + + '@typescript/typescript-linux-arm@7.0.2': + optional: true + + '@typescript/typescript-linux-loong64@7.0.2': + optional: true + + '@typescript/typescript-linux-mips64el@7.0.2': + optional: true + + '@typescript/typescript-linux-ppc64@7.0.2': + optional: true + + '@typescript/typescript-linux-riscv64@7.0.2': + optional: true + + '@typescript/typescript-linux-s390x@7.0.2': + optional: true + + '@typescript/typescript-linux-x64@7.0.2': + optional: true + + '@typescript/typescript-netbsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-netbsd-x64@7.0.2': + optional: true + + '@typescript/typescript-openbsd-arm64@7.0.2': + optional: true + + '@typescript/typescript-openbsd-x64@7.0.2': + optional: true + + '@typescript/typescript-sunos-x64@7.0.2': + optional: true + + '@typescript/typescript-win32-arm64@7.0.2': + optional: true + + '@typescript/typescript-win32-x64@7.0.2': + optional: true + + '@vitest/mocker@5.0.3(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1))': dependencies: '@jridgewell/trace-mapping': 0.3.31 '@vitest/spy': 5.0.3 estree-walker: 3.0.3 magic-string: 1.4.2 optionalDependencies: - vite: 8.3.2(yaml@2.9.1) + vite: 8.3.2(@types/node@24.19.1)(yaml@2.9.1) '@vitest/spy@5.0.3': {} @@ -887,7 +1088,32 @@ snapshots: fdir: 6.5.0(picomatch@4.0.7) picomatch: 4.0.7 - vite@8.3.2(yaml@2.9.1): + typescript@7.0.2: + optionalDependencies: + '@typescript/typescript-aix-ppc64': 7.0.2 + '@typescript/typescript-darwin-arm64': 7.0.2 + '@typescript/typescript-darwin-x64': 7.0.2 + '@typescript/typescript-freebsd-arm64': 7.0.2 + '@typescript/typescript-freebsd-x64': 7.0.2 + '@typescript/typescript-linux-arm': 7.0.2 + '@typescript/typescript-linux-arm64': 7.0.2 + '@typescript/typescript-linux-loong64': 7.0.2 + '@typescript/typescript-linux-mips64el': 7.0.2 + '@typescript/typescript-linux-ppc64': 7.0.2 + '@typescript/typescript-linux-riscv64': 7.0.2 + '@typescript/typescript-linux-s390x': 7.0.2 + '@typescript/typescript-linux-x64': 7.0.2 + '@typescript/typescript-netbsd-arm64': 7.0.2 + '@typescript/typescript-netbsd-x64': 7.0.2 + '@typescript/typescript-openbsd-arm64': 7.0.2 + '@typescript/typescript-openbsd-x64': 7.0.2 + '@typescript/typescript-sunos-x64': 7.0.2 + '@typescript/typescript-win32-arm64': 7.0.2 + '@typescript/typescript-win32-x64': 7.0.2 + + undici-types@7.24.6: {} + + vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1): dependencies: lightningcss: 1.33.0 picomatch: 4.0.7 @@ -895,13 +1121,14 @@ snapshots: rolldown: 1.2.12 tinyglobby: 0.2.17 optionalDependencies: + '@types/node': 24.19.1 fsevents: 2.3.3 yaml: 2.9.1 - vitest@5.0.3(vite@8.3.2(yaml@2.9.1)): + vitest@5.0.3(@types/node@24.19.1)(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1)): dependencies: '@types/chai': 5.2.3 - '@vitest/mocker': 5.0.3(vite@8.3.2(yaml@2.9.1)) + '@vitest/mocker': 5.0.3(vite@8.3.2(@types/node@24.19.1)(yaml@2.9.1)) chai: 6.3.0 es-module-lexer: 2.3.2 expect-type: 1.4.0 @@ -912,8 +1139,10 @@ snapshots: tinybench: 6.2.0 tinyexec: 1.3.1 tinyglobby: 0.2.17 - vite: 8.3.2(yaml@2.9.1) + vite: 8.3.2(@types/node@24.19.1)(yaml@2.9.1) why-is-node-running: 3.2.1 + optionalDependencies: + '@types/node': 24.19.1 transitivePeerDependencies: - msw diff --git a/evals/ratstack-scorecard/scorecard.schema.json b/evals/ratstack-scorecard/scorecard.schema.json index 62f1d3b..c9ba370 100644 --- a/evals/ratstack-scorecard/scorecard.schema.json +++ b/evals/ratstack-scorecard/scorecard.schema.json @@ -248,7 +248,18 @@ "type": "object", "additionalProperties": false, "required": ["_tag", "id"], - "properties": { "_tag": { "enum": ["Held", "New", "ReBaselined"] }, "id": { "type": "string" } } + "properties": { "_tag": { "enum": ["Held", "New"] }, "id": { "type": "string" } } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["_tag", "id", "mainHash", "prHash"], + "properties": { + "_tag": { "const": "ReBaselined" }, + "id": { "type": "string" }, + "mainHash": { "type": "string" }, + "prHash": { "type": "string" } + } }, { "type": "object", diff --git a/evals/ratstack-scorecard/src/families/static.ts b/evals/ratstack-scorecard/src/families/static.ts index 45b7414..3717ed2 100644 --- a/evals/ratstack-scorecard/src/families/static.ts +++ b/evals/ratstack-scorecard/src/families/static.ts @@ -1,6 +1,7 @@ import { join } from 'node:path' -import { arrayAt, numberAt, stringAt } from '../harness/decode.ts' +import { array, decodeJson, number, string, struct } from '../harness/decode.ts' import { + definitionHashesFor, type Instrument, materializeRatstack, materializeStarter, @@ -19,6 +20,12 @@ import { starter } from '../sides/starter.ts' const runs = 3 +const extracted = struct({ files: array(struct({ path: string, comments: array(string), errors: number })) }) + +const lockfileDocuments = struct({ + documents: array(struct({ lockfileVersion: string, packages: array(string), errors: number })), +}) + interface Subject { readonly adapter: SideAdapter readonly root: string @@ -77,15 +84,14 @@ const countDirectives = async ( if (result.code !== 0) { return instrumentError(`extract-comments exited ${result.code}: ${result.stderr.slice(-2000)}`) } - const parsed = JSON.parse(await Deno.readTextFile(output)) - const files = arrayAt(parsed, ['files'], 'extract-comments output') - const comments = files.flatMap((file) => arrayAt(file, ['comments'], 'extract-comments file').map(String)) + const { files } = decodeJson(extracted, await Deno.readTextFile(output), 'extract-comments output') + const comments = files.flatMap((file) => file.comments) const tally = tallyDirectives(comments) totals.push(totalDirectives(tally)) detail = { ...tally, filesCounted: counted.length, - parseErrors: files.reduce((sum, file) => sum + numberAt(file, ['errors'], 'extract-comments file'), 0), + parseErrors: files.reduce((sum, file) => sum + file.errors, 0), ...Object.fromEntries(excluded.map((v) => [`excluded:${v.root}`, `${v.files} files: ${v.reason}`])), } } @@ -105,13 +111,13 @@ const countPackages = async ( const output = join(work, `lockfile-${subject.adapter.side}-${run}.json`) const result = await node(instrument, work, tools, ['src/tools/parse-lockfile.mjs', lockfile, output]) if (result.code !== 0) return instrumentError(`parse-lockfile exited ${result.code}: ${result.stderr.slice(-2000)}`) - const documents = arrayAt(JSON.parse(await Deno.readTextFile(output)), ['documents'], 'parse-lockfile output') - const keys = documents.flatMap((doc) => arrayAt(doc, ['packages'], 'lockfile document').map(String)) + const { documents } = decodeJson(lockfileDocuments, await Deno.readTextFile(output), 'parse-lockfile output') + const keys = documents.flatMap((doc) => doc.packages) counts.push(distinctPackages(keys).length) detail = { lockfile: subject.adapter.lockfile, documents: documents.length, - lockfileVersions: documents.map((doc) => stringAt(doc, ['lockfileVersion'], 'lockfile document')).join(', '), + lockfileVersions: documents.map((doc) => doc.lockfileVersion).join(', '), } } return { cell: { _tag: 'Measured', runs: counts }, detail } @@ -138,5 +144,11 @@ export const measureStatic = async ( }, ) } - return { family: 'static', wallMs: Math.round(performance.now() - started), cells, flags: [] } + return { + family: 'static', + wallMs: Math.round(performance.now() - started), + cells, + flags: [], + definitionHashes: await definitionHashesFor(instrument, 'static'), + } } diff --git a/evals/ratstack-scorecard/src/harness/decide.ts b/evals/ratstack-scorecard/src/harness/decide.ts new file mode 100644 index 0000000..a45b4e8 --- /dev/null +++ b/evals/ratstack-scorecard/src/harness/decide.ts @@ -0,0 +1,49 @@ +import { dirname, join } from 'node:path' +import { type Instrument, walkFiles } from './instrument.ts' +import { runSandboxed } from './sandbox.ts' + +const generated = ['node_modules/', '.cache/', 'journeys/__records__/'] + +const githubHosts = ['api.github.com', '*.blob.core.windows.net'] + +export interface DecideWork { + readonly instrument: Instrument + readonly work: string +} + +export const prepareDecide = async (instrument: Instrument, work: string): Promise => { + const copy = join(work, 'instrument') + for (const file of await walkFiles(instrument.dir)) { + if (generated.some((prefix) => file.startsWith(prefix))) continue + await Deno.mkdir(dirname(join(copy, file)), { recursive: true }) + await Deno.writeFile(join(copy, file), await Deno.readFile(join(instrument.dir, file))) + } + const install = await runSandboxed(instrument.launcher, { + project: work, + cwd: copy, + command: ['pnpm', 'install', '--frozen-lockfile'], + pnpmStore: instrument.toolsStore, + deadlineMs: 10 * 60_000, + }) + if (install.code !== 0) throw new Error(`installing the decide step's dependencies failed:\n${install.stderr}`) + return { instrument, work } +} + +export const decide = async ( + { instrument, work }: DecideWork, + args: readonly string[], + options: { readonly github: boolean }, +): Promise => { + const token = Deno.env.get('GITHUB_TOKEN') + const result = await runSandboxed(instrument.launcher, { + project: work, + cwd: work, + command: ['node', 'instrument/decide/main.ts', ...args], + ...(options.github ? { allowHosts: githubHosts } : {}), + ...(options.github && token !== undefined ? { env: { GITHUB_TOKEN: token } } : {}), + deadlineMs: 5 * 60_000, + }) + await Deno.stdout.write(new TextEncoder().encode(result.stdout)) + await Deno.stderr.write(new TextEncoder().encode(result.stderr)) + return result.code +} diff --git a/evals/ratstack-scorecard/src/harness/decode.ts b/evals/ratstack-scorecard/src/harness/decode.ts index e271455..7a277e6 100644 --- a/evals/ratstack-scorecard/src/harness/decode.ts +++ b/evals/ratstack-scorecard/src/harness/decode.ts @@ -1,41 +1,77 @@ -export const at = (value: unknown, key: string): unknown => - typeof value === 'object' && value !== null && key in value - ? Object.getOwnPropertyDescriptor(value, key)?.value - : undefined +export type Decoder = (value: unknown, path: string) => T -const pathOf = (value: unknown, path: readonly string[]): unknown => path.reduce(at, value) - -const refuse = (what: string, path: readonly string[], expected: string): never => { - throw new Error(`${what}: expected ${expected} at ${path.join('.') || ''}`) +export class DecodeError extends Error { + constructor(readonly path: string, readonly expected: string, readonly found: unknown) { + super(`expected ${expected} at ${path || ''}, found ${JSON.stringify(found)?.slice(0, 80)}`) + } } -export const stringAt = (value: unknown, path: readonly string[], what: string): string => { - const found = pathOf(value, path) - return typeof found === 'string' ? found : refuse(what, path, 'a string') +const fail = (path: string, expected: string, found: unknown): never => { + throw new DecodeError(path, expected, found) } -export const numberAt = (value: unknown, path: readonly string[], what: string): number => { - const found = pathOf(value, path) - return typeof found === 'number' ? found : refuse(what, path, 'a number') -} +const isObject = (value: unknown): value is object => + typeof value === 'object' && value !== null && !Array.isArray(value) -export const arrayAt = (value: unknown, path: readonly string[], what: string): readonly unknown[] => { - const found = pathOf(value, path) - return Array.isArray(found) ? found : refuse(what, path, 'an array') -} +const fieldOf = (value: object, key: string): unknown => Object.getOwnPropertyDescriptor(value, key)?.value + +export const string: Decoder = (value, path) => + typeof value === 'string' ? value : fail(path, 'a string', value) + +export const number: Decoder = (value, path) => + typeof value === 'number' && Number.isFinite(value) ? value : fail(path, 'a finite number', value) + +export const literal = (expected: L): Decoder => (value, path) => + value === expected ? expected : fail(path, JSON.stringify(expected), value) + +export const oneOf = (options: readonly L[]): Decoder => (value, path) => + options.find((option) => option === value) ?? fail(path, `one of ${options.join(', ')}`, value) + +export const array = (item: Decoder): Decoder => (value, path) => + Array.isArray(value) ? value.map((entry, index) => item(entry, `${path}[${index}]`)) : fail(path, 'an array', value) + +export const tuple2 = (first: Decoder, second: Decoder): Decoder => (value, path) => + Array.isArray(value) && value.length === 2 + ? [first(value[0], `${path}[0]`), second(value[1], `${path}[1]`)] + : fail(path, 'a pair', value) + +export const record = (entry: Decoder): Decoder>> => (value, path) => + isObject(value) + ? Object.fromEntries(Object.entries(value).map(([key, item]) => [key, entry(item, `${path}.${key}`)])) + : fail(path, 'an object', value) + +type Fields = Readonly>> +type Decoded = { readonly [K in keyof F]: F[K] extends Decoder ? T : never } + +export const struct = (fields: F): Decoder> => (value, path) => + isObject(value) + ? Object.fromEntries( + Object.entries(fields).map(([key, decode]) => [key, decode(fieldOf(value, key), `${path}.${key}`)]), + ) as Decoded + : fail(path, 'an object', value) + +export const optional = (decode: Decoder): Decoder => (value, path) => + value === undefined ? undefined : decode(value, path) + +export const nullable = (decode: Decoder): Decoder => (value, path) => + value === null ? null : decode(value, path) -export const entriesAt = ( - value: unknown, - path: readonly string[], - what: string, -): readonly (readonly [string, unknown])[] => { - const found = pathOf(value, path) - return typeof found === 'object' && found !== null && !Array.isArray(found) - ? Object.entries(found) - : refuse(what, path, 'an object') +export const union = (...options: readonly Decoder[]): Decoder => (value, path) => { + const errors: string[] = [] + for (const option of options) { + try { + return option(value, path) + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)) + } + } + return fail(path, `one of ${options.length} shapes (${errors.join('; ')})`, value) } -export const optionalEntriesAt = (value: unknown, path: readonly string[]): readonly (readonly [string, unknown])[] => { - const found = pathOf(value, path) - return typeof found === 'object' && found !== null && !Array.isArray(found) ? Object.entries(found) : [] +export const decodeJson = (decode: Decoder, text: string, what: string): T => { + try { + return decode(JSON.parse(text), what) + } catch (error) { + throw new Error(`${what}: ${error instanceof Error ? error.message : String(error)}`) + } } diff --git a/evals/ratstack-scorecard/src/harness/instrument.ts b/evals/ratstack-scorecard/src/harness/instrument.ts index 7393b3f..eaad545 100644 --- a/evals/ratstack-scorecard/src/harness/instrument.ts +++ b/evals/ratstack-scorecard/src/harness/instrument.ts @@ -1,6 +1,7 @@ import { dirname, join } from 'node:path' -import type { CellProvenance, Side } from '../model/cell.ts' -import { optionalEntriesAt, stringAt } from './decode.ts' +import { rowDefinitions } from '../metrics/registry.ts' +import type { CellProvenance, Family, Side } from '../model/cell.ts' +import { decodeJson, type Decoder, record, string, struct } from './decode.ts' import { type Launcher, runSandboxed } from './sandbox.ts' export interface Pin { @@ -43,19 +44,31 @@ export const git = async (cwd: string, args: readonly string[]): Promise => decoder.decode(await git(cwd, args)).trim() -const readJson = async (path: string): Promise => JSON.parse(await Deno.readTextFile(path)) +const readJson = async (decode: Decoder, path: string): Promise => + decodeJson(decode, await Deno.readTextFile(path), path) -const nixpkgsRevOf = (lock: unknown): string => stringAt(lock, ['nodes', 'nixpkgs', 'locked', 'rev'], 'flake.lock') +const flakeLock = struct({ nodes: struct({ nixpkgs: struct({ locked: struct({ rev: string }) }) }) }) -const pinOf = (pin: unknown): Pin => ({ - owner: stringAt(pin, ['owner'], 'ratstack.pin.json'), - repo: stringAt(pin, ['repo'], 'ratstack.pin.json'), - commit: stringAt(pin, ['commit'], 'ratstack.pin.json'), - narHash: stringAt(pin, ['narHash'], 'ratstack.pin.json'), -}) +const pinFile = struct({ owner: string, repo: string, commit: string, narHash: string }) + +const manifest = struct({ dependencies: record(string) }) + +const sha256Bytes = async (bytes: BufferSource): Promise => + [...new Uint8Array(await crypto.subtle.digest('SHA-256', bytes))] + .map((byte) => byte.toString(16).padStart(2, '0')) + .join('') -const dependencyVersions = (manifest: unknown): Record => - Object.fromEntries(optionalEntriesAt(manifest, ['dependencies']).map(([name, version]) => [name, String(version)])) +const sha256 = (text: string): Promise => sha256Bytes(new TextEncoder().encode(text)) + +const generated = ['node_modules/', '.cache/', 'journeys/__records__/'] + +const instrumentTreeHash = async (dir: string): Promise => { + const files = (await walkFiles(dir)).filter((file) => !generated.some((prefix) => file.startsWith(prefix))).sort() + const parts = await Promise.all( + files.map(async (file) => `${file}\0${await sha256Bytes(await Deno.readFile(join(dir, file)))}`), + ) + return sha256(parts.join('\n')) +} export const loadInstrument = async (checkoutArg: string | undefined): Promise => { const dir = required('SCORECARD_INSTRUMENT') @@ -66,13 +79,13 @@ export const loadInstrument = async (checkoutArg: string | undefined): Promise => { + const measurement = await Deno.readTextFile(join(instrument.dir, 'src/families', `${family}.ts`)) + return await Promise.all( + rowDefinitions + .filter((row) => row.family === family) + .map(async (row) => ({ id: row.id, hash: await sha256(JSON.stringify({ row, measurement })) })), + ) +} + const copyTracked = async (from: string, to: string, files: readonly string[]): Promise => { for (const file of files) { const target = join(to, file) diff --git a/evals/ratstack-scorecard/src/journeys.ts b/evals/ratstack-scorecard/src/journeys.ts index 475e108..55f8184 100644 --- a/evals/ratstack-scorecard/src/journeys.ts +++ b/evals/ratstack-scorecard/src/journeys.ts @@ -1,6 +1,6 @@ import { join, relative } from 'node:path' import { measureStatic } from './families/static.ts' -import { arrayAt, at, stringAt } from './harness/decode.ts' +import { array, decodeJson, type Decoder, literal, nullable, string, struct, union } from './harness/decode.ts' import { git, type Instrument, walkFiles } from './harness/instrument.ts' import { runSandboxed } from './harness/sandbox.ts' @@ -13,10 +13,14 @@ interface MeasureStatic { interface JourneyEntry { readonly id: string - readonly produce: MeasureStatic + readonly produce: MeasureStatic | Aggregate | InstrumentRef readonly inputs: readonly string[] } +interface InstrumentRef { + readonly kind: 'instrument-ref' +} + export interface LauncherRecord { readonly id: string readonly inputHash: string @@ -29,20 +33,22 @@ export interface LauncherRecord { readonly wallMs: number } -const journeyEntry = (value: unknown): JourneyEntry => { - const produce = at(value, 'produce') - const failingTool = at(produce, 'failingTool') - return { - id: stringAt(value, ['id'], 'journey'), - produce: { - kind: 'measure-static', - ratstackRepo: stringAt(produce, ['ratstackRepo'], 'journey produce'), - starterRepo: stringAt(produce, ['starterRepo'], 'journey produce'), - failingTool: failingTool === null ? null : stringAt(produce, ['failingTool'], 'journey produce'), - }, - inputs: arrayAt(value, ['inputs'], 'journey').map((input) => stringAt({ input }, ['input'], 'journey input')), - } -} +const journeyEntry: Decoder = struct({ + id: string, + produce: union( + struct({ + kind: literal('measure-static'), + ratstackRepo: string, + starterRepo: string, + failingTool: nullable(string), + }), + struct({ kind: literal('aggregate'), families: string, main: nullable(string) }), + struct({ kind: literal('instrument-ref') }), + ), + inputs: array(string), +}) + +const manifestOf = struct({ journeys: array(journeyEntry) }) const encoder = new TextEncoder() @@ -91,17 +97,85 @@ const overlayWithFailingTool = async (instrument: Instrument, work: string, tool return dir } -const produce = async (instrument: Instrument, root: string, entry: JourneyEntry): Promise => { +interface Aggregate { + readonly kind: 'aggregate' + readonly families: string + readonly main: string | null +} + +const produceAggregate = async ( + instrument: Instrument, + root: string, + entry: JourneyEntry, + spec: Aggregate, +): Promise => { + await Deno.mkdir(join(root, '.cache'), { recursive: true }) + const out = await Deno.makeTempDir({ dir: join(root, '.cache'), prefix: `journey-${entry.id}-` }) + const started = performance.now() + try { + const argv = [ + 'aggregate', + '--families', + join(root, spec.families), + ...(spec.main === null ? [] : ['--main', join(root, spec.main)]), + '--out', + join(out, 'scorecard.json'), + '--summary', + join(out, 'summary.md'), + ] + const run = await new Deno.Command(Deno.execPath(), { + args: [ + 'run', + '--no-config', + '--allow-read', + '--allow-write', + '--allow-env', + '--allow-sys=hostname', + `--allow-run=git,${instrument.launcher.executable}`, + join(root, 'src/main.ts'), + ...argv, + ], + env: { ...Deno.env.toObject(), DENO_NO_PACKAGE_JSON: '1' }, + stdout: 'piped', + stderr: 'piped', + }).output() + const files: Record = {} + for (const name of ['scorecard.json', 'summary.md']) { + const text = await Deno.readTextFile(join(out, name)).catch(() => undefined) + if (text !== undefined) files[name] = text + } + return { + id: entry.id, + inputHash: '', + argv: ['scorecard', ...argv.map((arg) => arg.startsWith(out) ? arg.slice(out.length + 1) : arg)], + code: run.code, + stdout: new TextDecoder().decode(run.stdout), + stderr: new TextDecoder().decode(run.stderr), + files, + egressLog: null, + wallMs: Math.round(performance.now() - started), + } + } finally { + await Deno.remove(out, { recursive: true }) + } +} + +const produceStatic = async ( + instrument: Instrument, + root: string, + entry: JourneyEntry, + spec: MeasureStatic, +): Promise => { const work = await Deno.makeTempDir({ prefix: `journey-${entry.id}-` }) const started = performance.now() try { - const ratstackSrc = await gitRepoFrom(join(root, entry.produce.ratstackRepo), join(work, 'sources/ratstack')) - const checkout = await gitRepoFrom(join(root, entry.produce.starterRepo), join(work, 'sources/starter-checkout')) + const ratstackSrc = await gitRepoFrom(join(root, spec.ratstackRepo), join(work, 'sources/ratstack')) + const checkout = await gitRepoFrom(join(root, spec.starterRepo), join(work, 'sources/starter-checkout')) const project = join(work, 'project') await Deno.mkdir(project) - const dir = entry.produce.failingTool === null + const dir = spec.failingTool === null ? instrument.dir - : await overlayWithFailingTool(instrument, work, entry.produce.failingTool) + : await overlayWithFailingTool(instrument, work, spec.failingTool) const subject: Instrument = { ...instrument, dir, @@ -112,7 +186,7 @@ const produce = async (instrument: Instrument, root: string, entry: JourneyEntry const result = await measureStatic(subject, project, ['ratstack', 'starter']) return { id: entry.id, - inputHash: await inputHashOf(root, entry.inputs, instrument.launcher.executable), + inputHash: '', argv: ['scorecard', 'measure', '--family', 'static'], code: 0, stdout: '', @@ -126,10 +200,101 @@ const produce = async (instrument: Instrument, root: string, entry: JourneyEntry } } +const identity = ['-c', 'user.name=journey', '-c', 'user.email=journey@invalid', '-c', 'commit.gpgsign=false'] + +const commitTree = async (repo: string, files: Readonly>, message: string) => { + for (const [path, content] of Object.entries(files)) { + if (content === null) await Deno.remove(join(repo, path), { recursive: true }) + else { + await Deno.mkdir(join(repo, path, '..'), { recursive: true }) + await Deno.writeTextFile(join(repo, path), content) + } + } + await git(repo, ['add', '-A']) + await git(repo, [...identity, 'commit', '-q', '--allow-empty', '-m', message]) + return new TextDecoder().decode(await git(repo, ['rev-parse', 'HEAD'])).trim() +} + +const produceInstrumentRef = async (root: string, entry: JourneyEntry): Promise => { + const repo = await Deno.makeTempDir({ prefix: `journey-${entry.id}-` }) + const started = performance.now() + try { + await git(repo, ['init', '-q', '-b', 'main']) + const workflow = '.github/workflows/scorecard.yml' + const verdict = 'evals/ratstack-scorecard/verdict' + const baseWithout = await commitTree(repo, { 'README.md': 'starter\n' }, 'base without the scorecard') + const introduce = await commitTree(repo, { [workflow]: 'name: Scorecard\n', [verdict]: 'green\n' }, 'add scorecard') + const baseWith = await commitTree(repo, { [verdict]: 'red\n' }, 'main instrument grades the row red') + const editsInstrument = await commitTree(repo, { [verdict]: 'green\n' }, 'PR edits the instrument to green') + await git(repo, ['checkout', '-q', baseWith]) + const deletesInstrument = await commitTree(repo, { 'evals/ratstack-scorecard': null }, 'PR deletes the instrument') + await git(repo, ['checkout', '-q', baseWith]) + const deletesWorkflow = await commitTree( + repo, + { [workflow]: null, [verdict]: 'green\n' }, + 'PR deletes the workflow', + ) + const cases: Readonly> = { + 'bootstrap': [baseWithout, introduce], + 'edits-instrument': [baseWith, editsInstrument], + 'deletes-instrument': [baseWith, deletesInstrument], + 'deletes-workflow': [baseWith, deletesWorkflow], + 'base-missing': ['0'.repeat(40), editsInstrument], + } + const files: Record = {} + for (const [name, [base, head]] of Object.entries(cases)) { + const run = await new Deno.Command('bash', { + args: [join(root, 'ci/instrument-ref.sh'), base, head], + cwd: repo, + stdout: 'piped', + stderr: 'piped', + }).output() + const stdout = new TextDecoder().decode(run.stdout) + const ref = /^ref=(\S+)$/mu.exec(stdout)?.[1] + const show = ref === undefined + ? undefined + : await new Deno.Command('git', { args: ['show', `${ref}:${verdict}`], cwd: repo, stdout: 'piped' }).output() + files[`${name}.json`] = JSON.stringify({ + code: run.code, + stdout, + stderr: new TextDecoder().decode(run.stderr), + base, + head, + graded: show?.success === true ? new TextDecoder().decode(show.stdout).trim() : null, + }) + } + return { + id: entry.id, + inputHash: '', + argv: ['bash', 'ci/instrument-ref.sh', '', ''], + code: 0, + stdout: '', + stderr: '', + files, + egressLog: null, + wallMs: Math.round(performance.now() - started), + } + } finally { + await Deno.remove(repo, { recursive: true }) + } +} + +const produce = async (instrument: Instrument, root: string, entry: JourneyEntry): Promise => { + const spec = entry.produce + const record = spec.kind === 'aggregate' + ? await produceAggregate(instrument, root, entry, spec) + : spec.kind === 'instrument-ref' + ? await produceInstrumentRef(root, entry) + : await produceStatic(instrument, root, entry, spec) + return { ...record, inputHash: await inputHashOf(root, entry.inputs, instrument.launcher.executable) } +} + export const runJourneys = async (instrument: Instrument, root: string): Promise => { - const manifest = arrayAt(JSON.parse(await Deno.readTextFile(join(root, 'journeys/manifest.json'))), [ - 'journeys', - ], 'journeys/manifest.json').map(journeyEntry) + const { journeys: manifest } = decodeJson( + manifestOf, + await Deno.readTextFile(join(root, 'journeys/manifest.json')), + 'journeys/manifest.json', + ) const records = join(root, 'journeys/__records__') await Deno.remove(records, { recursive: true }).catch(() => undefined) await Deno.mkdir(records, { recursive: true }) diff --git a/evals/ratstack-scorecard/src/main.ts b/evals/ratstack-scorecard/src/main.ts index 2a1ca1f..92e201b 100644 --- a/evals/ratstack-scorecard/src/main.ts +++ b/evals/ratstack-scorecard/src/main.ts @@ -2,27 +2,42 @@ import { join } from 'node:path' import { parseArgs } from 'node:util' import { checkImports } from './check-imports.ts' import { measureStatic } from './families/static.ts' -import { type Instrument, loadInstrument } from './harness/instrument.ts' +import { decide, type DecideWork, prepareDecide } from './harness/decide.ts' +import { type Instrument, loadInstrument, walkFiles } from './harness/instrument.ts' import { runJourneys } from './journeys.ts' -import type { FamilyResult, Side } from './model/cell.ts' +import { ratstackCacheKey } from './model/cache-key.ts' +import type { Family, FamilyResult, Side } from './model/cell.ts' const usage = [ 'usage:', + ' scorecard ci [--checkout ] --cache --out [--main ] [--title ]', ' scorecard measure --family [--side ratstack|starter] [--out ] [--checkout ]', + ' scorecard aggregate --families [--main ] --out --summary ', + ' scorecard pin check', + ' scorecard pin write --commit --nar-hash [--checkout ]', ' scorecard journeys [--checkout ]', ' scorecard check [--checkout ] (import-graph rules, then journeys)', ].join('\n') -const families: Readonly< - Record Promise> -> = { - static: measureStatic, +type FamilyRunner = (instrument: Instrument, work: string, sides: readonly Side[]) => Promise + +const familyRunners: readonly (readonly [Family, FamilyRunner])[] = [['static', measureStatic]] + +const implementedFamilies: readonly Family[] = familyRunners.map(([family]) => family) + +const fail = (message: string): never => { + throw new Error(`${message}\n${usage}`) +} + +const writeOut = async (path: string | undefined, text: string): Promise => { + if (path === undefined) await Deno.stdout.write(new TextEncoder().encode(text)) + else await Deno.writeTextFile(path, text) } const sidesOf = (side: string | undefined): readonly Side[] => { if (side === undefined) return ['ratstack', 'starter'] if (side === 'ratstack' || side === 'starter') return [side] - throw new Error(`unknown side ${side}\n${usage}`) + return fail(`unknown side ${side}`) } const measure = async (args: readonly string[]): Promise => { @@ -36,15 +51,13 @@ const measure = async (args: readonly string[]): Promise => { }, strict: true, }) - const run = families[values.family ?? ''] - if (run === undefined) throw new Error(`unknown family ${values.family}\n${usage}`) + const family = values.family ?? fail('--family is required') + const run = familyRunners.find(([name]) => name === family)?.[1] ?? fail(`unknown or unbuilt family ${family}`) const instrument = await loadInstrument(values.checkout) - const work = await Deno.makeTempDir({ prefix: `scorecard-${values.family}-` }) + const work = await Deno.makeTempDir({ prefix: `scorecard-${family}-` }) try { - const result: FamilyResult = await run(instrument, work, sidesOf(values.side)) - const json = `${JSON.stringify(result, null, 2)}\n` - if (values.out === undefined) await Deno.stdout.write(new TextEncoder().encode(json)) - else await Deno.writeTextFile(values.out, json) + const result = await run(instrument, work, sidesOf(values.side)) + await writeOut(values.out, `${JSON.stringify(result, null, 2)}\n`) } finally { await Deno.remove(work, { recursive: true }) } @@ -67,7 +80,196 @@ const check = async (args: readonly string[]): Promise => { Deno.exit(await runJourneys(instrument, root)) } -const commands: Readonly Promise>> = { measure, journeys, check } +const exists = (path: string): Promise => Deno.stat(path).then(() => true, () => false) + +const copyInto = async (from: string, to: string): Promise => { + await Deno.mkdir(join(to, '..'), { recursive: true }) + await Deno.copyFile(from, to) +} + +const withDecide = async (instrument: Instrument, run: (work: DecideWork) => Promise): Promise => { + const work = await Deno.makeTempDir({ prefix: 'scorecard-decide-' }) + try { + return await run(await prepareDecide(instrument, work)) + } finally { + await Deno.remove(work, { recursive: true }) + } +} + +const aggregateIn = (work: DecideWork, instrument: Instrument, main: boolean): Promise => + decide(work, [ + 'aggregate', + '--families', + 'families', + '--implemented', + implementedFamilies.join(','), + '--commit', + Deno.env.get('GITHUB_SHA') ?? instrument.starterCommit, + ...(main ? ['--main', 'main/scorecard.json'] : []), + '--out', + 'out/scorecard.json', + '--summary', + 'out/summary.md', + ], { github: false }) + +const aggregate = async (args: readonly string[]): Promise => { + const { values } = parseArgs({ + args: [...args], + options: { + families: { type: 'string' }, + main: { type: 'string' }, + out: { type: 'string' }, + summary: { type: 'string' }, + }, + strict: true, + }) + const families = values.families ?? fail('--families is required') + const out = values.out ?? fail('--out is required') + const summary = values.summary ?? fail('--summary is required') + const instrument = await loadInstrument(undefined) + Deno.exit( + await withDecide(instrument, async (work) => { + for (const file of await walkFiles(families)) { + await copyInto(join(families, file), join(work.work, 'families', file)) + } + if (values.main !== undefined) await copyInto(values.main, join(work.work, 'main/scorecard.json')) + const code = await aggregateIn(work, instrument, values.main !== undefined) + if (await exists(join(work.work, 'out/scorecard.json'))) { + await copyInto(join(work.work, 'out/scorecard.json'), out) + } + if (await exists(join(work.work, 'out/summary.md'))) await copyInto(join(work.work, 'out/summary.md'), summary) + return code + }), + ) +} + +const ci = async (args: readonly string[]): Promise => { + const { values } = parseArgs({ + args: [...args], + options: { + checkout: { type: 'string' }, + cache: { type: 'string' }, + out: { type: 'string' }, + main: { type: 'string' }, + title: { type: 'string' }, + }, + strict: true, + }) + const cache = values.cache ?? fail('--cache is required') + const out = values.out ?? fail('--out is required') + const pullRequest = Deno.env.get('GITHUB_EVENT_NAME') === 'pull_request' + const instrument = await loadInstrument(values.checkout) + await Deno.mkdir(cache, { recursive: true }) + const code = await withDecide(instrument, async (work) => { + const families = join(work.work, 'families') + await Deno.mkdir(families, { recursive: true }) + const keys: string[] = [] + for (const [family, run] of familyRunners) { + const key = ratstackCacheKey({ + family, + ratstackCommit: instrument.pin.commit, + instrumentHash: instrument.instrumentHash, + nixpkgsRev: instrument.nixpkgsRev, + }).replaceAll('/', '__') + keys.push(`${key}.json`) + const cached = join(cache, `${key}.json`) + const ratstackFile = join(families, `ratstack-${family}.json`) + const usable = await exists(cached) && + (await copyInto(cached, ratstackFile), + (await decide(work, ['cache-check', '--file', `families/ratstack-${family}.json`], { github: false })) === 0) + if (!usable) { + if (await exists(cached)) console.error(`ci: cached rat-stack ${family} side refused; measuring it again`) + await measureInto(instrument, run, family, ['ratstack'], ratstackFile) + const savable = !pullRequest && + (await decide(work, ['cache-check', '--file', `families/ratstack-${family}.json`], { github: false })) === 0 + if (savable) await copyInto(ratstackFile, cached) + } + await measureInto(instrument, run, family, ['starter'], join(families, `starter-${family}.json`)) + } + if (!pullRequest) { + for (const file of await walkFiles(cache)) if (!keys.includes(file)) await Deno.remove(join(cache, file)) + } + const main = values.main !== undefined + ? (await copyInto(values.main, join(work.work, 'main/scorecard.json')), true) + : (await Deno.mkdir(join(work.work, 'main'), { recursive: true }), + (await decide(work, [ + 'main-scorecard', + '--repository', + Deno.env.get('GITHUB_REPOSITORY') ?? fail('GITHUB_REPOSITORY is unset'), + '--out', + 'main/scorecard.json', + ], { github: true })) === 0 || fail("finding main's scorecard failed"), + await exists(join(work.work, 'main/scorecard.json'))) + const verdict = await aggregateIn(work, instrument, main) + await Deno.mkdir(out, { recursive: true }) + for (const file of ['scorecard.json', 'summary.md']) { + if (await exists(join(work.work, 'out', file))) await copyInto(join(work.work, 'out', file), join(out, file)) + } + return verdict + }) + const summaryFile = Deno.env.get('GITHUB_STEP_SUMMARY') + if (summaryFile !== undefined && await exists(join(out, 'summary.md'))) { + const title = values.title === undefined ? '' : `## ${values.title}\n\n` + await Deno.writeTextFile(summaryFile, `${title}${await Deno.readTextFile(join(out, 'summary.md'))}\n`, { + append: true, + }) + } + Deno.exit(code) +} + +const measureInto = async ( + instrument: Instrument, + run: FamilyRunner, + family: Family, + sides: readonly Side[], + file: string, +): Promise => { + const work = await Deno.makeTempDir({ prefix: `scorecard-${family}-` }) + try { + await Deno.writeTextFile(file, `${JSON.stringify(await run(instrument, work, sides), null, 2)}\n`) + } finally { + await Deno.remove(work, { recursive: true }) + } +} + +const pinCommand = async (args: readonly string[]): Promise => { + const [action, ...rest] = args + const { values } = parseArgs({ + args: rest, + options: { 'commit': { type: 'string' }, 'nar-hash': { type: 'string' }, 'checkout': { type: 'string' } }, + strict: true, + }) + const instrument = await loadInstrument(values.checkout) + if (action === 'check') { + const { owner, repo, commit } = instrument.pin + Deno.exit( + await withDecide( + instrument, + (work) => decide(work, ['pin-check', '--owner', owner, '--repo', repo, '--pinned', commit], { github: true }), + ), + ) + } + if (action === 'write') { + const pin = { + ...instrument.pin, + commit: values.commit ?? fail('--commit is required'), + narHash: values['nar-hash'] ?? fail('--nar-hash is required'), + } + const path = join(instrument.checkout, 'evals/ratstack-scorecard/ratstack.pin.json') + await Deno.writeTextFile(path, `${JSON.stringify(pin, null, 2)}\n`) + return + } + fail(`unknown pin action ${action}`) +} + +const commands: Readonly Promise>> = { + 'ci': ci, + 'measure': measure, + 'aggregate': aggregate, + 'pin': pinCommand, + 'journeys': journeys, + 'check': check, +} const [command, ...rest] = Deno.args const handler = commands[command ?? ''] diff --git a/evals/ratstack-scorecard/src/metrics/registry.ts b/evals/ratstack-scorecard/src/metrics/registry.ts index d1e364b..5f85495 100644 --- a/evals/ratstack-scorecard/src/metrics/registry.ts +++ b/evals/ratstack-scorecard/src/metrics/registry.ts @@ -1,19 +1,10 @@ -import type { Family, RowDefinition } from '../model/cell.ts' +import type { RowDefinition } from '../model/cell.ts' const count3 = { kind: 'count', runs: 3 } as const const measured5 = { kind: 'measurement', runs: 5 } as const const measured3 = { kind: 'measurement', runs: 3 } as const const required = { ratstackSupport: { _tag: 'Required' } } as const -export const familyTimeoutMinutes: Readonly> = { - 'static': 20, - 'cold-path': 120, - 'gate-mutation': 240, - 'running-stack': 90, - 'agent-surfaces': 90, - 'networked': 60, -} - export const rowDefinitions: readonly RowDefinition[] = [ { id: 'M1.checks', diff --git a/evals/ratstack-scorecard/src/model/cache-key.property.test.ts b/evals/ratstack-scorecard/src/model/cache-key.property.test.ts new file mode 100644 index 0000000..824088a --- /dev/null +++ b/evals/ratstack-scorecard/src/model/cache-key.property.test.ts @@ -0,0 +1,25 @@ +import { fc, test } from '@fast-check/vitest' +import { describe } from 'vitest' +import { ratstackCacheKey } from './cache-key.ts' +import { sha } from './scorecard.arbitrary.ts' + +const family = fc.constantFrom( + 'static' as const, + 'cold-path' as const, + 'gate-mutation' as const, + 'running-stack' as const, + 'agent-surfaces' as const, + 'networked' as const, +) + +const inputs = fc.record({ family, ratstackCommit: sha, instrumentHash: sha, nixpkgsRev: sha }) + +describe('ratstackCacheKey', () => { + test.prop([inputs, fc.constantFrom('family', 'ratstackCommit', 'instrumentHash', 'nixpkgsRev' as const), inputs])( + 'a key moves exactly when the family, the rat-stack commit, the instrument hash or the nixpkgs rev moves', + (base, field, other) => { + const changed = { ...base, [field]: other[field] } + return (ratstackCacheKey(base) === ratstackCacheKey(changed)) === (base[field] === other[field]) + }, + ) +}) diff --git a/evals/ratstack-scorecard/src/model/cache-key.ts b/evals/ratstack-scorecard/src/model/cache-key.ts new file mode 100644 index 0000000..78249cf --- /dev/null +++ b/evals/ratstack-scorecard/src/model/cache-key.ts @@ -0,0 +1,17 @@ +import type { Family } from './cell.ts' + +export interface RatstackCacheInputs { + readonly family: Family + readonly ratstackCommit: string + readonly instrumentHash: string + readonly nixpkgsRev: string +} + +export const ratstackCacheKey = (inputs: RatstackCacheInputs): string => + [ + 'scorecard-ratstack', + inputs.family, + `rat-stack=${inputs.ratstackCommit}`, + `instrument=${inputs.instrumentHash}`, + `nixpkgs=${inputs.nixpkgsRev}`, + ].join('/') diff --git a/evals/ratstack-scorecard/src/model/cell.ts b/evals/ratstack-scorecard/src/model/cell.ts index 97f086d..3e2eb7e 100644 --- a/evals/ratstack-scorecard/src/model/cell.ts +++ b/evals/ratstack-scorecard/src/model/cell.ts @@ -107,12 +107,13 @@ export interface FamilyResult { readonly wallMs: number readonly cells: readonly SideCell[] readonly flags: readonly RowFlag[] + readonly definitionHashes: readonly { readonly id: string; readonly hash: string }[] } export type RowOutcome = | { readonly _tag: 'Held'; readonly id: string } | { readonly _tag: 'New'; readonly id: string } - | { readonly _tag: 'ReBaselined'; readonly id: string } + | { readonly _tag: 'ReBaselined'; readonly id: string; readonly mainHash: string; readonly prHash: string } | { readonly _tag: 'Neutral'; readonly id: string; readonly cause: CellTag } | { readonly _tag: 'InstrumentError'; readonly id: string; readonly error: string } | { readonly _tag: 'LostBeaten'; readonly id: string; readonly ratstack: CellTag; readonly starter: CellTag } diff --git a/evals/ratstack-scorecard/src/model/compare-with-main.workflow.property.test.ts b/evals/ratstack-scorecard/src/model/compare-with-main.workflow.property.test.ts index e96db22..19cd01b 100644 --- a/evals/ratstack-scorecard/src/model/compare-with-main.workflow.property.test.ts +++ b/evals/ratstack-scorecard/src/model/compare-with-main.workflow.property.test.ts @@ -48,7 +48,9 @@ describe('compareWithMain', () => { rows: [pr], main: { _tag: 'Found', commit, rows: [{ ...main, verdict: { _tag: 'Beaten' } }] }, }) - return ratchet.failures.length === 0 && ratchet.outcomes[0]!._tag === 'ReBaselined' + return ratchet.failures.length === 0 && + JSON.stringify(ratchet.outcomes) === + JSON.stringify([{ _tag: 'ReBaselined', id: main.definition.id, mainHash: main.definitionHash, prHash: hash }]) }) test.prop([ diff --git a/evals/ratstack-scorecard/src/model/compare-with-main.workflow.ts b/evals/ratstack-scorecard/src/model/compare-with-main.workflow.ts index f313ae2..b81eaf9 100644 --- a/evals/ratstack-scorecard/src/model/compare-with-main.workflow.ts +++ b/evals/ratstack-scorecard/src/model/compare-with-main.workflow.ts @@ -61,7 +61,10 @@ const starterAgainstMain = (main: Row, pr: Row): RowOutcome => const compareRow = (main: Row, pr: Row): RowOutcome => firstRule([ - [main.definitionHash !== pr.definitionHash, () => ({ _tag: 'ReBaselined', id: pr.definition.id })], + [ + main.definitionHash !== pr.definitionHash, + () => ({ _tag: 'ReBaselined', id: pr.definition.id, mainHash: main.definitionHash, prHash: pr.definitionHash }), + ], [ beatenWeight(main.verdict) > beatenWeight(pr.verdict), () => ({ diff --git a/evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.property.test.ts b/evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.property.test.ts new file mode 100644 index 0000000..a3c472f --- /dev/null +++ b/evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.property.test.ts @@ -0,0 +1,23 @@ +import { fc, test } from '@fast-check/vitest' +import { describe } from 'vitest' +import { planPinBump } from './plan-pin-bump.workflow.ts' +import { sha } from './scorecard.arbitrary.ts' + +const otherThan = (commit: string, index: number): string => { + const at = index % commit.length + const replacement = commit[at] === 'a' ? 'b' : 'a' + return `${commit.slice(0, at)}${replacement}${commit.slice(at + 1)}` +} + +describe('planPinBump', () => { + test.prop([sha])( + 'a remote head equal to the pin plans nothing', + (commit) => planPinBump({ pinned: commit, remoteHead: commit })._tag === 'PinCurrent', + ) + + test.prop([sha, fc.nat()])('any other remote head plans a bump to exactly that commit', (pinned, index) => { + const remoteHead = otherThan(pinned, index) + const plan = planPinBump({ pinned, remoteHead }) + return plan._tag === 'PinMoved' && plan.to === remoteHead && plan.from === pinned + }) +}) diff --git a/evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.ts b/evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.ts new file mode 100644 index 0000000..393fc15 --- /dev/null +++ b/evals/ratstack-scorecard/src/model/plan-pin-bump.workflow.ts @@ -0,0 +1,16 @@ +import { firstRule } from './dispatch.ts' + +export interface PlanPinBumpInput { + readonly pinned: string + readonly remoteHead: string +} + +export type PinPlan = + | { readonly _tag: 'PinCurrent'; readonly commit: string } + | { readonly _tag: 'PinMoved'; readonly from: string; readonly to: string } + +export const planPinBump = (input: PlanPinBumpInput): PinPlan => + firstRule( + [[input.remoteHead === input.pinned, () => ({ _tag: 'PinCurrent', commit: input.pinned })]], + () => ({ _tag: 'PinMoved', from: input.pinned, to: input.remoteHead }), + ) diff --git a/evals/ratstack-scorecard/src/model/summary-table.ts b/evals/ratstack-scorecard/src/model/summary-table.ts index 5c159cd..3d6b439 100644 --- a/evals/ratstack-scorecard/src/model/summary-table.ts +++ b/evals/ratstack-scorecard/src/model/summary-table.ts @@ -44,7 +44,7 @@ const showOutcome = (outcome: RowOutcome): string => matchOutcome(outcome, { Held: () => 'held', New: () => 'new', - ReBaselined: () => 're-baselined', + ReBaselined: (o) => `re-baselined (definition \`${short(o.mainHash)}\` → \`${short(o.prHash)}\`)`, Neutral: (o) => `neutral (${o.cause})`, InstrumentError: (o) => `**failed**: ${o.error}`, LostBeaten: (o) => `**failed**: no longer beaten (rat-stack ${o.ratstack}, starter ${o.starter})`, @@ -57,6 +57,29 @@ const outcomesOf = (doc: ScorecardDocument): readonly RowOutcome[] => const failuresOf = (doc: ScorecardDocument): readonly RowOutcome[] => matchRatchet(doc.ratchet, { FirstBaseline: (r) => r.failures, Compared: (r) => r.failures }) +const rebaselinedOf = (outcome: RowOutcome): readonly { id: string; mainHash: string; prHash: string }[] => + matchOutcome(outcome, { + Held: () => [], + New: () => [], + ReBaselined: (o) => [o], + Neutral: () => [], + InstrumentError: () => [], + LostBeaten: () => [], + Regressed: () => [], + }) + +const drift = (outcomes: readonly RowOutcome[]): readonly string[] => { + const rebaselined = outcomes.flatMap(rebaselinedOf) + return [ + ...rebaselined.slice(0, 1).flatMap(() => [ + `**Definition drift:** ${rebaselined.length} rows re-baselined; their ratchet starts over from this run.`, + '', + ]), + ...rebaselined.map((o) => `- ${o.id}: \`${o.mainHash}\` on \`main\`, \`${o.prHash}\` here`), + ...rebaselined.slice(0, 1).map(() => ''), + ] +} + const ratchetLine = (doc: ScorecardDocument): string => matchRatchet(doc.ratchet, { FirstBaseline: (r) => `first baseline (no \`main\` artifact), ${r.failures.length} failing rows`, @@ -91,6 +114,7 @@ export const renderSummary = (doc: ScorecardDocument): string => '', ...failuresOf(doc).map((failure) => `- ${failure.id}: ${cellText(showOutcome(failure))}`), '', + ...drift(outcomesOf(doc)), `Beaten: ${doc.rows.filter((row) => row.verdict._tag === 'Beaten').length} of ${doc.rows.length} rows.`, '', '| Row | Bin | Metric | rat-stack | starter | Verdict | Ratchet | Flags |', diff --git a/evals/ratstack-scorecard/tsconfig.json b/evals/ratstack-scorecard/tsconfig.json new file mode 100644 index 0000000..9483e66 --- /dev/null +++ b/evals/ratstack-scorecard/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "strict": true, + "exactOptionalPropertyTypes": true, + "noUncheckedIndexedAccess": true, + "target": "es2024", + "module": "nodenext", + "moduleResolution": "nodenext", + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "resolveJsonModule": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["decide/**/*.ts", "journeys/**/*.ts"] +}