diff --git a/.github/workflows/hook-latency-drift.yml b/.github/workflows/hook-latency-drift.yml index b0f4e59d4..d64da13a0 100644 --- a/.github/workflows/hook-latency-drift.yml +++ b/.github/workflows/hook-latency-drift.yml @@ -56,4 +56,112 @@ jobs: - uses: jdx/mise-action@3c2e0cf82a5b2e5249f0d3635a4d83d0ae861518 # v4.2.5 (CLOUD-404 retry fix, now a release) - name: Report a fast tier that no longer matches its budget shell: bash - run: mise run hook-latency-drift + env: + # The budget, in seconds, written once as data — the placement + # `perf-assert`'s BUDGETS table uses. It is a ceiling DERIVED from + # measurement (CLOUD-509), not a target chosen in advance. + # + # SLACK is what keeps this from firing on noise: a shared runner's wall + # clock moves several seconds run to run, and a report that cries wolf + # is a report nobody reads. LOOSE_FACTOR is the other end — the tier has + # to be dramatically under budget before re-deriving is worth a commit. + # + # Measured 2026-08-13 on a session container: 8s, 7s, 8s over three runs + # of `hk check --all --profile '!slow'`, against 275s for the same gate + # with the tier enabled. 15s is that median with room for a slower + # runner, and deliberately not so tight that the LOOSE_FACTOR floor + # (15/3 = 5s) sits above the measured 7s, which would make the report + # fire on its own baseline. + BUDGET_SECONDS: "15" + SLACK_SECONDS: "10" + LOOSE_FACTOR: "3" + RUNS: "3" + PROFILE: "slow" + # CLOUD-1270, disposition 3: `mise-tasks/hook-latency-drift.sh` and its + # suite are retired and the measurement lives here, inline. The step is + # the whole successor — no task, no verb, no record — because the subject + # is a wall clock rather than a property of any commit, and + # `.claude/rules/toolchain.md`'s split puts a property of the world on a + # clock. Nothing entered the core, so CLOUD-1176 is not reopened. + # + # WHY IT MEASURES `check --profile '!slow'` RATHER THAN A REAL COMMIT: + # `hk run pre-commit` stashes the worktree and applies fixers, so timing + # it would mutate the tree and measure the stash as much as the steps. The + # `check` hook runs the same step mapping with the same profile, + # read-only, which is the honest proxy for what a commit pays. + # + # NOTHING HERE WRITES TO THE TREE. Re-baselining the budget is a + # deliberate commit, prompted by this report — a bot silently rewriting + # the number it is supposed to defend is exactly the failure a + # hand-authored budget prevents. + # + # BOTH DIRECTIONS, on purpose. A budget that only complained about + # slowness would let every number rot upward: the tier gets faster, nobody + # re-derives the budget, and the ceiling stops bounding anything. + # + # `hk` ABSENT IS COULD-NOT-LOOK AND FAILS THE STEP. It must never collapse + # into a pass — the one property the retired program's `:70` got right and + # the one a successor could most easily lose. + run: | + set -euo pipefail + + if ! command -v hk >/dev/null 2>&1; then + echo "::error:: hook-latency-drift: hk is not on PATH, so the tier cannot be timed. unmeasurable" >&2 + exit 1 + fi + + scratch="$(mktemp -d)" + trap 'rm -rf "$scratch"' EXIT + + # Deliberately NOT hyperfine: `perf` uses it for a millisecond-scale + # binary where 100 runs are cheap and the tail is the point, while this + # is a multi-second whole-gate run where 3 samples is already minutes of + # runner time. The median of a few runs is the honest instrument at this + # scale, and pretending to a p95 over 3 samples would be a number with + # no meaning behind it. + samples="" + for _ in $(seq 1 "$RUNS"); do + start=$SECONDS + # The status is deliberately ignored: a RED gate still takes time, and + # this reports cost rather than correctness. A gate that cannot run at + # all is caught by the emptiness check below instead. + hk check --all --profile "!$PROFILE" >"$scratch/run.log" 2>&1 || true + samples+="$((SECONDS - start))"$'\n' + done + + count=$(grep -c '[0-9]' <<<"$samples" || true) + if [[ "$count" -lt "$RUNS" ]]; then + echo "::error:: hook-latency-drift: only $count of $RUNS runs produced a timing, so there is no measurement to judge. unmeasurable" >&2 + exit 1 + fi + + # Median of the sorted samples. Integer arithmetic throughout: the + # budget is in whole seconds and a fractional median would imply a + # precision the instrument does not have. + sorted=$(sort -n <<<"$samples" | grep '[0-9]') + median=$(sed -n "$(((RUNS + 1) / 2))p" <<<"$sorted") + if [[ -z "$median" ]]; then + echo "::error:: hook-latency-drift: the samples did not yield a median. unmeasurable" >&2 + exit 1 + fi + + ceiling=$((BUDGET_SECONDS + SLACK_SECONDS)) + floor=$((BUDGET_SECONDS / LOOSE_FACTOR)) + + # Pointer-only per non-negotiable rule 4: the measurement and the rule + # id, never a step's output. + drift="" + if [[ "$median" -gt "$ceiling" ]]; then + echo "::error:: hook-latency-drift: the fast tier measured ${median}s against a ${BUDGET_SECONDS}s budget (+${SLACK_SECONDS}s slack). Either a step has grown, or one has joined the fast tier that belongs in the $PROFILE one. drift-tight" >&2 + drift=1 + elif [[ "$median" -lt "$floor" ]]; then + echo "::error:: hook-latency-drift: the fast tier measured ${median}s against a ${BUDGET_SECONDS}s budget — under a ${LOOSE_FACTOR}x margin, so the budget has stopped bounding anything. Re-derive it in a deliberate commit. drift-loose" >&2 + drift=1 + fi + + if [[ -n "$drift" ]]; then + echo "::error:: hook-latency-drift: the budget no longer matches the measurement. This is a report, not a gate — nothing is broken and no branch is at fault; the number needs re-deriving in a commit of its own." >&2 + exit 1 + fi + + echo "hook-latency-drift: fast tier ${median}s over $RUNS run(s), within the ${BUDGET_SECONDS}s budget" diff --git a/batten.toml b/batten.toml index 5f067ce36..efee72a7b 100644 --- a/batten.toml +++ b/batten.toml @@ -1263,6 +1263,22 @@ regex = '^[a-z0-9]+:.+@.+$' id = "clause-label" regex = '^[[:space:]]*([*-][[:space:]]*)?\*\*[^*]*\((§|clause )[0-9]+\)|^#{2,6}[[:space:]]+[^#]*\((§|clause )[0-9]+\)|^[[:space:]]*[*-][[:space:]]+[^*#]{1,80}\((§|clause )[0-9]+\)\.' +# A SHA-pinned action reference on a workflow's `uses:` line (CLOUD-1318). +# +# A MATCH TEST, NEVER A CAPTURE, and that is forced rather than chosen: this build +# carries `regex.match` and nothing returning submatches, which is why +# `shell-retirement.rego` reaches for "`indexof` plus a NAME TEST rather than a +# capture" at its own site. `policy/sbom-inventory.rego` therefore never extracts +# the reference — it asks whether some declared table key appears in the line, +# which is the same question from the other end. +# +# The 40-hex bound is the whole discriminator: an unpinned `uses:` naming a tag, a +# branch or a local `./` path is NOT a pin and must not be demanded of the table, +# or the clause is satisfiable by matching every line in every workflow. +[[pattern]] +id = "sbom-action-pin" +regex = 'uses:[[:space:]]+[^[:space:]]+@[0-9a-f]{40}' + [[pattern]] id = "clause-one" regex = '\((§|clause )1\)' @@ -5128,6 +5144,51 @@ tool = "hk" version = "1.56.1" input = "hk.pkl" +# The published inventory, judged against the tree it claims to describe +# (CLOUD-262, ported off `mise-tasks/sbom-check.sh` under CLOUD-1318). +# +# THE SCAN STAYS OUTSIDE, which is `validator-verdict-clean`'s disposition and the +# same one house style §5 forces: `check` is `read` and cannot spawn, so `syft` +# stays a command on PATH and `mise-tasks/sbom.sh` stays the producer that derives +# the documents. `[tasks.record-sbom]` runs it twice and records the counts; this +# row adjudicates them. `sbom.sh` deliberately survives — it decides nothing, so it +# is a producer rather than a gate, and its own disposition is CLOUD-1159's. +# +# `line_sources` CARRIES TWO OF THE PREDICATES OUTRIGHT, and that is what keeps the +# producer's trusted surface narrow rather than total. The expected cargo count is +# `Cargo.lock`'s own `source = ` lines and the action mapping is every SHA-pinned +# `uses:` against `sbom-actions.tsv`'s key column — both properties of committed +# text, so the module reads them here and the producer cannot get them wrong on its +# behalf. Only the counts that require opening a DERIVED document travel through +# the record. +# +# SILENT UNTIL A PRODUCER WRITES. No record under this key means nothing has +# scanned these bytes at this pin — absent from the map, not a verdict — so the row +# is inert on a checkout whose globs never fired. Present-and-EMPTY is a finding, +# because every count below would otherwise pass over an absent key. +[[rule]] +id = "sbom-inventory" +kind = "policy" +scope = "tree" +module = "policy/sbom-inventory.rego" +line_sources = ["Cargo.lock", "mise-tasks/sbom-actions.tsv", ".github/workflows/*.yml"] +severity = "deny" + +# KEYED ON THE LOCKFILE, which is the input whose change must invalidate a verdict. +# The document is a function of the tree, and `Cargo.lock` is the part of it this +# gate's central invariant is stated against — so a dependency landing moves the +# digest, the key moves with it, and the previous scan's counts stop answering +# rather than answering about a tree that no longer exists. +# +# Keep `version` equal to the `aqua:anchore/syft` pin in `mise.toml`; they are two +# spellings of one decision, and CLOUD-664 is what happens when a syft bump changes +# what the document contains. +[[rule.tools]] +id = "sbom" +tool = "syft" +version = "1.51.1" +input = "Cargo.lock" + # What has shipped, from a tag GLOB rather than a named ref (CLOUD-1200, the # successor shape for the `released` family). # @@ -5986,14 +6047,33 @@ measured = "2026-09-02" # measurement I did not take is the honest half of the remedy, exactly as the # entry above says. +# THE SECOND 2026-09-02 MOVE, AND THE COUNT IS THE GATE'S OWN READING RATHER THAN +# A SCAN'S. CLOUD-843's bats bundle added `sbom_inventory.rs`, and the live count +# reached 175 — eleven past the basis and one outside the tolerance, so this run is +# the gate catching a drift its own predecessor entry predicted. +# +# `count` moves to the live 175 with `measured`, and THE FLOORS AGAIN DELIBERATELY +# DO NOT MOVE, for the reason the two entries above already state and which is +# unchanged: the lap that tripped this reported 21077MB free against a 7938MB +# declared warm floor, so free space was never what refused. Moving a floor down +# needs the independent measurement this block names and which was not taken here. +# +# WORTH ONE LINE OF ITS OWN: 175 IS WHAT `batten target prune` REPORTS AS `live`, +# and `git ls-files 'crates/batten/tests/**/*.rs'` answers 174 over the same glob. +# The two readings disagree by one and the gate's is the one written down, because +# the gate is what the number is compared against — a basis refreshed from a +# second reading of the tree would red again on the next lap while looking correct +# in review. Which file the two readers disagree about is unresolved and is not +# this bundle's; it is a pointer for whoever takes CLOUD-1158's floor re-derivation. + [prune.warm.basis] glob = "crates/batten/tests/**/*.rs" -count = 164 +count = 175 tolerance = 10 [prune.cold.basis] glob = "crates/batten/tests/**/*.rs" -count = 164 +count = 175 tolerance = 10 # THE REGROWABLE ROOTS THE ESCALATION MAY DROP (CLOUD-1157), in the order it drops @@ -7571,6 +7651,54 @@ id = "source read first" kind = "document" target = ".claude/hooks/git-hook.sh" +[[verdict]] +id = "tool read broken" +gloss = "the recorded SBOM scan cannot be judged, so the inventory is unverified" +class = """ +Four states reach this class and none of them is a drifted document: a catalog that found nothing, two scans of one tree that disagree once the volatile fields are removed, a document carrying no DESCRIBES edge to measure its components against, and a producer that recorded no counts at all. What they share is that the SCAN is at fault rather than the tree, so the remedy is never to edit an SBOM field: re-run `mise run record-sbom` and read what `mise-tasks/sbom.sh` actually produced. An empty catalog is the one that most looks like success -- two empty documents agree with each other and with every count -- which is why it refuses here instead of passing every equality trivially. +""" + +[[verdict.route]] +id = "module read first" +kind = "document" +target = "policy/sbom-inventory.rego" + +[[verdict]] +id = "manifest count wrong" +gloss = "the document and the tree it claims to describe do not agree" +class = """ +The cargo count is stated against `Cargo.lock`'s own `source = ` entries rather than a number anyone wrote down, because the issue that specified this recorded 156 cargo and 175 total and the total had moved by 2 a day later. So a disagreement means a cataloger missed something, or emitted a component per REFERENCE SITE rather than per thing depended on -- 340 entries for 290 distinct things, measured. Fix the normalisation in `mise-tasks/sbom.sh` and re-record; never adjust the expected count to match what the scan happened to produce, which is the one repair that makes the gate decide nothing. +""" + +[[verdict.route]] +id = "source read first" +kind = "document" +target = "mise-tasks/sbom.sh" + +[[verdict]] +id = "manifest state missing" +gloss = "a field the tree states is NOASSERTION in the published document" +class = """ +Supplier, originator, copyright and licence each have a source in this tree -- the lockfile's resolution, `cargo metadata`'s authors, the bytes `Cargo.lock` pins by checksum, and the manifests `cargo-deny` already reads. A `NOASSERTION` in any of them is data the tree states and the document dropped, and `NONE` is conformant where `NOASSERTION` is not, so only the third state refuses. The remedy is to enrich the field in `mise-tasks/sbom.sh` and re-record. Findings here are counts and never values on purpose: an author name, an email address and a copyright holder are all personal data, and echoing one would publish it into every CI log that reads this gate. +""" + +[[verdict.route]] +id = "source read first" +kind = "document" +target = "mise-tasks/sbom.sh" + +[[verdict]] +id = "pin table missing" +gloss = "a SHA-pinned action in a workflow has no row in the licence table" +class = """ +A pinned action's licence is immutable, which is what makes a committed table defensible at all -- but only while the table still describes the pins the workflows carry. This fires on the one event that breaks that: a pin moving. The row is matched on repository AND commit together, so a bump that changes the sha stops matching and the gate refuses rather than letting the document degrade quietly. Add the new commit's row to `mise-tasks/sbom-actions.tsv`. Renovate cannot write that table, which is why an action bump opens red and holds the queue slot (CLOUD-1213) -- that is the cost of the table, and it is paid deliberately. +""" + +[[verdict.route]] +id = "source read first" +kind = "document" +target = "mise-tasks/sbom-actions.tsv" + [[verdict]] id = "tool judge dirty" gloss = "a third-party validator judged this file and reported something" diff --git a/bench/suites/RESULTS.md b/bench/suites/RESULTS.md index bd663a96a..add2d775d 100644 --- a/bench/suites/RESULTS.md +++ b/bench/suites/RESULTS.md @@ -6,126 +6,124 @@ runner measured it; the suite runs `--no-parallelize-within-files`, so a file's number is its own serial cost and is what an author adding a case to it pays. -- suites: 118 -- serial total: 458.7s +- suites: 116 +- serial total: 530.7s | seconds | share | suite | | ---: | ---: | --- | -| 125.3 | 27.3% | `tests/land-lock.bats` | -| 70.3 | 15.3% | `tests/land.bats` | -| 34.5 | 7.5% | `tests/main-watch.bats` | -| 25.4 | 5.5% | `tests/sbom-check.bats` | -| 24.2 | 5.3% | `tests/hook-latency-drift.bats` | -| 15.3 | 3.3% | `tests/graph-check.bats` | -| 10.6 | 2.3% | `tests/board-diff-overlap.bats` | -| 7.2 | 1.6% | `tests/target-race.bats` | -| 6.9 | 1.5% | `tests/token-bench.bats` | -| 6.0 | 1.3% | `tests/ready-lint.bats` | -| 5.4 | 1.2% | `tests/released.bats` | -| 5.4 | 1.2% | `tests/singleton.bats` | -| 5.1 | 1.1% | `tests/ready-guard.bats` | -| 5.1 | 1.1% | `tests/board-sweep.bats` | -| 4.8 | 1.0% | `tests/sbom.bats` | -| 4.6 | 1.0% | `tests/signing-posture.bats` | -| 4.5 | 1.0% | `tests/replay.bats` | -| 4.4 | 0.9% | `tests/release-tracking-check.bats` | -| 4.0 | 0.9% | `tests/release-assets-check.bats` | -| 3.8 | 0.8% | `tests/step-receipt.bats` | -| 3.8 | 0.8% | `tests/task-registry.bats` | -| 3.8 | 0.8% | `tests/in-progress-drain.bats` | -| 3.4 | 0.7% | `tests/doctor-race.bats` | -| 3.3 | 0.7% | `tests/mcp-allow-check.bats` | -| 3.0 | 0.6% | `tests/with-lock.bats` | -| 2.8 | 0.6% | `tests/ready-cites-check.bats` | -| 2.7 | 0.6% | `tests/land-divergence.bats` | -| 2.7 | 0.6% | `tests/target-ensure.bats` | -| 2.6 | 0.6% | `tests/ntia-check.bats` | -| 2.5 | 0.5% | `tests/hk-selection.bats` | -| 1.9 | 0.4% | `tests/closing-key-check.bats` | -| 1.9 | 0.4% | `tests/install.bats` | -| 1.8 | 0.4% | `tests/landed-check.bats` | -| 1.8 | 0.4% | `tests/claim-race-check.bats` | -| 1.6 | 0.3% | `tests/suite-select.bats` | -| 1.5 | 0.3% | `tests/serena-mcp.bats` | -| 1.5 | 0.3% | `tests/spec-ref-check.bats` | -| 1.5 | 0.3% | `tests/finding-sink-check.bats` | -| 1.5 | 0.3% | `tests/reclaim-census.bats` | -| 1.4 | 0.3% | `tests/lint-deno.bats` | -| 1.3 | 0.3% | `tests/claimed-keys.bats` | -| 1.2 | 0.3% | `tests/tree-clean.bats` | -| 1.2 | 0.3% | `tests/ci-slow-needed.bats` | -| 1.2 | 0.3% | `tests/alive.bats` | -| 1.2 | 0.3% | `tests/ci-tools-check.bats` | -| 1.0 | 0.2% | `tests/verify.bats` | -| 1.0 | 0.2% | `tests/ready-lint-deferral.bats` | -| 0.9 | 0.2% | `tests/install-check.bats` | -| 0.9 | 0.2% | `tests/ci-lease-precondition.bats` | -| 0.9 | 0.2% | `tests/perf-record.bats` | -| 0.9 | 0.2% | `tests/land-divergence-assert.bats` | -| 0.9 | 0.2% | `tests/spawn-census.bats` | -| 0.9 | 0.2% | `tests/deferral-check.bats` | -| 0.9 | 0.2% | `tests/done-check.bats` | -| 0.9 | 0.2% | `tests/linear-check.bats` | -| 0.8 | 0.2% | `tests/nonverdict-scan.bats` | -| 0.8 | 0.2% | `tests/awk-regex-check.bats` | -| 0.8 | 0.2% | `tests/module-map-check.bats` | -| 0.7 | 0.2% | `tests/release-backfill.bats` | -| 0.7 | 0.2% | `tests/perf-assert.bats` | -| 0.7 | 0.2% | `tests/lint-rego.bats` | -| 0.7 | 0.1% | `tests/evaluator-closure-check.bats` | -| 0.7 | 0.1% | `tests/attestation-check.bats` | -| 0.6 | 0.1% | `tests/render-cli.bats` | -| 0.6 | 0.1% | `tests/doctor.bats` | -| 0.6 | 0.1% | `tests/pr-unsubscribed.bats` | -| 0.6 | 0.1% | `tests/done-pr-check.bats` | -| 0.6 | 0.1% | `tests/commit-attribution.bats` | -| 0.6 | 0.1% | `tests/timeout-drift.bats` | -| 0.6 | 0.1% | `tests/hook-matcher-check.bats` | -| 0.5 | 0.1% | `tests/duplicate-close-check.bats` | -| 0.5 | 0.1% | `tests/sbom-binary.bats` | -| 0.5 | 0.1% | `tests/perf-compare.bats` | -| 0.5 | 0.1% | `tests/verified.bats` | -| 0.5 | 0.1% | `tests/suite-bench-check.bats` | -| 0.5 | 0.1% | `tests/mcp-timeout-budget.bats` | -| 0.5 | 0.1% | `tests/merged-pr-keys.bats` | -| 0.5 | 0.1% | `tests/mcp-attach-check.bats` | -| 0.4 | 0.1% | `tests/macos-link-check.bats` | -| 0.4 | 0.1% | `tests/stop-posture-check.bats` | -| 0.4 | 0.1% | `tests/checksums.bats` | -| 0.4 | 0.1% | `tests/publish-credential-check.bats` | -| 0.4 | 0.1% | `tests/pipefail-grep-check.bats` | -| 0.4 | 0.1% | `tests/digest-major-agreement.bats` | -| 0.3 | 0.1% | `tests/connector-allow-guard.bats` | -| 0.3 | 0.1% | `tests/board-payloads.bats` | -| 0.3 | 0.1% | `tests/msrv-pin-agreement.bats` | -| 0.3 | 0.1% | `tests/abandon-matrix.bats` | -| 0.3 | 0.1% | `tests/branch-age-check.bats` | -| 0.3 | 0.1% | `tests/hook-pin-check.bats` | -| 0.3 | 0.1% | `tests/land-lock-check.bats` | -| 0.3 | 0.1% | `tests/sonar-gate.bats` | -| 0.3 | 0.1% | `tests/commit-convention.bats` | -| 0.3 | 0.1% | `tests/transcript-corpus-check.bats` | -| 0.3 | 0.1% | `tests/timeout-check.bats` | -| 0.3 | 0.1% | `tests/no-doctests.bats` | -| 0.3 | 0.1% | `tests/nonverdict-assert.bats` | +| 149.7 | 28.2% | `tests/land-lock.bats` | +| 89.6 | 16.9% | `tests/land.bats` | +| 34.5 | 6.5% | `tests/main-watch.bats` | +| 25.2 | 4.7% | `tests/graph-check.bats` | +| 15.9 | 3.0% | `tests/board-diff-overlap.bats` | +| 9.8 | 1.8% | `tests/ready-lint.bats` | +| 9.1 | 1.7% | `tests/token-bench.bats` | +| 8.9 | 1.7% | `tests/target-race.bats` | +| 8.6 | 1.6% | `tests/released.bats` | +| 8.3 | 1.6% | `tests/board-sweep.bats` | +| 7.4 | 1.4% | `tests/ready-guard.bats` | +| 6.7 | 1.3% | `tests/in-progress-drain.bats` | +| 6.7 | 1.3% | `tests/release-tracking-check.bats` | +| 6.6 | 1.2% | `tests/release-assets-check.bats` | +| 6.5 | 1.2% | `tests/replay.bats` | +| 5.6 | 1.0% | `tests/sbom.bats` | +| 5.2 | 1.0% | `tests/singleton.bats` | +| 5.1 | 1.0% | `tests/hk-selection.bats` | +| 5.1 | 1.0% | `tests/step-receipt.bats` | +| 4.8 | 0.9% | `tests/mcp-allow-check.bats` | +| 4.6 | 0.9% | `tests/land-divergence.bats` | +| 4.0 | 0.8% | `tests/ready-cites-check.bats` | +| 4.0 | 0.8% | `tests/ntia-check.bats` | +| 4.0 | 0.8% | `tests/task-registry.bats` | +| 3.7 | 0.7% | `tests/doctor-race.bats` | +| 3.3 | 0.6% | `tests/with-lock.bats` | +| 3.1 | 0.6% | `tests/closing-key-check.bats` | +| 3.1 | 0.6% | `tests/landed-check.bats` | +| 3.0 | 0.6% | `tests/finding-sink-check.bats` | +| 2.9 | 0.5% | `tests/target-ensure.bats` | +| 2.7 | 0.5% | `tests/claim-race-check.bats` | +| 2.5 | 0.5% | `tests/spec-ref-check.bats` | +| 2.5 | 0.5% | `tests/suite-select.bats` | +| 2.4 | 0.5% | `tests/install.bats` | +| 2.2 | 0.4% | `tests/claimed-keys.bats` | +| 1.9 | 0.4% | `tests/alive.bats` | +| 1.9 | 0.4% | `tests/ci-tools-check.bats` | +| 1.9 | 0.4% | `tests/reclaim-census.bats` | +| 1.9 | 0.4% | `tests/install-check.bats` | +| 1.8 | 0.3% | `tests/ci-slow-needed.bats` | +| 1.7 | 0.3% | `tests/signing-posture.bats` | +| 1.7 | 0.3% | `tests/tree-clean.bats` | +| 1.6 | 0.3% | `tests/ci-lease-precondition.bats` | +| 1.5 | 0.3% | `tests/land-divergence-assert.bats` | +| 1.5 | 0.3% | `tests/deferral-check.bats` | +| 1.4 | 0.3% | `tests/ready-lint-deferral.bats` | +| 1.4 | 0.3% | `tests/perf-record.bats` | +| 1.3 | 0.3% | `tests/verify.bats` | +| 1.3 | 0.2% | `tests/hook-matcher-check.bats` | +| 1.3 | 0.2% | `tests/awk-regex-check.bats` | +| 1.3 | 0.2% | `tests/linear-check.bats` | +| 1.3 | 0.2% | `tests/done-check.bats` | +| 1.2 | 0.2% | `tests/spawn-census.bats` | +| 1.2 | 0.2% | `tests/lint-rego.bats` | +| 1.1 | 0.2% | `tests/nonverdict-scan.bats` | +| 1.1 | 0.2% | `tests/perf-assert.bats` | +| 1.1 | 0.2% | `tests/release-backfill.bats` | +| 1.1 | 0.2% | `tests/attestation-check.bats` | +| 1.0 | 0.2% | `tests/done-pr-check.bats` | +| 1.0 | 0.2% | `tests/module-map-check.bats` | +| 0.9 | 0.2% | `tests/pr-unsubscribed.bats` | +| 0.9 | 0.2% | `tests/render-cli.bats` | +| 0.9 | 0.2% | `tests/doctor.bats` | +| 0.9 | 0.2% | `tests/evaluator-closure-check.bats` | +| 0.9 | 0.2% | `tests/duplicate-close-check.bats` | +| 0.9 | 0.2% | `tests/sbom-binary.bats` | +| 0.8 | 0.2% | `tests/perf-compare.bats` | +| 0.8 | 0.2% | `tests/commit-attribution.bats` | +| 0.8 | 0.2% | `tests/lint-deno.bats` | +| 0.8 | 0.1% | `tests/timeout-drift.bats` | +| 0.7 | 0.1% | `tests/checksums.bats` | +| 0.7 | 0.1% | `tests/merged-pr-keys.bats` | +| 0.7 | 0.1% | `tests/suite-bench-check.bats` | +| 0.7 | 0.1% | `tests/mcp-timeout-budget.bats` | +| 0.7 | 0.1% | `tests/mcp-attach-check.bats` | +| 0.6 | 0.1% | `tests/verified.bats` | +| 0.6 | 0.1% | `tests/macos-link-check.bats` | +| 0.6 | 0.1% | `tests/land-lock-check.bats` | +| 0.6 | 0.1% | `tests/publish-credential-check.bats` | +| 0.6 | 0.1% | `tests/stop-posture-check.bats` | +| 0.6 | 0.1% | `tests/connector-allow-guard.bats` | +| 0.6 | 0.1% | `tests/branch-age-check.bats` | +| 0.6 | 0.1% | `tests/digest-major-agreement.bats` | +| 0.6 | 0.1% | `tests/hook-pin-check.bats` | +| 0.5 | 0.1% | `tests/board-payloads.bats` | +| 0.5 | 0.1% | `tests/pipefail-grep-check.bats` | +| 0.5 | 0.1% | `tests/abandon-matrix.bats` | +| 0.5 | 0.1% | `tests/license-table-check.bats` | +| 0.5 | 0.1% | `tests/sonar-gate.bats` | +| 0.5 | 0.1% | `tests/msrv-pin-agreement.bats` | +| 0.4 | 0.1% | `tests/commit-convention.bats` | +| 0.4 | 0.1% | `tests/no-doctests.bats` | +| 0.4 | 0.1% | `tests/serena-mcp.bats` | +| 0.4 | 0.1% | `tests/release-due.bats` | +| 0.4 | 0.1% | `tests/transcript-corpus-check.bats` | +| 0.4 | 0.1% | `tests/connector-allow-resolve.bats` | +| 0.4 | 0.1% | `tests/nonverdict-assert.bats` | +| 0.4 | 0.1% | `tests/timeout-check.bats` | +| 0.4 | 0.1% | `tests/coderabbit-config-check.bats` | +| 0.4 | 0.1% | `tests/cap-drift.bats` | +| 0.4 | 0.1% | `tests/container-preflight.bats` | +| 0.4 | 0.1% | `tests/batten-glob-check.bats` | +| 0.3 | 0.1% | `tests/git-hook.bats` | | 0.3 | 0.1% | `tests/report-only-check.bats` | -| 0.3 | 0.1% | `tests/release-due.bats` | -| 0.2 | 0.1% | `tests/license-table-check.bats` | -| 0.2 | 0.1% | `tests/connector-allow-resolve.bats` | -| 0.2 | 0.1% | `tests/batten-glob-check.bats` | -| 0.2 | 0.0% | `tests/container-preflight.bats` | -| 0.2 | 0.0% | `tests/coderabbit-config-check.bats` | -| 0.2 | 0.0% | `tests/cap-drift.bats` | -| 0.2 | 0.0% | `tests/mise-action-floor.bats` | -| 0.2 | 0.0% | `tests/git-hook.bats` | | 0.2 | 0.0% | `tests/rust-paths-check.bats` | -| 0.1 | 0.0% | `tests/perf-gate.bats` | -| 0.1 | 0.0% | `tests/remedy-payload-source.bats` | -| 0.1 | 0.0% | `tests/token-bench-check.bats` | -| 0.1 | 0.0% | `tests/task-fail-closed.bats` | -| 0.1 | 0.0% | `tests/dist.bats` | +| 0.2 | 0.0% | `tests/mise-action-floor.bats` | +| 0.2 | 0.0% | `tests/perf-gate.bats` | +| 0.2 | 0.0% | `tests/dist.bats` | +| 0.2 | 0.0% | `tests/remedy-payload-source.bats` | +| 0.2 | 0.0% | `tests/token-bench-check.bats` | +| 0.2 | 0.0% | `tests/task-fail-closed.bats` | +| 0.2 | 0.0% | `tests/evaluator-io-check.bats` | | 0.1 | 0.0% | `tests/egress-check.bats` | -| 0.1 | 0.0% | `tests/evaluator-io-check.bats` | +| 0.1 | 0.0% | `tests/darwin-link.bats` | | 0.1 | 0.0% | `tests/cross-check.bats` | -| 0.0 | 0.0% | `tests/darwin-link.bats` | -| 0.0 | 0.0% | `tests/zizmor-split.bats` | +| 0.1 | 0.0% | `tests/zizmor-split.bats` | diff --git a/crates/batten/tests/it/main.rs b/crates/batten/tests/it/main.rs index 4daf10555..79e06cfba 100644 --- a/crates/batten/tests/it/main.rs +++ b/crates/batten/tests/it/main.rs @@ -176,6 +176,7 @@ mod rules_drift; mod run_shape; mod run_shape_guard_door; mod runner_verdict; +mod sbom_inventory; mod scanner_taxonomy; mod secrets_kind; mod semver_gate; diff --git a/crates/batten/tests/it/ratchet.rs b/crates/batten/tests/it/ratchet.rs index 250ef27d2..edae29364 100644 --- a/crates/batten/tests/it/ratchet.rs +++ b/crates/batten/tests/it/ratchet.rs @@ -2026,3 +2026,41 @@ fn removing_an_inline_body_never_violates_either_row() { // withdrawn: "an install.sh that refuses is not reported as ready" the wrapper propagated install.sh's exit status; with nothing between the caller and install.sh there is no propagation to assert // withdrawn: "THE REFUSAL: with no checkout, a script the manifest disagrees with is not run" install.sh deliberately does NOT verify its own bytes — a one-liner cannot, and its trust is TLS plus the release digest it checks on the BINARY // withdrawn: "with no checkout and no install.sh asset, the gate that should have caught it is named" the wrapper's fetch fallback is gone; `release-assets-check` still demands the asset, which is that obligation's real home + +// --- the ledger for a WITHDRAWAL: `hook-latency-drift` (CLOUD-1270) --- +// +// Beside the block above for the same reason it is here at all: a withdrawal has +// no successor to sit with. CLOUD-1270's decided disposition 3 moves the +// measurement into `.github/workflows/hook-latency-drift.yml` as an inline step — +// ungoverned, no task, no verb, no record — so there is no policy surface and no +// compiled-binary tier to name, which is exactly what makes this a withdrawal +// rather than a port. +// +// WHAT THE SUITE ACTUALLY PINNED, stated plainly because the arms below are all +// withdrawals and a reader is entitled to know what that costs. Every case drove +// a `sleep`-based `hk` stub and asserted the ARITHMETIC over it: median of N +// samples, a `BUDGET + SLACK` ceiling, a `BUDGET / LOOSE_FACTOR` floor, and the +// three verdicts those produce. That arithmetic survives verbatim in the step's +// own body, and the four numbers it reads are now `env:` entries beside it rather +// than shell defaults — but nothing re-asserts them, because a workflow step is +// not drivable from `crates/batten/tests/**` and inventing a fixture that re-ran +// the same shell in a test would be a second implementation of the thing under +// test rather than coverage of it. +// +// SO THE COVERAGE LOSS IS REAL AND IS THE PRICE THE ROW PRICED: 24.3s of a corpus +// against six cases over a reporter that decides nothing and gates no commit. The +// one property worth more than the arithmetic — `hk` absent must fail rather than +// pass — is preserved by construction instead of by assertion, because the step +// runs under `set -euo pipefail` and exits 1 on that branch, where the retired +// program exited 2 into a task runner that discarded it on a schedule. +// +// The two file arms name each other as the path this same delta retires, which is +// what `withdrawn_subjects` demands and what stops a row excusing its own deletion. +// withdrawn: mise-tasks/hook-latency-drift.sh tests/hook-latency-drift.bats the successor is an inline step of the scheduled workflow, not a mechanism — nothing enters `crates/batten` and no `policy/*.rego` decides it, so there is no surface to name; CLOUD-1270 disposition 3 +// withdrawn: tests/hook-latency-drift.bats mise-tasks/hook-latency-drift.sh the subject died in this same delta and the successor is a workflow step no test target can drive, so the cases have nowhere to go rather than somewhere unnamed; CLOUD-1270 disposition 3 +// withdrawn: "a tier inside its budget passes" the in-budget arm is the step's own final `echo` on the path where neither comparison fires; there is no runner-independent way to assert a wall clock from a test target +// withdrawn: "a tier over budget plus slack is drift-tight" the `median > BUDGET + SLACK` comparison moves verbatim into the step, with the tokens `drift-tight` and the two numbers still in the message; nothing re-asserts it because the step is not drivable from a test +// withdrawn: "slack absorbs a small overshoot rather than crying wolf" this pinned that SLACK is added to the ceiling rather than ignored — the same single `ceiling=$((BUDGET_SECONDS + SLACK_SECONDS))` line, now data in the step's `env:` block +// withdrawn: "a tier far under budget is drift-loose, not a silent pass" the ratchet direction, preserved as the `elif` on `median < BUDGET / LOOSE_FACTOR`; it is the arm most likely to rot unnoticed and that is stated here rather than hidden +// withdrawn: "a red gate is still timed, because cost is not correctness" preserved as the `|| true` on the `hk check` line, which is now load-bearing without a case behind it +// withdrawn: "no hk on PATH is could-not-look, never a verdict" the one property that must not collapse into a pass; preserved structurally — `command -v hk` failing exits 1 under `set -e` — rather than by assertion diff --git a/crates/batten/tests/it/sbom_inventory.rs b/crates/batten/tests/it/sbom_inventory.rs new file mode 100644 index 000000000..af2685bda --- /dev/null +++ b/crates/batten/tests/it/sbom_inventory.rs @@ -0,0 +1,493 @@ +//! `policy/sbom-inventory.rego` over the compiled binary (CLOUD-262, retired +//! under CLOUD-1318). +//! +//! **The load-time tier pins the predicate and cannot pin that the engine builds +//! what it reads.** The module's own `test_` rules fabricate +//! `input.tree["tool-verdict"]["sbom"]` and `input.tree.lines` with `with input +//! as`, which passes whether or not anything can produce that shape — the class +//! `.claude/rules/policy-modules.md` opens with. This file runs the real producer +//! verb and the real engine, over a real lockfile and a real workflow. +//! +//! **The two halves are deliberately split, and the split is the subject here.** +//! Counts that require opening a DERIVED document travel through the record, +//! because `check` is `read` and cannot run `syft`. The two predicates readable +//! from committed text — the expected cargo count and the action-pin mapping — +//! are decided by the module from `line_sources`, so the producer cannot get them +//! wrong on its behalf. Cases below drive both routes. +//! +//! # RETIREMENT LEDGER, PER PATH — what `shell-retirement` reads +//! +//! `sbom-check.sh` re-ran `sbom.sh` twice and adjudicated the documents in shell. +//! The scan stays outside — §9's prior art, and §5 makes `check` `read` — so +//! `mise run record-sbom` derives and records, and the adjudication moves here. +//! `sbom.sh` itself SURVIVES: it decides nothing, so it is a producer rather than +//! a gate, and its disposition is CLOUD-1159's rather than this row's. + +// carried: mise-tasks/sbom-check.sh policy/sbom-inventory.rego crates/batten/tests/it/sbom_inventory.rs +// carried: tests/sbom-check.bats policy/sbom-inventory.rego crates/batten/tests/it/sbom_inventory.rs + +//! # RETIREMENT LEDGER — `tests/sbom-check.bats`, 14 cases +//! +//! CARRIED — the decision table, which is what the gate was for. + +// carried: "a matching, stable inventory passes — and that IS the normalizer working" crates/batten/tests/it/sbom_inventory.rs +// carried: "a cargo count that disagrees with Cargo.lock fails, naming both numbers" crates/batten/tests/it/sbom_inventory.rs +// carried: "an SBOM that catalogs nothing must not report green" crates/batten/tests/it/sbom_inventory.rs +// carried: "output is pointer-only — no document body reaches the log" crates/batten/tests/it/sbom_inventory.rs +// carried: "a lockfile whose local member has no source still matches: 1 purl, 1 sourced of 2" crates/batten/tests/it/sbom_inventory.rs +// carried: "the inflated shapes syft produces are all absorbed before the gate judges" crates/batten/tests/it/sbom_inventory.rs +// carried: "a document that DESCRIBES nothing is could-not-look, not a clean inventory" crates/batten/tests/it/sbom_inventory.rs +// carried: "THE DRIFT DETECTOR: a pin with no table row fails, which is how a bump arrives" crates/batten/tests/it/sbom_inventory.rs + +//! CHANGED — four cases whose SUBJECT moved from the gate to the producer, so the +//! property is conserved where it is now decided rather than where it was. + +// changed: "THE NEGATIVE SELF-TEST: a renamed package still fails after normalization" crates/batten/tests/it/sbom_inventory.rs the shell drove a syft stub whose two runs differed in a package NAME and asserted the normalizer did not absorb it. That comparison is `record-sbom`'s `stable()` now — it normalises the four volatile leaves and `cmp`s the rest — and what reaches the module is a yes/no token. `an_unstable_scan_is_refused` drives the token; the discrimination it protects lives in the producer's `jq -S 'del(...)'`, which still names exactly four leaves +// changed: "a syft that cannot run exits 2 — could not look is not a verdict" crates/batten/tests/it/sbom_inventory.rs deriving the document is the producer's job now, so a syft that cannot run fails `record-sbom` and writes NO record — leaving the id absent, which the module reads as could-not-look and refuses nothing. `an_unrecorded_scan_is_clean` is the successor; the exit code is the producer's rather than a gate's +// changed: "a missing Cargo.lock exits 2 rather than passing vacuously" crates/batten/tests/it/sbom_inventory.rs the module abstains from the count comparison when the lockfile was not read (`is_array(lock_lines)`) and `input.tree.missing` reports it, rather than the gate exiting 2 itself. Conserved as the module's `test_an_unreadable_lockfile_reports_no_drift` plus its `missing` clause — the vacuous PASS the case names is exactly what the guard prevents +// changed: "this repo's real tree satisfies the gate — with the real syft" crates/batten/tests/it/sbom_inventory.rs a whole-tree `syft scan dir:.` inside a cargo test would put two minutes of scanning in the test tier for what the hk gate already does. The successor is the `record-sbom` step plus `batten-check` running on this repository's own globs, which is where the real syft belongs; `a_clean_scan_over_the_real_lockfile_is_clean` keeps the end-to-end shape over a recorded scan + +//! WITHDRAWN — two cases whose subject is unrepresentable in the successor, each +//! because the engine makes the property structural rather than assertable. + +// withdrawn: "the failure names an asset, not a scratch path" the module never sees a scratch path: it receives counts and decides over `line_sources`, and its subjects are a tagged `{path}` naming the tracked lockfile or the tracked table. There is no filesystem path in the finding to get wrong, so the assertion has nothing left to discriminate +// withdrawn: "the gate leaves the tree it judges unmodified, and fails twice" `check` is declared `read` and `evaluator-io-check` is the standing gate on the engine opening nothing, so a module that wrote to the tree is unrepresentable rather than merely untested. The producer's two runs still go to scratch directories, which is `record-sbom`'s own concern + +// Panicking on setup failure is the idiomatic way for a test to fail loudly. +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use crate::common; + +use std::path::{Path, PathBuf}; + +use common::{batten, git_in, run_with_stdin, scratch, stderr, stdout, write}; + +/// The `syft` pin the row declares, and the one a record must be keyed to. +const DECLARED_VERSION: &str = "1.51.1"; + +/// A lockfile with TWO packages and ONE `source` line. +/// +/// This is CLOUD-664's shape, and it is the fixture rather than a convenience: +/// syft gives the local workspace member no registry purl, so the expected count +/// is the SOURCED entries and not the total. A fixture with a `source` on every +/// entry would pass under either reading and discriminate nothing. +const LOCKFILE: &str = concat!( + "[[package]]\nname = \"local\"\nversion = \"0.1.0\"\n\n", + "[[package]]\nname = \"dep\"\nversion = \"1.0.0\"\nsource = \"registry+https://example.invalid\"\n", +); + +/// One SHA-pinned action, and the table row that declares it. +const WORKFLOW: &str = "jobs:\n build:\n steps:\n - uses: acme/checkout@0123456789abcdef0123456789abcdef01234567 # v1\n"; + +const TABLE: &str = "acme/checkout@0123456789abcdef0123456789abcdef01234567\tMIT\tAcme\n"; + +/// A recorded scan agreeing with [`LOCKFILE`]'s one sourced entry. +fn clean_scan() -> String { + counts(&[("spdx-cargo", "1"), ("cdx-cargo", "1")]) +} + +/// The clean record with the named keys overridden. +fn counts(overrides: &[(&str, &str)]) -> String { + use std::fmt::Write as _; + + let mut rows: Vec<(&str, &str)> = [ + ("spdx-cargo", "1"), + ("cdx-cargo", "1"), + ("spdx-stable", "yes"), + ("cdx-stable", "yes"), + ("subject", "1"), + ("entries", "1"), + ("distinct", "1"), + ("pathlike", "0"), + ("unversioned", "0"), + ("nosupplier", "0"), + ("subject-unset", "0"), + ("originator-disagrees", "0"), + ("copyright-unset", "0"), + ("license-unset", "0"), + ("license-slashed", "0"), + ("action-unset", "0"), + ] + .to_vec(); + for (key, value) in overrides { + if let Some(row) = rows.iter_mut().find(|(name, _)| name == key) { + row.1 = value; + } + } + rows.iter().fold(String::new(), |mut record, (key, value)| { + let _ = writeln!(record, "{key} {value}"); + record + }) +} + +fn config() -> String { + format!( + r#"version = 1 + +[[rule]] +id = "sbom-inventory" +kind = "policy" +scope = "tree" +module = "sbom-inventory.rego" +line_sources = ["Cargo.lock", "mise-tasks/sbom-actions.tsv", ".github/workflows/*.yml"] +severity = "deny" + +[[rule.tools]] +id = "sbom" +tool = "syft" +version = "{DECLARED_VERSION}" +input = "Cargo.lock" + +[[pattern]] +id = "sbom-action-pin" +regex = 'uses:[[:space:]]+[^[:space:]]+@[0-9a-f]{{40}}' + +[[verdict]] +id = "tool read broken" +gloss = "the recorded SBOM scan cannot be judged, so the inventory is unverified" +class = "A fixture class, mirroring the committed row." + +[[verdict.route]] +id = "module read first" +kind = "document" +target = "sbom-inventory.rego" + +[[verdict]] +id = "manifest count wrong" +gloss = "the document and the tree it claims to describe do not agree" +class = "A fixture class, mirroring the committed row." + +[[verdict.route]] +id = "source read first" +kind = "document" +target = "Cargo.lock" + +[[verdict]] +id = "manifest state missing" +gloss = "a field the tree states is NOASSERTION in the published document" +class = "A fixture class, mirroring the committed row." + +[[verdict.route]] +id = "source read first" +kind = "document" +target = "Cargo.lock" + +[[verdict]] +id = "pin table missing" +gloss = "a SHA-pinned action in a workflow has no row in the licence table" +class = "A fixture class, mirroring the committed row." + +[[verdict.route]] +id = "source read first" +kind = "document" +target = "mise-tasks/sbom-actions.tsv" +"# + ) +} + +/// A repository carrying the committed module, the row that reads it, a lockfile, +/// a workflow and the licence table. +fn fixture(name: &str, table: &str) -> PathBuf { + let dir = scratch(&format!("sbom-inventory-{name}-{}", std::process::id())); + write(&dir, "batten.toml", &config()); + write( + &dir, + "sbom-inventory.rego", + &std::fs::read_to_string(at_repo("policy/sbom-inventory.rego")).expect("read the module"), + ); + write(&dir, "Cargo.lock", LOCKFILE); + write(&dir, ".github/workflows/ci.yml", WORKFLOW); + write(&dir, "mise-tasks/sbom-actions.tsv", table); + git_in(&dir, &["init", "-q", "-b", "main", "."]); + dir +} + +/// A path inside this repository, resolved from the test binary's manifest dir. +fn at_repo(relative: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join(relative) +} + +/// Record a scan through the real producer verb. +fn record(dir: &Path, scan: &str) -> std::process::Output { + run_with_stdin(dir, &["record", "tool", "sbom"], scan) +} + +fn check(dir: &Path) -> std::process::Output { + let mut command = batten(); + command.current_dir(dir).arg("check"); + command.output().expect("run batten check") +} + +#[test] +fn a_clean_scan_over_the_real_lockfile_is_clean() { + // THE ANTI-VACUITY MIRROR, listed first because every refusal below is only + // evidence if this one passes: a module denying unconditionally would satisfy + // all of them. It is also what proves the ENGINE fills both halves — the + // record AND `input.tree.lines` for three declared sources — since a count of + // 1 only agrees with this lockfile if `Cargo.lock` was actually read. + let dir = fixture("clean", TABLE); + assert_eq!(record(&dir, &clean_scan()).status.code(), Some(0)); + + let outcome = check(&dir); + let (answer, cause) = (stdout(&outcome), stderr(&outcome)); + assert_eq!( + outcome.status.code(), + Some(0), + "a matching, stable inventory passes\n{answer}{cause}" + ); +} + +#[test] +fn a_drifted_cargo_count_is_refused_over_the_real_lockfile() { + // THE CENTRAL INVARIANT, and the one the declared mutation sits on. The + // expected number is never written down: it is this lockfile's own `source =` + // lines, so a count that disagrees means a cataloger missed something. + let dir = fixture("drift", TABLE); + assert_eq!( + record(&dir, &counts(&[("spdx-cargo", "2")])).status.code(), + Some(0) + ); + + let outcome = check(&dir); + let (answer, cause) = (stdout(&outcome), stderr(&outcome)); + assert_eq!( + outcome.status.code(), + Some(2), + "a cargo count disagreeing with the lockfile is a policy verdict\n{answer}{cause}" + ); + assert!(answer.contains("sbom-package-drift"), "{answer}{cause}"); +} + +#[test] +fn the_sourced_entries_are_counted_rather_than_every_package() { + // CLOUD-664 AS A CASE. The fixture has two `[[package]]` entries and one + // `source`, so a successor counting packages would expect 2 and refuse the + // honest 1. This is the case that fails if anyone "simplifies" the predicate + // back to counting entries. + let dir = fixture("sourced", TABLE); + assert_eq!(record(&dir, &clean_scan()).status.code(), Some(0)); + assert_eq!(check(&dir).status.code(), Some(0)); + + let dir = fixture("sourced-two", TABLE); + assert_eq!( + record(&dir, &counts(&[("spdx-cargo", "2"), ("cdx-cargo", "2")])) + .status + .code(), + Some(0) + ); + assert_eq!( + check(&dir).status.code(), + Some(2), + "counting every package rather than the sourced ones must not pass" + ); +} + +#[test] +fn an_empty_catalog_is_refused() { + // A scan whose catalogers all missed agrees with every equality trivially: + // two empty documents match, and an empty count matches an empty count. + let dir = fixture("empty", TABLE); + assert_eq!( + record(&dir, &counts(&[("spdx-cargo", "0"), ("cdx-cargo", "0")])) + .status + .code(), + Some(0) + ); + + let outcome = check(&dir); + let answer = stdout(&outcome); + assert_eq!(outcome.status.code(), Some(2), "{answer}"); + assert!(answer.contains("sbom-empty"), "{answer}"); +} + +#[test] +fn an_unstable_scan_is_refused() { + // Two scans of one tree must produce identical bytes once the four volatile + // leaves are removed. The comparison is the producer's; the token is the + // module's to judge. + let dir = fixture("unstable", TABLE); + assert_eq!( + record(&dir, &counts(&[("spdx-stable", "no")])) + .status + .code(), + Some(0) + ); + + let outcome = check(&dir); + let answer = stdout(&outcome); + assert_eq!(outcome.status.code(), Some(2), "{answer}"); + assert!(answer.contains("sbom-unstable"), "{answer}"); +} + +#[test] +fn an_inflated_component_set_is_refused() { + // syft emits a component per REFERENCE SITE. Measured once at 340 entries for + // 290 distinct things; this is the clause that keeps `sbom.sh`'s normalisation + // honest without anyone having predicted which shape comes next. + let dir = fixture("inflated", TABLE); + assert_eq!( + record(&dir, &counts(&[("entries", "3")])).status.code(), + Some(0) + ); + + let outcome = check(&dir); + let answer = stdout(&outcome); + assert_eq!(outcome.status.code(), Some(2), "{answer}"); + assert!(answer.contains("sbom-components-inflated"), "{answer}"); +} + +#[test] +fn a_document_describing_nothing_is_refused() { + // The subject is what every component count is measured against, so a document + // carrying no DESCRIBES edge leaves them all taken over the wrong set. + let dir = fixture("describes", TABLE); + assert_eq!( + record(&dir, &counts(&[("subject", "0")])).status.code(), + Some(0) + ); + assert_eq!(check(&dir).status.code(), Some(2)); +} + +#[test] +fn an_unmapped_action_pin_is_refused_from_committed_text_alone() { + // THE DRIFT DETECTOR, decided with NO record involvement: the workflow and the + // table are both `line_sources`, so this is the half a producer cannot get + // wrong. A bump moves the sha, the row stops matching, and the gate fires. + let dir = fixture("unmapped", ""); + assert_eq!(record(&dir, &clean_scan()).status.code(), Some(0)); + + let outcome = check(&dir); + let answer = stdout(&outcome); + assert_eq!(outcome.status.code(), Some(2), "{answer}"); + assert!(answer.contains("sbom-action-unmapped"), "{answer}"); +} + +#[test] +fn a_stale_sha_in_the_table_does_not_map_the_pin() { + // The row is matched on repository AND commit together, because a row whose + // sha is stale is exactly the drift this detects. + let dir = fixture( + "stale-sha", + "acme/checkout@ffffffffffffffffffffffffffffffffffffffff\tMIT\tAcme\n", + ); + assert_eq!(record(&dir, &clean_scan()).status.code(), Some(0)); + assert_eq!(check(&dir).status.code(), Some(2)); +} + +#[test] +fn an_unrecorded_scan_is_clean() { + // NOTHING HAS SCANNED THESE BYTES is not a verdict. This is the ordinary state + // of a checkout whose globs never fired, and refusing here would deny every + // clone until a producer runs. + let dir = fixture("unrecorded", TABLE); + let outcome = check(&dir); + let (answer, cause) = (stdout(&outcome), stderr(&outcome)); + assert_eq!( + outcome.status.code(), + Some(0), + "an absent record is could-not-look\n{answer}{cause}" + ); +} + +#[test] +fn a_recorded_but_empty_scan_is_refused() { + // PRESENT AND EMPTY is the producer having written nothing, which would let + // every count above pass over an absent key — the vacuity arm. + let dir = fixture("blank", TABLE); + assert_eq!(record(&dir, "").status.code(), Some(0)); + + let outcome = check(&dir); + let answer = stdout(&outcome); + assert_eq!(outcome.status.code(), Some(2), "{answer}"); + assert!(answer.contains("sbom-unrecorded"), "{answer}"); +} + +#[test] +fn a_record_from_another_version_does_not_answer() { + // THE KEY IS A TRIPLE, and the pinned version is one leg of it. A scan taken + // at another syft is not evidence about this one — it is not found at all, + // which is what keeps a stale verdict from reading as a fresh one. + let dir = fixture("version", TABLE); + assert_eq!( + record(&dir, &counts(&[("spdx-cargo", "99")])).status.code(), + Some(0) + ); + assert_eq!(check(&dir).status.code(), Some(2)); + + // Move the pin; the record's key moves with it and the drift stops answering. + write( + &dir, + "batten.toml", + &config().replace(DECLARED_VERSION, "9.9.9"), + ); + let outcome = check(&dir); + let (answer, cause) = (stdout(&outcome), stderr(&outcome)); + assert_eq!( + outcome.status.code(), + Some(0), + "a record keyed to another pin must not answer\n{answer}{cause}" + ); +} + +#[test] +fn a_verdict_does_not_survive_its_input() { + // THE INPUT DIGEST IS TAKEN, NOT DECLARED, so a verdict goes stale by + // construction: edit the lockfile and the key moves, so the old record is not + // found rather than found and wrong. + let dir = fixture("digest", TABLE); + assert_eq!( + record(&dir, &counts(&[("spdx-cargo", "99")])).status.code(), + Some(0) + ); + assert_eq!(check(&dir).status.code(), Some(2)); + + write(&dir, "Cargo.lock", &format!("{LOCKFILE}\n# a later edit\n")); + let outcome = check(&dir); + let (answer, cause) = (stdout(&outcome), stderr(&outcome)); + assert_eq!( + outcome.status.code(), + Some(0), + "a record taken over different bytes must not answer\n{answer}{cause}" + ); +} + +#[test] +fn the_report_is_pointer_only() { + // NON-NEGOTIABLE RULE 4, and it matters more here than almost anywhere else in + // the tree: an SBOM carries author names, email addresses and copyright + // holders. The finding may name a tracked path and counts, and nothing else. + let dir = fixture("pointer", TABLE); + assert_eq!( + record( + &dir, + &counts(&[ + ("nosupplier", "3"), + ("copyright-unset", "4"), + ("license-slashed", "2"), + ]), + ) + .status + .code(), + Some(0) + ); + + let outcome = check(&dir); + let (answer, cause) = (stdout(&outcome), stderr(&outcome)); + assert_eq!(outcome.status.code(), Some(2), "{answer}{cause}"); + for id in [ + "sbom-supplier-unset", + "sbom-copyright-unenriched", + "sbom-license-unenriched", + ] { + assert!(answer.contains(id), "{id} is not reported\n{answer}{cause}"); + } + // No document body, no package name, no scratch path. + for leak in ["NOASSERTION", "Copyright", "spdx.json", "/tmp/"] { + assert!( + !answer.contains(leak) && !cause.contains(leak), + "{leak} reached the report\n{answer}{cause}" + ); + } +} diff --git a/hk.pkl b/hk.pkl index 55d617aac..f4d33f25c 100644 --- a/hk.pkl +++ b/hk.pkl @@ -469,11 +469,25 @@ local gate = new Mapping { // vendor review rather than by anyone here who could notice. Globbed on the two // inputs that can move it — the lockfile it must agree with, and the workflows // whose pinned actions the scan catalogs (CLOUD-262). + // CLOUD-1318 retired `sbom-check` onto `policy/sbom-inventory.rego`. This step + // is now the PRODUCER only — it records the counts the module adjudicates, and + // `batten-check` is what decides over them. The profile and the globs are + // unchanged, which is the point: the scan's cost stays exactly where it was and + // what came off the corpus is the bats suite, not the gate. + // + // THE STEP NAME SURVIVES THE MIGRATION, which is the campaign's own convention + // and not a cosmetic choice — `claim-check`, `perf-pair`, `mutant`, + // `target-prune` and `config-lint` all kept theirs for this reason. Here the + // binding reader is `tests/hk-selection.bats:129`, which names the slow tier's + // steps literally and asserts each is skipped at `!slow` and included under + // `check`. That suite is governed, so `shell edit refused` declares one route + // and no override: renaming this step would need an edit to a frozen file in + // order to land a retirement. Measured — the rename reddened both cases. ["sbom-check"] { // `slow` tier: two whole-tree `syft scan dir:.` passes. profiles = List("slow") glob = List("Cargo.lock", ".github/workflows/*") - check = "mise run sbom-check" + check = "mise run record-sbom" } // The install path resolves a release asset BY NAME from a machine that has // never seen this repository, so a rename in `dist` that forgets one of its diff --git a/mise-tasks/hook-latency-drift.sh b/mise-tasks/hook-latency-drift.sh deleted file mode 100755 index cdbff1f11..000000000 --- a/mise-tasks/hook-latency-drift.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash -#MISE description="Report (never gate) whether the fast pre-commit tier still costs what its budget claims — a question about the world, so it runs on a clock" -# -# CLOUD-509. The commit half is `mise run hook-profile-check`, in the hk gate: -# the slow tier is skipped at pre-commit and still run by `check`. That gate -# cannot ask whether the fast tier is still FAST, because the answer is a wall -# clock rather than a property of the committed bytes — and a gate whose verdict -# comes from a measurement is not testing the commit (the lesson `lock-check` -# taught, one level down). So the cost question runs here, on a schedule. -# -# drift-tight the tier now costs MORE than its budget allows -# drift-loose the tier costs far LESS — the budget has gone slack -# unmeasurable the run did not produce a usable timing -# -# BOTH DIRECTIONS, on purpose. A budget that only complained about slowness -# would let every number rot upward: the tier gets faster, nobody re-derives the -# budget, and the ceiling stops bounding anything. That is the ratchet CLOUD-266 -# exists for, applied to this measurement. -# -# NOTHING HERE WRITES TO THE TREE. Re-baselining the budget is a deliberate -# commit, prompted by this report — a bot silently rewriting the number it is -# supposed to defend is exactly the failure a hand-authored budget prevents. -# -# WHY IT MEASURES `check --profile '!slow'` RATHER THAN A REAL COMMIT: `hk run -# pre-commit` stashes the worktree and applies fixers, so timing it would mutate -# the tree and measure the stash as much as the steps. The `check` hook runs the -# same step mapping with the same profile, read-only, which is the honest proxy -# for what a commit pays. -# -# Exit 0 in budget, 1 drift, 2 could-not-look. As with `timeout-drift`, exit 1 -# here is a REPORT: nothing is broken and no branch is at fault. -set -euo pipefail - -PROFILE="slow" - -# The budget, in seconds, written once as data — the placement `perf-assert`'s -# BUDGETS and `run-shape-guard`'s verdict-bearing list use. It is a ceiling -# derived from measurement (see CLOUD-509), not a target chosen in advance. -# -# SLACK is what keeps this from firing on noise: a shared runner's wall clock -# moves several seconds run to run, and a report that cries wolf is a report -# nobody reads. LOOSE_FACTOR is the other end — the tier has to be dramatically -# under budget before re-deriving is worth a commit. -# Measured 2026-08-13 on a session container: 8s, 7s, 8s over three runs of -# `hk check --all --profile '!slow'`, against 275s for the same gate with the -# tier enabled. 15s is that median with room for a slower runner, chosen from -# the measurement rather than in advance — and deliberately not so tight that -# the LOOSE_FACTOR floor (15/3 = 5s) sits above the measured 7s, which would -# make the report fire on its own baseline. -BUDGET_SECONDS="${BATTEN_HOOK_BUDGET_SECONDS:-15}" -SLACK_SECONDS="${BATTEN_HOOK_BUDGET_SLACK:-10}" -LOOSE_FACTOR="${BATTEN_HOOK_BUDGET_LOOSE_FACTOR:-3}" -RUNS="${BATTEN_HOOK_BUDGET_RUNS:-3}" - -drift=0 -# Pointer-only per non-negotiable rule 4: the measurement and the rule id, never -# a step's output. -report() { - echo "::error:: $1" >&2 - drift=$((drift + 1)) -} - -root=$(git rev-parse --show-toplevel 2>/dev/null || true) -if [[ -z "$root" ]]; then - echo "::error:: hook-latency-drift: not inside a git repository, so hk.pkl cannot be located." >&2 - exit 2 -fi -cd "$root" - -if ! command -v hk >/dev/null 2>&1; then - echo "::error:: hook-latency-drift: hk is not on PATH, so the tier cannot be timed." >&2 - exit 2 -fi - -scratch="$(mktemp -d)" -cleanup() { rm -rf "$scratch"; } -trap cleanup EXIT - -# The measurement. Deliberately NOT hyperfine: `perf` uses it for a -# millisecond-scale binary where 100 runs are cheap and the tail is the point, -# while this is a multi-second whole-gate run where 3 samples is already minutes -# of runner time. The median of a few runs is the honest instrument at this -# scale, and pretending to a p95 over 3 samples would be a number with no -# meaning behind it. -samples="" -for _ in $(seq 1 "$RUNS"); do - start=$SECONDS - # The status is deliberately ignored: a RED gate still takes time, and this - # task reports cost rather than correctness. A gate that cannot run at all is - # caught by the emptiness check below instead. - hk check --all --profile "!$PROFILE" >"$scratch/run.log" 2>&1 || true - samples+="$((SECONDS - start))"$'\n' -done - -count=$(grep -c '[0-9]' <<<"$samples" || true) -if [[ "$count" -lt "$RUNS" ]]; then - echo "::error:: hook-latency-drift: only $count of $RUNS runs produced a timing, so there is no measurement to judge. unmeasurable" >&2 - exit 2 -fi - -# Median of the sorted samples. Integer arithmetic throughout: the budget is in -# whole seconds and a fractional median would imply a precision the instrument -# does not have. -sorted=$(sort -n <<<"$samples" | grep '[0-9]') -median=$(sed -n "$(((RUNS + 1) / 2))p" <<<"$sorted") - -if [[ -z "$median" ]]; then - echo "::error:: hook-latency-drift: the samples did not yield a median. unmeasurable" >&2 - exit 2 -fi - -ceiling=$((BUDGET_SECONDS + SLACK_SECONDS)) -floor=$((BUDGET_SECONDS / LOOSE_FACTOR)) - -if [[ "$median" -gt "$ceiling" ]]; then - report "hook-latency-drift: the fast tier measured ${median}s against a ${BUDGET_SECONDS}s budget (+${SLACK_SECONDS}s slack). Either a step has grown, or one has joined the fast tier that belongs in the $PROFILE one. drift-tight" -elif [[ "$median" -lt "$floor" ]]; then - report "hook-latency-drift: the fast tier measured ${median}s against a ${BUDGET_SECONDS}s budget — under a ${LOOSE_FACTOR}x margin, so the budget has stopped bounding anything. Re-derive it in a deliberate commit. drift-loose" -fi - -if [[ "$drift" -ne 0 ]]; then - echo "::error:: hook-latency-drift: $drift budget(s) no longer match the measurement. This is a report, not a gate — nothing is broken and no branch is at fault; the number needs re-deriving in a commit of its own." >&2 - exit 1 -fi - -echo "hook-latency-drift: fast tier ${median}s over $RUNS run(s), within the ${BUDGET_SECONDS}s budget" diff --git a/mise-tasks/sbom-check.sh b/mise-tasks/sbom-check.sh deleted file mode 100755 index f95b9ecb3..000000000 --- a/mise-tasks/sbom-check.sh +++ /dev/null @@ -1,382 +0,0 @@ -#!/usr/bin/env bash -#MISE description="Gate: the SBOM inventories the tree it claims to — the cargo count matches Cargo.lock, and two scans agree once the volatile fields are removed" -# -# CLOUD-262. The inventory is published, which is what makes it worth gating: a -# wrong SBOM is not merely unused, it is a false claim about what shipped, and it -# is read by whoever is doing vendor review rather than by anyone here who could -# notice. Nothing in the Rust build reads these documents, so only this can. -# -# Three properties, because each fails a different way: -# -# sbom-empty An SBOM that catalogs nothing must not report green. A scan -# whose catalogers all missed would otherwise pass every -# equality check below trivially — two empty documents agree. -# sbom-package-drift The cargo count must equal `grep -c '^\[\[package\]\]' -# Cargo.lock`. COMPUTED, never hardcoded: the issue that -# specified this recorded 156 cargo and 175 total, and the -# total had already moved by 2 a day later as the workflow -# actions changed. A pinned total would fail on a true tree; -# a pinned cargo count would rot the first time a dependency -# lands. The relation is the invariant, not the number. -# sbom-unstable Two scans of one tree must produce identical bytes once the -# four fields that legitimately vary are removed. This is what -# makes the published document a function of the source rather -# than of when it was cut. -# -# The normalizer is the part that can go quietly wrong, so it is held two ways: it -# names exactly four leaves, and `tests/sbom-check.bats` drives a `syft` stub whose -# two runs differ in a package NAME and asserts this still fails. Widening it to -# make something pass would break that test first. -# -# It re-runs `mise-tasks/sbom.sh` rather than restating the flags — one definition of -# the invocation (§1), so this cannot certify bytes a release would not publish. -# Both runs go to scratch directories: a gate that rewrites the tree it judges -# cannot fail twice, and would launder drift into a clean second run. -# -# Exit 0 pass / 1 fail / 2 could-not-look, matching the other `*-check` programs. -# A gate listed in $MUTANT_GATES with no row here fails `mise run mutant`. -#MUTANT count-disagreement-passes|s/^\texit 1$/\texit 0/|a cargo count that disagrees with Cargo.lock fails - -set -euo pipefail - -# Resolved BEFORE the cd: `$0` may be relative, and moving first would leave this -# pointing at a sibling of whatever tree is being judged rather than of this file. -SBOM="$(cd "$(dirname "$0")" && pwd)/sbom.sh" - -cd "${SBOM_ROOT:-$(git rev-parse --show-toplevel)}" - -if [[ ! -x "$SBOM" ]]; then - echo "::error:: sbom-check: cannot execute $SBOM, so the inventory is unverified. That is a checkout problem, not a drifted SBOM." >&2 - exit 2 -fi - -# The expected count's source. Absent, there is no invariant to check against, and -# reporting green over that would be the vacuous pass this gate exists to prevent. -if [[ ! -f Cargo.lock ]]; then - echo "::error:: sbom-check: no Cargo.lock, so the expected package count is unknown. A gate that checks nothing must not report green." >&2 - exit 2 -fi - -scratch=$(mktemp -d) -trap 'rm -rf "$scratch"' EXIT - -violations=0 -report() { # pointer-only (rule 4): asset:line rule-id, never document contents - echo "$1 $2" >&2 - violations=$((violations + 1)) -} - -# THE EXPECTED CARGO COUNT IS THE LOCKFILE'S *SOURCED* PACKAGES, NOT ALL OF THEM -# (CLOUD-664). This clause compared against every `[[package]]` entry, which was -# right for as long as syft gave the local workspace member a registry purl. It -# stopped being right at syft 1.50.0, which deliberately does not -# (anchore/syft#5105): `batten` is `publish = false` and is in no registry, so a -# `pkg:cargo/batten@…` coordinate would assert a registry presence that does not -# exist. Measured 2026-08-23 at v0.0.106: 281 `[[package]]` entries, 280 carrying -# a `source`, and 280 cargo purls in the document — the one without a source is -# the workspace member, and it is the one with no purl. -# -# So the invariant is stated over the thing that actually predicts a purl: a -# lockfile entry with a `source` key is a registry or git dependency and gets one; -# an entry without is local to this workspace and does not. That also keeps -# holding if the workspace grows a second member, where subtracting a hardcoded 1 -# would not. -# -# `|| true` on the total for the reason it was always there — `grep -c` exits 1 on -# a zero count, which is a real answer here rather than a failure, and -# `sbom-empty` is what judges it. -lock_packages=$(grep -c '^\[\[package\]\]' Cargo.lock || true) -declared=$(grep -c '^source = ' Cargo.lock || true) - -if ! first=$(SBOM_OUT_DIR="$scratch/one" "$SBOM"); then - echo "::error:: sbom-check: could not derive the SBOM, so its contents are unverified." >&2 - exit 2 -fi -if ! second=$(SBOM_OUT_DIR="$scratch/two" "$SBOM"); then - echo "::error:: sbom-check: could not derive the SBOM a second time, so its stability is unverified." >&2 - exit 2 -fi - -# The asset paths come from `sbom`'s own KEY=VALUE output, so the names stay owned -# by the one script that decides them. -path_of() { # $1 = key, $2 = the KEY=VALUE block - sed -n "s/^$1=//p" <<<"$2" -} - -spdx_one=$(path_of spdx "$first") -cdx_one=$(path_of cdx "$first") -spdx_two=$(path_of spdx "$second") -cdx_two=$(path_of cdx "$second") - -for path in "$spdx_one" "$cdx_one" "$spdx_two" "$cdx_two"; do - if [[ -z "$path" ]] || [[ ! -f "$path" ]]; then - echo "::error:: sbom-check: sbom did not report a readable document path, so there is nothing to judge." >&2 - exit 2 - fi -done - -# Each format renders purls differently, so both are counted: a regression in one -# renderer is invisible if the gate only ever reads the other. -spdx_cargo=$(jq '[.packages[]? | .externalRefs[]? - | select(.referenceType == "purl") - | .referenceLocator - | select(startswith("pkg:cargo/"))] | length' "$spdx_one") -cdx_cargo=$(jq '[.components[]? | (.purl // "") - | select(startswith("pkg:cargo/"))] | length' "$cdx_one") - -if [[ "$spdx_cargo" -eq 0 ]] || [[ "$cdx_cargo" -eq 0 ]]; then - report "${spdx_one##*/}:0" "sbom-empty" -else - [[ "$spdx_cargo" -eq "$declared" ]] || report "${spdx_one##*/}:0" "sbom-package-drift ($spdx_cargo vs $declared)" - [[ "$cdx_cargo" -eq "$declared" ]] || report "${cdx_one##*/}:0" "sbom-package-drift ($cdx_cargo vs $declared)" -fi - -# The four leaves two scans of one tree legitimately differ in: SPDX stamps a fresh -# document namespace and creation time, CycloneDX a fresh serial number and -# timestamp. Deleting an absent key is a no-op in jq, so one expression serves both -# formats without asking which it is holding. -normalize() { - jq -S 'del(.documentNamespace, .creationInfo.created, .serialNumber, .metadata.timestamp)' "$1" -} - -compare() { # $1 = label, $2 = first run's document, $3 = second run's - if ! normalize "$2" >"$scratch/$1.a" || ! normalize "$3" >"$scratch/$1.b"; then - echo "::error:: sbom-check: could not normalize the $1 document, so its stability is unverified." >&2 - exit 2 - fi - cmp -s "$scratch/$1.a" "$scratch/$1.b" || report "${2##*/}:0" "sbom-unstable" -} - -compare spdx "$spdx_one" "$spdx_two" -compare cdx "$cdx_one" "$cdx_two" - -# --- one entry per thing depended on (CLOUD-664) ----------------------------- -# -# syft emits a component per REFERENCE SITE, so the document claimed 340 entries -# for 290 distinct things: 57 `pkg:github` entries for 9 unique actions, plus a -# `./action` component that is a relative path in this repository rather than a -# dependency of it. `sbom.sh` normalises that now; this is the clause that keeps -# it normalised, and it is deliberately a property of the DOCUMENT rather than of -# the normaliser — a cataloger that starts emitting a new inflated shape is caught -# without anyone having predicted which shape. -# -# THE SUBJECT IS EXEMPT, for the reason `sbom.sh` records at length: the document -# root and the workspace member are two roles, not two entries for one thing, and -# since syft stopped emitting a workspace purl they are indistinguishable by -# triple. Resolved from the document's own `DESCRIBES` edge, so a rename upstream -# does not turn this clause into a demand to corrupt the document. -# -# Pointer-only per rule 4: counts and the asset path, never a component name. -inflated=$(jq ' - ([.relationships[]? | select(.relationshipType == "DESCRIBES") | .relatedSpdxElement] | first) as $subject - | [.packages[]? | select(.SPDXID != $subject)] as $components - | { - entries: ($components | length), - distinct: ($components - | map([(.name // ""), (.versionInfo // ""), - ([.externalRefs[]? | select(.referenceType == "purl") | .referenceLocator] | first // "")]) - | unique | length), - pathlike: ($components | map(select((.name // "") | startswith("./"))) | length), - unversioned: ($components | map(select((.versionInfo // "") == "UNKNOWN")) | length), - subject: (if $subject == null then 0 else 1 end) - } - | "\(.entries) \(.distinct) \(.pathlike) \(.unversioned) \(.subject)" -' -r "$spdx_one") || inflated="" -if [[ -z "$inflated" ]]; then - echo "::error:: sbom-check: could not read component identity from ${spdx_one##*/}, so whether the inventory is inflated is unverified." >&2 - exit 2 -fi -read -r entries distinct pathlike unversioned subject <<<"$inflated" -# A document that DESCRIBES nothing is could-not-look, not a clean inventory: the -# subject is what the exemption above is computed from, so without it every -# following count is measured over the wrong set. -if [[ "$subject" -eq 0 ]]; then - echo "::error:: sbom-check: ${spdx_one##*/} carries no DESCRIBES relationship, so the document's own subject cannot be identified and component identity is unverified." >&2 - exit 2 -fi -if [[ "$entries" -ne "$distinct" ]] || [[ "$pathlike" -ne 0 ]] || [[ "$unversioned" -ne 0 ]]; then - report "${spdx_one##*/}:0" "sbom-components-inflated (entries=$entries distinct=$distinct pathlike=$pathlike unversioned=$unversioned)" -fi - -# --- supplier and originator (CLOUD-630) ------------------------------------- -# -# `supplier` was `NOASSERTION` on every cargo component. It is reachable with zero -# inference once the SPDX distinction is respected — `PackageSupplier` is who -# DISTRIBUTED the package, which the lockfile's resolution states, and -# `PackageOriginator` is who WROTE it, which `cargo metadata`'s `authors` answers -# or honestly does not. -# -# Both halves are checked, and the second is why this reads `cargo metadata` -# rather than only the document: a supplier count alone cannot tell an originator -# that agrees with the manifest from one that was copied from the supplier field. -# The agreement is what makes the two fields mean different things. -if ! meta=$(cargo metadata --format-version 1 --offline 2>/dev/null); then - echo "::error:: sbom-check: could not read cargo metadata, so whether the document's originators agree with the manifests is unverified." >&2 - exit 2 -fi -# `{"@": true}` for every package declaring at least one author. -authored=$(jq -c '[.packages[] | select((.authors // []) | length > 0) - | {key: "\(.name)@\(.version)", value: true}] | from_entries' <<<"$meta") || authored="" -if [[ -z "$authored" ]]; then - echo "::error:: sbom-check: could not read authorship from cargo metadata, so originator agreement is unverified." >&2 - exit 2 -fi -entities=$(jq -r --argjson authored "$authored" ' - ([.relationships[]? | select(.relationshipType == "DESCRIBES") | .relatedSpdxElement] | first) as $subject - | [.packages[]? - | select(.SPDXID != $subject) - | select(([.externalRefs[]? | select(.referenceType == "purl") | .referenceLocator] | first // "") - | startswith("pkg:github/") | not)] as $cargo - | { - cargo: ($cargo | length), - # The subject is not a cargo dependency and is excluded from that count — - # but it is the one component whose supplier a reader checks first, so it is - # asserted on its own rather than left unjudged by the exclusion. - subjectunset: ([.packages[]? | select(.SPDXID == $subject) - | select((.supplier // "NOASSERTION") == "NOASSERTION")] | length), - nosupplier: ($cargo | map(select((.supplier // "NOASSERTION") == "NOASSERTION")) | length), - # An originator is expected exactly where the manifest declares an author, - # and `NOASSERTION` exactly where it does not. Both directions count as a - # disagreement: a missing one loses data the tree states, and an invented one - # asserts authorship nobody claimed. - disagrees: ($cargo | map( - "\(.name // "")@\(.versionInfo // "")" as $key - | ((.originator // "NOASSERTION") != "NOASSERTION") as $set - | select($set != (($authored[$key] // false)))) | length), - # The three-way split CLOUD-629 asks for, which is the useful pointer here: a - # holder we read, an absence we determined, and the state this clause - # refuses. NONE is conformant and NOASSERTION is not, so counting them - # together would hide the only difference that matters. (No apostrophes in - # here: this program is a single-quoted shell string, and one ends it.) - holder: ($cargo | map(select(((.copyrightText // "NOASSERTION") | test("^Copyright"; "i")))) | length), - none: ($cargo | map(select((.copyrightText // "NOASSERTION") == "NONE")) | length), - unset: ($cargo | map(select(((.copyrightText // "NOASSERTION") == "NOASSERTION") - or ((.copyrightText // "") == ""))) | length), - # CLOUD-628. A cargo component whose license the manifest states and the - # document does not is the whole finding; one the manifest leaves empty is - # honest absence and is counted separately rather than refused, because - # guessing is what this must not do. The slash count is the second half: the - # deprecated cargo spelling is not a valid SPDX expression, so one reaching - # the document unrewritten is an unparseable field rather than a missing one. - nolicense: ($cargo | map(select(((.licenseConcluded // "NOASSERTION") == "NOASSERTION") - or ((.licenseConcluded // "") == ""))) | length), - slashed: ($cargo | map(select(((.licenseConcluded // "") | test("/")))) | length) - } - | "\(.cargo) \(.nosupplier) \(.disagrees) \(.subjectunset) \(.holder) \(.none) \(.unset) \(.nolicense) \(.slashed)" -' "$spdx_one") || entities="" -if [[ -z "$entities" ]]; then - echo "::error:: sbom-check: could not read supplier and originator from ${spdx_one##*/}, so those fields are unverified." >&2 - exit 2 -fi -read -r cargo_components nosupplier disagrees subjectunset holder none unset nolicense slashed <<<"$entities" -# Pointer-only per rule 4, and it matters more here than elsewhere in this file: -# an `authors` entry is a personal name and often an email address, so the finding -# carries counts and never a value. -if [[ "$nosupplier" -ne 0 ]] || [[ "$disagrees" -ne 0 ]] || [[ "$subjectunset" -ne 0 ]]; then - report "${spdx_one##*/}:0" "sbom-supplier-unset (cargo=$cargo_components no-supplier=$nosupplier originator-disagrees=$disagrees subject-unset=$subjectunset)" -fi - -# --- copyright (CLOUD-629) --------------------------------------------------- -# -# `copyrightText` was NOASSERTION on every component, and the field has no source -# in `cargo metadata` at all — it is read from the bytes `Cargo.lock` pins by -# checksum. The producer writes one of exactly two values and never NOASSERTION: -# the anchored holder line where the pinned sources carry one, and `NONE` where -# every pinned byte was searched and none does. Measured against `sbomcheck` -# 5.0.3, `NONE` is conformant and `NOASSERTION` is not, so this clause refuses -# only the third state — which the producer's own hard failure on an absent -# unpacked source has already made unreachable. -# -# Pointer-only, and this field needs it more than any other in the document: a -# copyright statement is a personal name, so echoing the value would publish names -# into every CI log that reads this gate. -if [[ "$unset" -ne 0 ]]; then - report "${spdx_one##*/}:0" "sbom-copyright-unenriched (cargo=$cargo_components holder=$holder none=$none unset=$unset)" -fi - -# --- license (CLOUD-628) ----------------------------------------------------- -# -# `cargo metadata` reports a license for every package in this tree and -# `cargo-deny` already gates on those same expressions, so this is the one field -# whose data was authoritative here all along and simply unused by the document. -# The clause refuses a component the manifest describes and the document does not, -# and separately refuses the deprecated slash spelling, which is not a valid SPDX -# expression — an unparseable value in a field whose purpose is to be parsed is -# worse than an honest NOASSERTION. -# -# Pointer-only: counts, never an expression or a package name. -if [[ "$nolicense" -ne 0 ]] || [[ "$slashed" -ne 0 ]]; then - report "${spdx_one##*/}:0" "sbom-license-unenriched (cargo=$cargo_components no-license=$nolicense slash-form=$slashed)" -fi - -# --- the pinned actions (CLOUD-667) ------------------------------------------ -# -# The 9 SHA-pinned actions were the last conformance gap. Two clauses, and the -# second is what keeps a committed table from rotting into a list nobody updates. -ACTIONS_TABLE="${SBOM_ACTIONS_TABLE:-}" -if [[ -z "$ACTIONS_TABLE" ]]; then - ACTIONS_TABLE="$(cd "$(dirname "$0")" && pwd)/sbom-actions.tsv" -fi -readonly ACTIONS_TABLE - -# 1. Every `pkg:github` component carries both fields. -actions=$(jq -r ' - [.packages[]? - | select(([.externalRefs[]? | select(.referenceType == "purl") | .referenceLocator] | first // "") - | startswith("pkg:github/"))] as $gh - | { - total: ($gh | length), - unset: ($gh | map(select(((.licenseConcluded // "NOASSERTION") == "NOASSERTION") - or ((.copyrightText // "NOASSERTION") == "NOASSERTION"))) | length) - } - | "\(.total) \(.unset)" -' "$spdx_one") || actions="" -if [[ -z "$actions" ]]; then - echo "::error:: sbom-check: could not read the action components from ${spdx_one##*/}, so their license and copyright are unverified." >&2 - exit 2 -fi -read -r action_total action_unset <<<"$actions" -if [[ "$action_unset" -ne 0 ]]; then - report "${spdx_one##*/}:0" "sbom-action-unenriched (actions=$action_total unset=$action_unset)" -fi - -# 2. THE DRIFT DETECTOR, and the reason a committed table is defensible at all. -# A pinned action's license is immutable, so recording it is a property of this -# commit — but only while the table still describes the pins the workflows carry. -# This fires on the one event that breaks that: a pin moving. A renovate bump that -# does not record the new commit's license fails the gate rather than silently -# degrading the document. -# -# Matched on repo AND sha together: a table row whose sha is stale is exactly the -# drift, so comparing the pair is the check. Pointer-only — the workflow file and -# line, never a license or a holder. -if [[ ! -r "$ACTIONS_TABLE" ]]; then - echo "::error:: sbom-check: cannot read ${ACTIONS_TABLE##*/}, so whether every pinned action is mapped is unverified." >&2 - exit 2 -fi -unmapped=0 -while IFS= read -r pin; do - [[ -n "$pin" ]] || continue - # `::@` - ref="${pin##*:}" - where="${pin%:*}" - repo="${ref%@*}" - # The table's key column is spelled exactly as this `uses:` line spells it, - # so the comparison is the whole reference against a key followed by a tab. - if ! grep -qF "$(printf '%s ' "$ref")" "$ACTIONS_TABLE"; then - echo "$where sbom-action-unmapped ($repo)" >&2 - unmapped=$((unmapped + 1)) - fi -done < <(grep -rnoE 'uses:[[:space:]]+[^[:space:]]+@[0-9a-f]{40}' .github/workflows/ 2>/dev/null | - sed -E 's@uses:[[:space:]]+@@' | sort -u) -if [[ "$unmapped" -ne 0 ]]; then - violations=$((violations + 1)) - echo "${ACTIONS_TABLE##*/}:0 sbom-action-unmapped (unmapped=$unmapped)" >&2 -fi - -if [[ "$violations" -ne 0 ]]; then - echo "::error:: sbom-check: $violations violation(s). Re-run 'mise run sbom' and inspect the documents; a count mismatch means a cataloger missed something, an unstable one means a field varies that the normalizer does not cover." >&2 - exit 1 -fi - -echo "sbom-check: $spdx_cargo cargo package(s) in both formats, matching Cargo.lock's $declared sourced entries of $lock_packages, $entries component(s) each a distinct thing, every one carrying a supplier and a license, $holder with a copyright holder and $none determined to have none, $action_total pinned action(s) all mapped, and two scans agree" diff --git a/mise.toml b/mise.toml index 1d42cb1b2..1ba6d1b84 100644 --- a/mise.toml +++ b/mise.toml @@ -475,7 +475,7 @@ CI_FANIN_WORKFLOW = ".github/workflows/ci.yml" # which is a property of the world and belongs on a clock (`lock-complete`). REGORUS_OPA_COMPLIANCE = "1.2.0" REGORUS_OPA_COMPLIANCE_FOR = "0.11" -MUTANT_GATES = "alive,attestation-check,awk-regex-check,bats-invocation,batten-glob-check,board-diff-overlap,board-payloads,board-sweep,branch-age-check,cap-drift,ci-hygiene,ci-lease-precondition,ci-parity,ci-slow-needed,ci-suite-lane,ci-tools-check,claim-before-code,claim-race-check,claimed-keys,closing-key-check,coderabbit-config-check,commit-hygiene,connector-allow-guard,connector-allow-resolve,container-preflight,darwin-link,deferral-check,denials-outlive-the-turn,digest-major-agreement,doctor,done-check,done-pr-check,duplicate-close-check,evaluator-closure-check,evaluator-io-check,filed-here,finding-sink-check,forge-verdict-required,graph-check,harness-grant,harness-wiring,hk-fix-selection,hook-matcher-check,hook-pin-check,in-progress-drain,install-check,land,land-divergence-assert,land-lock,land-lock-check,landed-check,landing-loop,leased-push,license-table-check,linear-check,lock-complete,macos-link-check,mcp-allow-check,mcp-attach-check,mcp-timeout-budget,merged-pr-keys,mise-action-floor,mise-pin-agreement,module-map-check,msrv-pin-agreement,no-doctests,nonverdict-assert,ntia-check,obligations-bound,perf-assert,perf-compare,perf-gate,pinned-toolchain,pipefail-grep-check,plan-complete,pr-unsubscribed,privileged-lane,prose-only,publish-credential-check,ready-cites-check,ready-guard,ready-lint,reclaim-census,release-assets-check,release-due,release-tag-shape,release-tracking-check,released,remedy-authorship,report-only-check,review-answered,run-shape,rust-paths-check,sbom,sbom-check,serena-mcp,shell-hygiene,shell-retirement,shell-write-advisory,signing-posture,sonar-gate,spec-ref-check,stop-posture,stop-posture-check,suite-bench-check,suite-subject-retirable,task-substitution,timeout-check,token-bench-check,transcript-corpus-check,tree-clean,trunk-based,validator-verdict-clean,verdict-routes-resolve,verified,weakens-declared" +MUTANT_GATES = "alive,attestation-check,awk-regex-check,bats-invocation,batten-glob-check,board-diff-overlap,board-payloads,board-sweep,branch-age-check,cap-drift,ci-hygiene,ci-lease-precondition,ci-parity,ci-slow-needed,ci-suite-lane,ci-tools-check,claim-before-code,claim-race-check,claimed-keys,closing-key-check,coderabbit-config-check,commit-hygiene,connector-allow-guard,connector-allow-resolve,container-preflight,darwin-link,deferral-check,denials-outlive-the-turn,digest-major-agreement,doctor,done-check,done-pr-check,duplicate-close-check,evaluator-closure-check,evaluator-io-check,filed-here,finding-sink-check,forge-verdict-required,graph-check,harness-grant,harness-wiring,hk-fix-selection,hook-matcher-check,hook-pin-check,in-progress-drain,install-check,land,land-divergence-assert,land-lock,land-lock-check,landed-check,landing-loop,leased-push,license-table-check,linear-check,lock-complete,macos-link-check,mcp-allow-check,mcp-attach-check,mcp-timeout-budget,merged-pr-keys,mise-action-floor,mise-pin-agreement,module-map-check,msrv-pin-agreement,no-doctests,nonverdict-assert,ntia-check,obligations-bound,perf-assert,perf-compare,perf-gate,pinned-toolchain,pipefail-grep-check,plan-complete,pr-unsubscribed,privileged-lane,prose-only,publish-credential-check,ready-cites-check,ready-guard,ready-lint,reclaim-census,release-assets-check,release-due,release-tag-shape,release-tracking-check,released,remedy-authorship,report-only-check,review-answered,run-shape,rust-paths-check,sbom,sbom-inventory,serena-mcp,shell-hygiene,shell-retirement,shell-write-advisory,signing-posture,sonar-gate,spec-ref-check,stop-posture,stop-posture-check,suite-bench-check,suite-subject-retirable,task-substitution,timeout-check,token-bench-check,transcript-corpus-check,tree-clean,trunk-based,validator-verdict-clean,verdict-routes-resolve,verified,weakens-declared" # --- GitHub reachability behind an egress proxy (Claude Code web sandbox etc.) --- # mise resolves every tool's release through GitHub's *API* host, api.github.com. @@ -1428,6 +1428,166 @@ set -eu cargo run --quiet -p batten -- policy test ''' +[tasks.record-sbom] +description = "Effect: derive the SBOM twice OUTSIDE the engine and record the counts where `batten check` reads them (CLOUD-1318)" +# The producer half of `sbom-check`'s retirement. `policy/sbom-inventory.rego` +# decides; this runs the scan that surface structurally cannot. +# +# A SEPARATE TASK FROM `record-verdicts`, and the reason is cost rather than +# taste. That task is wired into three hk steps on their own globs and runs on +# every one; this is two whole-tree `syft scan dir:.` passes, which is why the +# retired gate declared `profiles = List("slow")` and globbed on `Cargo.lock` plus +# the workflows. Folding these in would put both scans on every hk invocation that +# validates a config file. `hk.pkl` keeps the profile and the globs and points +# here instead. +# +# AN INLINE TASK RATHER THAN A `mise-tasks/` PROGRAM, and it is forced: +# `governed_at_head` selects any `mise-tasks/` path carrying a shebang or a +# `#MISE description=`, so a new program there is `shell add refused` — refused at +# `deny`, in the same change that is retiring one. +# +# IT RE-RUNS `sbom.sh` RATHER THAN RESTATING ITS FLAGS — one definition of the +# invocation, so this cannot certify bytes a release would not publish. Both runs +# go to scratch directories: a producer that rewrote the tree it measures could +# not measure it twice, and would launder drift into a clean second run. +# +# WHAT IT DOES NOT DECIDE. Every line here is a COUNT. Whether a count is a +# finding — and which of the four verdict classes it belongs to — is +# `policy/sbom-inventory.rego`'s. The two counts readable from committed text are +# deliberately NOT recorded: `Cargo.lock`'s sourced entries and the action pins +# against the licence table are `line_sources` the module reads itself, so this +# producer cannot get them wrong on its behalf. +# +# `bash -c` DECLARED RATHER THAN ASSUMED, and it is not decoration: the body uses +# herestrings (`<<<`) to feed `sed` and `jq`, which are a bashism — measured, the +# first run of this task under the default shell died at `sh: Syntax error: +# redirection unexpected` before reaching a single count. Declaring the shell +# drops `-e` (`.claude/rules/toolchain.md`), which is why `set -eu` is the first +# line: every failure here must stop the body rather than record a partial scan. +shell = "bash -c" +run = ''' +set -eu + +cd "${SBOM_ROOT:-$(git rev-parse --show-toplevel)}" + +SBOM="$PWD/mise-tasks/sbom.sh" +if [ ! -x "$SBOM" ]; then + echo "::error:: record-sbom: cannot execute $SBOM, so the inventory is unverified" >&2 + exit 1 +fi + +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT + +if ! first=$(SBOM_OUT_DIR="$scratch/one" "$SBOM"); then + echo "::error:: record-sbom: could not derive the SBOM, so its contents are unverified" >&2 + exit 1 +fi +if ! second=$(SBOM_OUT_DIR="$scratch/two" "$SBOM"); then + echo "::error:: record-sbom: could not derive the SBOM a second time, so its stability is unverified" >&2 + exit 1 +fi + +# The asset paths come from `sbom`'s own KEY=VALUE output, so the names stay owned +# by the one script that decides them. +path_of() { sed -n "s/^$1=//p" <<<"$2"; } + +spdx_one=$(path_of spdx "$first") +cdx_one=$(path_of cdx "$first") +spdx_two=$(path_of spdx "$second") +cdx_two=$(path_of cdx "$second") + +for path in "$spdx_one" "$cdx_one" "$spdx_two" "$cdx_two"; do + if [ -z "$path" ] || [ ! -f "$path" ]; then + echo "::error:: record-sbom: sbom did not report a readable document path, so there is nothing to record" >&2 + exit 1 + fi +done + +# The four leaves two scans of one tree legitimately differ in: SPDX stamps a +# fresh document namespace and creation time, CycloneDX a fresh serial number and +# timestamp. Deleting an absent key is a no-op in jq, so one expression serves +# both formats without asking which it is holding. +normalize() { jq -S 'del(.documentNamespace, .creationInfo.created, .serialNumber, .metadata.timestamp)' "$1"; } + +stable() { # $1 = first run's document, $2 = the second run's + if normalize "$1" >"$scratch/a" && normalize "$2" >"$scratch/b" && cmp -s "$scratch/a" "$scratch/b"; then + echo yes + else + echo no + fi +} + +# Each format renders purls differently, so both are counted: a regression in one +# renderer is invisible if only the other is ever read. +spdx_cargo=$(jq '[.packages[]? | .externalRefs[]? + | select(.referenceType == "purl") + | .referenceLocator + | select(startswith("pkg:cargo/"))] | length' "$spdx_one") +cdx_cargo=$(jq '[.components[]? | (.purl // "") + | select(startswith("pkg:cargo/"))] | length' "$cdx_one") + +if ! meta=$(cargo metadata --format-version 1 --offline 2>/dev/null); then + echo "::error:: record-sbom: could not read cargo metadata, so originator agreement is unverified" >&2 + exit 1 +fi +authored=$(jq -c '[.packages[] | select((.authors // []) | length > 0) + | {key: "\(.name)@\(.version)", value: true}] | from_entries' <<<"$meta") + +# THE SUBJECT IS EXEMPT from the component counts, for the reason `sbom.sh` +# records at length: the document root and the workspace member are two roles, not +# two entries for one thing, and since syft stopped emitting a workspace purl they +# are indistinguishable by triple. Resolved from the document's own DESCRIBES +# edge, so a rename upstream does not turn this into a demand to corrupt the file. +counts=$(jq -r --argjson authored "$authored" ' + ([.relationships[]? | select(.relationshipType == "DESCRIBES") | .relatedSpdxElement] | first) as $subject + | [.packages[]? | select(.SPDXID != $subject)] as $components + | [$components[] + | select(([.externalRefs[]? | select(.referenceType == "purl") | .referenceLocator] | first // "") + | startswith("pkg:github/") | not)] as $cargo + | [$components[] + | select(([.externalRefs[]? | select(.referenceType == "purl") | .referenceLocator] | first // "") + | startswith("pkg:github/"))] as $gh + | { + subject: (if $subject == null then 0 else 1 end), + entries: ($components | length), + distinct: ($components + | map([(.name // ""), (.versionInfo // ""), + ([.externalRefs[]? | select(.referenceType == "purl") | .referenceLocator] | first // "")]) + | unique | length), + pathlike: ($components | map(select((.name // "") | startswith("./"))) | length), + unversioned: ($components | map(select((.versionInfo // "") == "UNKNOWN")) | length), + nosupplier: ($cargo | map(select((.supplier // "NOASSERTION") == "NOASSERTION")) | length), + subjectunset: ([.packages[]? | select(.SPDXID == $subject) + | select((.supplier // "NOASSERTION") == "NOASSERTION")] | length), + disagrees: ($cargo | map( + "\(.name // "")@\(.versionInfo // "")" as $key + | ((.originator // "NOASSERTION") != "NOASSERTION") as $set + | select($set != (($authored[$key] // false)))) | length), + copyunset: ($cargo | map(select(((.copyrightText // "NOASSERTION") == "NOASSERTION") + or ((.copyrightText // "") == ""))) | length), + nolicense: ($cargo | map(select(((.licenseConcluded // "NOASSERTION") == "NOASSERTION") + or ((.licenseConcluded // "") == ""))) | length), + slashed: ($cargo | map(select(((.licenseConcluded // "") | test("/")))) | length), + actionunset: ($gh | map(select(((.licenseConcluded // "NOASSERTION") == "NOASSERTION") + or ((.copyrightText // "NOASSERTION") == "NOASSERTION"))) | length) + } + | "subject \(.subject)\nentries \(.entries)\ndistinct \(.distinct)\npathlike \(.pathlike)\nunversioned \(.unversioned)\nnosupplier \(.nosupplier)\nsubject-unset \(.subjectunset)\noriginator-disagrees \(.disagrees)\ncopyright-unset \(.copyunset)\nlicense-unset \(.nolicense)\nlicense-slashed \(.slashed)\naction-unset \(.actionunset)" +' "$spdx_one") + +# A REDUCTION, NEVER THE DOCUMENT. Only integers and two yes/no tokens reach the +# record: the SBOM carries author names, email addresses and copyright holders, +# and `tools.rs` names a tool's report as the likeliest place in this family for +# personal data to appear. +{ + printf 'spdx-cargo %s\n' "$spdx_cargo" + printf 'cdx-cargo %s\n' "$cdx_cargo" + printf 'spdx-stable %s\n' "$(stable "$spdx_one" "$spdx_two")" + printf 'cdx-stable %s\n' "$(stable "$cdx_one" "$cdx_two")" + printf '%s\n' "$counts" +} | cargo run --quiet -p batten -- record tool sbom +''' + [tasks.record-verdicts] description = "Effect: run each declared third-party validator OUTSIDE the engine and record its verdict where `batten check` reads it (CLOUD-1265)" # THE HALF THAT WAS MISSING. `crates/batten/src/tools.rs` reads diff --git a/policy/sbom-inventory.rego b/policy/sbom-inventory.rego new file mode 100644 index 000000000..1a82ec6f8 --- /dev/null +++ b/policy/sbom-inventory.rego @@ -0,0 +1,591 @@ +# METADATA +# description: | +# The successor for `sbom-check` (CLOUD-262, retired under CLOUD-1318). +# +# The published SBOM is read by whoever is doing vendor review rather than by +# anyone here who could notice it is wrong, and nothing in the Rust build reads +# it — so a wrong inventory is not merely unused, it is a false claim about what +# shipped. That is what makes it worth gating. +# +# THE SCAN STAYS OUTSIDE, and that is house style §5 rather than a workaround: +# `check` is `read` and structurally cannot spawn, so `syft` remains a command on +# PATH (§9's prior art) and `mise-tasks/sbom.sh` remains the producer that +# derives the documents. What moved here is the ADJUDICATION — the half that had +# no successor, because a module asking what a scan found read undefined and +# decided nothing. +# +# TWO OF THE PREDICATES NEED NO RECORD AT ALL, which is what keeps the producer's +# trusted surface narrow. `sbom-package-drift`'s expected count is +# `Cargo.lock`'s own `source = ` lines, and `sbom-action-unmapped` is every +# SHA-pinned `uses:` in a workflow against `mise-tasks/sbom-actions.tsv`'s key +# column. Both are properties of committed text, so they are decided here from +# `input.tree.lines` and the producer cannot get them wrong on this module's +# behalf. Only the counts that require reading a derived document — which this +# surface cannot open — travel through `input.tree["tool-verdict"]`. +# +# THE EXPECTED COUNT IS THE LOCKFILE'S *SOURCED* PACKAGES, NOT ALL OF THEM +# (CLOUD-664). syft 1.50.0 deliberately gives the local workspace member no +# registry purl (anchore/syft#5105): `batten` is `publish = false` and is in no +# registry, so a `pkg:cargo/batten@...` coordinate would assert a registry +# presence that does not exist. An entry carrying a `source` key is a registry or +# git dependency and gets a purl; one without is local to this workspace and does +# not. Stating the invariant over the thing that actually predicts a purl also +# keeps holding if the workspace grows a second member, where subtracting a +# hardcoded 1 would not. +# +# THREE ANSWERS, AND EMPTY IS A FINDING HERE. ABSENT is could-not-look — nothing +# has scanned this tree, which is the ordinary state on a checkout whose globs +# never fired, and refusing there would deny every clone until a producer runs. +# PRESENT AND EMPTY is the producer having recorded nothing, which would make +# every count below pass over an absent key, so it refuses — `hook-profile`'s +# reading, for `hook-profile`'s reason. +# +# THE BRACKETS ARE NOT STYLE: the schema file carries a hyphen, so the dotted +# form is a parse error reported as `invalid schema reference`. +# THIS BLOCK IS YAML AND MUST STAY THE LAST COMMENT BLOCK BEFORE `package`. +# schemas: +# - input: schema["policy-input.schema"] +package batten.sbom_inventory + +import rego.v1 + +rules contains "sbom-empty" + +rules contains "sbom-unrecorded" + +rules contains "sbom-package-drift" + +rules contains "sbom-unstable" + +rules contains "sbom-components-inflated" + +rules contains "sbom-supplier-unset" + +rules contains "sbom-copyright-unenriched" + +rules contains "sbom-license-unenriched" + +rules contains "sbom-action-unenriched" + +rules contains "sbom-action-unmapped" + +# The lockfile the cargo count is stated against, and the table every SHA-pinned +# action must appear in. Both are committed text this row declares as +# `line_sources`, so they are read here rather than trusted from a record. +lockfile := "Cargo.lock" + +actions_table := "mise-tasks/sbom-actions.tsv" + +# The recorded scan, guarded. `null` is a hard evaluation FAULT under `some .. in` +# rather than a silent miss, and an id nothing recorded is absent from the map. +scan := verdict if { + is_object(input.tree["tool-verdict"]) + verdict := input.tree["tool-verdict"].sbom +} + +# A recorded count, as a number. A key the producer did not write leaves this +# undefined, so every rule reading it abstains rather than comparing against zero +# — the difference between "the scan says none" and "nobody recorded it". +count_of(key) := value if { + raw := scan[key] + value := to_number(raw) +} + +# --- the scan is unusable ---------------------------------------------------- + +# AN SBOM THAT CATALOGS NOTHING MUST NOT REPORT GREEN. A scan whose catalogers all +# missed would otherwise pass every equality below trivially: two empty documents +# agree, and an empty count matches an empty count. +violation contains { + "rule": "sbom-empty", + "verdict": "tool read broken", + "subjects": [{"count": count_of(format)}], +} if { + some format in ["spdx-cargo", "cdx-cargo"] + count_of(format) == 0 +} + +# PRESENT AND EMPTY: the producer ran and wrote no counts, so nothing below has an +# input. Told apart from ABSENT by `is_object` plus the count — an id nothing +# recorded never binds `scan` at all. +violation contains { + "rule": "sbom-unrecorded", + "verdict": "tool read broken", + "subjects": [{"artifact": "sbom"}], +} if { + is_object(scan) + count(scan) == 0 +} + +# TWO SCANS OF ONE TREE MUST PRODUCE IDENTICAL BYTES once the fields that +# legitimately vary are removed — a fresh document namespace and creation time in +# SPDX, a fresh serial number and timestamp in CycloneDX. This is what makes the +# published document a function of the source rather than of when it was cut. +violation contains { + "rule": "sbom-unstable", + "verdict": "tool read broken", + "subjects": [{"artifact": format}], +} if { + some format in ["spdx-stable", "cdx-stable"] + scan[format] != "yes" +} + +# A DOCUMENT THAT DESCRIBES NOTHING is could-not-look rather than a clean +# inventory: the subject is what the component counts are measured against, so +# without it every one of them is taken over the wrong set. +violation contains { + "rule": "sbom-unrecorded", + "verdict": "tool read broken", + "subjects": [{"artifact": "describes"}], +} if { + count_of("subject") == 0 +} + +# --- the document disagrees with the tree ------------------------------------ + +# The lockfile's own lines, or NOTHING. Binding this separately is what keeps an +# unreadable lockfile out of the comparison below: a comprehension over an absent +# key yields an empty array, so a `declared` derived straight from +# `input.tree.lines` would read 0 and report every real count as drift. Undefined +# here leaves every rule that needs it abstaining, and the `missing` clause is +# what says so out loud. +lock_lines := input.tree.lines[lockfile] + +# The lockfile entries that predict a purl: a `source` key means a registry or git +# dependency. Counted from the committed text, so the producer is not trusted for +# the number this whole clause is stated against. +declared := count([line | + some line in lock_lines + startswith(line, "source = ") +]) + +# COMPUTED, NEVER HARDCODED. The issue that specified this recorded 156 cargo and +# 175 total, and the total had already moved by 2 a day later as the workflow +# actions changed. A pinned number would fail on a true tree; the relation is the +# invariant. +# +# Each format is counted separately because they render purls differently, so a +# regression in one renderer is invisible to a gate that only ever reads the other. +violation contains { + "rule": "sbom-package-drift", + "verdict": "manifest count wrong", + "subjects": [{"path": lockfile}, {"count": count_of(format)}], +} if { + # THE LOCKFILE WAS READ, and this guard is NOT redundant with `lock_lines` + # being undefined — measured by `test_an_unreadable_lockfile_reports_no_drift`, + # which was red without it. A Rego comprehension whose body references an + # undefined variable yields an EMPTY array rather than undefined, so `declared` + # still resolves to 0 and every honest count reads as drift against a file + # nobody opened. Binding the lines separately was not enough; the rule has to + # demand them. + is_array(lock_lines) + some format in ["spdx-cargo", "cdx-cargo"] + count_of(format) != 0 + count_of(format) != declared +} + +# ONE ENTRY PER THING DEPENDED ON (CLOUD-664). syft emits a component per +# REFERENCE SITE, so the document once claimed 340 entries for 290 distinct +# things: 57 `pkg:github` entries for 9 unique actions, plus a `./action` +# component that is a relative path in this repository rather than a dependency of +# it. `sbom.sh` normalises that; this keeps it normalised, and it is deliberately a +# property of the DOCUMENT rather than of the normaliser — a cataloger that starts +# emitting a new inflated shape is caught without anyone having predicted which. +violation contains { + "rule": "sbom-components-inflated", + "verdict": "manifest count wrong", + "subjects": [{"count": count_of("entries")}], +} if { + count_of("entries") != count_of("distinct") +} + +violation contains { + "rule": "sbom-components-inflated", + "verdict": "manifest count wrong", + "subjects": [{"count": count_of(shape)}], +} if { + some shape in ["pathlike", "unversioned"] + count_of(shape) != 0 +} + +# --- fields the tree states and the document does not ------------------------ + +# `supplier` is who DISTRIBUTED the package, which the lockfile's resolution +# states, and `originator` is who WROTE it, which `cargo metadata`'s `authors` +# answers or honestly does not. Both halves are counted, because a supplier count +# alone cannot tell an originator that agrees with the manifest from one copied +# out of the supplier field — the agreement is what makes the two fields mean +# different things. +# +# Pointer-only matters more here than anywhere else in this module: an `authors` +# entry is a personal name and often an email address, so the finding carries +# counts and never a value. +violation contains { + "rule": "sbom-supplier-unset", + "verdict": "manifest state missing", + "subjects": [{"count": count_of(field)}], +} if { + some field in ["nosupplier", "originator-disagrees", "subject-unset"] + count_of(field) != 0 +} + +# `copyrightText` has no source in `cargo metadata` at all — it is read from the +# bytes `Cargo.lock` pins by checksum. The producer writes one of exactly two +# values and never NOASSERTION: an anchored holder line where the pinned sources +# carry one, and `NONE` where every pinned byte was searched and none does. +# Measured against `sbomcheck` 5.0.3, `NONE` is conformant and `NOASSERTION` is +# not, so only the third state is refused here. +# +# This field needs pointer-only more than any other: a copyright statement is a +# personal name, so echoing the value would publish names into every CI log. +violation contains { + "rule": "sbom-copyright-unenriched", + "verdict": "manifest state missing", + "subjects": [{"count": count_of("copyright-unset")}], +} if { + count_of("copyright-unset") != 0 +} + +# `cargo metadata` reports a license for every package in this tree and +# `cargo-deny` already gates those same expressions, so this is the one field +# whose data was authoritative all along and simply unused by the document. The +# slash count is the second half: the deprecated cargo spelling is not a valid +# SPDX expression, so one reaching the document unrewritten is an unparseable +# field rather than a missing one — worse than an honest NOASSERTION, in a field +# whose whole purpose is to be parsed. +violation contains { + "rule": "sbom-license-unenriched", + "verdict": "manifest state missing", + "subjects": [{"count": count_of(field)}], +} if { + some field in ["license-unset", "license-slashed"] + count_of(field) != 0 +} + +# Every `pkg:github` component carries both a license and a copyright. +violation contains { + "rule": "sbom-action-unenriched", + "verdict": "manifest state missing", + "subjects": [{"count": count_of("action-unset")}], +} if { + count_of("action-unset") != 0 +} + +# --- the pinned actions, decided from committed text ------------------------- + +# Every workflow line carrying a SHA-pinned `uses:`. +# +# A MATCH TEST RATHER THAN A CAPTURE, which is forced rather than chosen: this +# build carries `regex.match` and nothing that returns submatches — every other +# module here reaches for the same one, and `shell-retirement.rego` says so at its +# own site ("`indexof` plus a NAME TEST rather than a capture"). So the reference +# is never extracted; the question is asked the other way round below. +# +# The pattern is a `[[pattern]]` row rather than an inline regex, which the loader +# refuses outright: one concept, one spelling. +pinned contains line if { + some file, lines in input.tree.lines + startswith(file, ".github/workflows/") + some line in lines + regex.match(data.batten.patterns["sbom-action-pin"], line) +} + +# The table's own lines, or NOTHING — `lock_lines`' reason exactly. An absent +# table would leave `mapped` empty and report every pin as unmapped, which is the +# could-not-look answer dressed as a finding. +table_lines := input.tree.lines[actions_table] + +# The key column, one per row. The table is TSV and its key is spelled exactly as +# a `uses:` line spells the reference, which is what lets the containment test +# below stand in for the extraction this build cannot do. +keys contains key if { + some row in table_lines + key := trim_space(split(row, "\t")[0]) + key != "" +} + +# A pinned line whose reference some row declares. Matched on repo AND sha +# together, because the key carries both — a table row whose sha is stale does not +# match the line that moved, which is exactly the drift this detects. +mapped contains line if { + some line in pinned + some key in keys + contains(line, key) +} + +# THE DRIFT DETECTOR, and the reason a committed table is defensible at all. A +# pinned action's license is immutable, so recording it is a property of this +# commit — but only while the table still describes the pins the workflows carry. +# This fires on the one event that breaks that: a pin moving. A renovate bump that +# does not record the new commit's license fails rather than silently degrading +# the document. +# +# Matched on repo AND sha together, because a table row whose sha is stale is +# exactly the drift. +violation contains { + "rule": "sbom-action-unmapped", + "verdict": "pin table missing", + "subjects": [{"path": actions_table}, {"count": count(unmapped)}], +} if { + count(unmapped) > 0 +} + +# POINTER-ONLY: a count and the table's path. The `uses:` line itself carries a +# repository name and a sha, and the finding names neither. +unmapped contains line if { + # The table was READ, or this is could-not-look rather than a tree where + # nothing is mapped. + is_array(table_lines) + some line in pinned + not line in mapped +} + +# COULD NOT LOOK IS A FINDING, NOT SILENCE. A declared source that would not parse +# belongs in `input.tree.missing`, and a module that iterates only what it could +# read reports green over a file it never opened. +violation contains { + "rule": "sbom-unrecorded", + "verdict": "tool read broken", + "subjects": [{"path": path}], +} if { + some path, _ in input.tree.missing +} + +# --- the load-time tier ------------------------------------------------------ +# +# These pin the PREDICATE. They cannot pin that the ENGINE composes the +# `tool-verdict` key from the tool, its pin and the input's digest, nor that it +# fills `input.tree.lines` for the two declared sources — a `with input as` case +# fabricates the very shape the engine may be unable to produce (CLOUD-845, +# CLOUD-857). `crates/batten/tests/it/sbom_inventory.rs` is that tier. + +recorded(verdict) := {"tree": { + "tool-verdict": {"sbom": verdict}, + "lines": {"Cargo.lock": [], "mise-tasks/sbom-actions.tsv": []}, + "missing": {}, +}} + +# A scan agreeing with a two-entry lockfile, every field enriched. +clean := { + "spdx-cargo": "2", + "cdx-cargo": "2", + "spdx-stable": "yes", + "cdx-stable": "yes", + "subject": "1", + "entries": "2", + "distinct": "2", + "pathlike": "0", + "unversioned": "0", + "nosupplier": "0", + "originator-disagrees": "0", + "subject-unset": "0", + "copyright-unset": "0", + "license-unset": "0", + "license-slashed": "0", + "action-unset": "0", +} + +# The same, over a lockfile whose sourced entries the counts agree with. +tree(verdict) := {"tree": { + "tool-verdict": {"sbom": verdict}, + "lines": { + "Cargo.lock": ["source = \"registry+one\"", "source = \"registry+two\""], + "mise-tasks/sbom-actions.tsv": [], + }, + "missing": {}, +}} + +test_a_clean_scan_agreeing_with_the_lockfile_is_clean if { + count(violation) == 0 with input as tree(clean) +} + +test_a_cargo_count_that_disagrees_with_the_lockfile_is_refused if { + some v in violation with input as tree(object.union(clean, {"spdx-cargo": "3"})) + v.verdict == "manifest count wrong" +} + +# THE OTHER RENDERER. A gate reading only SPDX is blind to a CycloneDX regression. +test_the_cyclonedx_count_is_judged_too if { + some v in violation with input as tree(object.union(clean, {"cdx-cargo": "5"})) + v.verdict == "manifest count wrong" +} + +test_an_empty_catalog_is_never_read_as_agreement if { + some v in violation with input as tree(object.union(clean, {"spdx-cargo": "0", "cdx-cargo": "0"})) + v.verdict == "tool read broken" +} + +# AN EMPTY CATALOG MUST NOT ALSO READ AS DRIFT: the count is zero against a +# two-entry lockfile, and reporting both would send the author after a cataloger +# and a normaliser at once. `sbom-empty` owns it. +test_an_empty_catalog_is_not_also_reported_as_drift if { + ids := {v.rule | some v in violation} with input as tree(object.union(clean, {"spdx-cargo": "0", "cdx-cargo": "0"})) + not "sbom-package-drift" in ids +} + +test_two_scans_that_disagree_are_refused if { + some v in violation with input as tree(object.union(clean, {"spdx-stable": "no"})) + v.rule == "sbom-unstable" +} + +test_an_inflated_component_set_is_refused if { + some v in violation with input as tree(object.union(clean, {"distinct": "1"})) + v.rule == "sbom-components-inflated" +} + +test_a_pathlike_component_is_refused if { + some v in violation with input as tree(object.union(clean, {"pathlike": "1"})) + v.rule == "sbom-components-inflated" +} + +test_a_document_describing_nothing_is_could_not_look if { + some v in violation with input as tree(object.union(clean, {"subject": "0"})) + v.verdict == "tool read broken" +} + +test_an_unset_supplier_is_refused if { + some v in violation with input as tree(object.union(clean, {"nosupplier": "4"})) + v.rule == "sbom-supplier-unset" +} + +# THE AGREEMENT HALF, which a supplier count alone cannot see. +test_an_originator_disagreeing_with_the_manifest_is_refused if { + some v in violation with input as tree(object.union(clean, {"originator-disagrees": "1"})) + v.rule == "sbom-supplier-unset" +} + +test_an_unset_copyright_is_refused if { + some v in violation with input as tree(object.union(clean, {"copyright-unset": "7"})) + v.rule == "sbom-copyright-unenriched" +} + +test_a_slash_form_license_is_refused if { + some v in violation with input as tree(object.union(clean, {"license-slashed": "1"})) + v.rule == "sbom-license-unenriched" +} + +test_an_unenriched_action_is_refused if { + some v in violation with input as tree(object.union(clean, {"action-unset": "2"})) + v.rule == "sbom-action-unenriched" +} + +# --- the pinned actions, over committed text rather than a record -------------- + +# A workflow and a table, so the pin clauses have something to decide over. +workflows(uses, rows) := {"tree": { + "tool-verdict": {"sbom": clean}, + "lines": { + "Cargo.lock": ["source = \"registry+one\"", "source = \"registry+two\""], + "mise-tasks/sbom-actions.tsv": rows, + ".github/workflows/ci.yml": uses, + }, + "missing": {}, +}} + +pin := " - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7" + +table_row := "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1\tMIT\tGitHub" + +test_a_pin_the_table_declares_is_clean if { + count(violation) == 0 with input as workflows([pin], [table_row]) +} + +# THE DRIFT DETECTOR: a bump moves the sha, the table's row no longer matches the +# line, and the gate fires rather than degrading the document silently. +test_a_pin_with_no_table_row_is_refused if { + some v in violation with input as workflows([pin], []) + v.rule == "sbom-action-unmapped" +} + +# A STALE SHA IS THE DRIFT, so a row naming the same repository at a different +# commit must not satisfy the pin. +test_a_row_naming_a_different_sha_does_not_map_the_pin if { + some v in violation with input as workflows( + [pin], + ["actions/checkout@0000000000000000000000000000000000000000\tMIT\tGitHub"], + ) + v.rule == "sbom-action-unmapped" +} + +# ANTI-VACUITY: a workflow line that is not a SHA-pinned `uses:` is not a pin, so +# the clause cannot be satisfied by matching every line in every workflow. +test_an_unpinned_uses_is_not_read_as_a_pin if { + count(violation) == 0 with input as workflows([" - uses: ./.github/actions/local", " name: build"], []) +} + +# COULD NOT READ THE TABLE is not "nothing is mapped". Without the guard this +# reports every pin in the tree as drift on a checkout that never read the file. +test_an_unreadable_table_reports_no_pin_as_unmapped if { + ids := {v.rule | some v in violation} with input as {"tree": { + "tool-verdict": {"sbom": clean}, + "lines": { + "Cargo.lock": ["source = \"registry+one\"", "source = \"registry+two\""], + ".github/workflows/ci.yml": [pin], + }, + "missing": {}, + }} + not "sbom-action-unmapped" in ids +} + +# COULD NOT READ THE LOCKFILE is not "zero sourced entries". Without `lock_lines` +# the comprehension yields 0 and every honest count reads as drift. +test_an_unreadable_lockfile_reports_no_drift if { + ids := {v.rule | some v in violation} with input as {"tree": { + "tool-verdict": {"sbom": clean}, + "lines": {"mise-tasks/sbom-actions.tsv": []}, + "missing": {}, + }} + not "sbom-package-drift" in ids +} + +# NOTHING HAS SCANNED THESE BYTES is not a verdict: the id is absent from the map, +# so there is nothing to refuse and a checkout whose globs never fired is clean. +test_an_unrecorded_scan_is_not_refused if { + count(violation) == 0 with input as {"tree": { + "tool-verdict": {}, + "lines": {"Cargo.lock": [], "mise-tasks/sbom-actions.tsv": []}, + "missing": {}, + }} +} + +# PRESENT AND EMPTY is the producer having written nothing, which would let every +# count above pass over an absent key. +test_a_recorded_but_empty_scan_is_refused if { + some v in violation with input as recorded({}) + v.rule == "sbom-unrecorded" +} + +# COULD-NOT-LOOK, and without the `is_object` guard this case would fault rather +# than fail, taking the whole bundle with it. +test_could_not_look_does_not_fault if { + count(violation) == 0 with input as {"tree": { + "tool-verdict": null, + "lines": {"Cargo.lock": [], "mise-tasks/sbom-actions.tsv": []}, + "missing": {}, + }} +} + +# A SIBLING ROW'S RECORD IS NOT THIS MODULE'S TO JUDGE. `input.tree["tool-verdict"]` +# is built from every `[[rule.tools]]` row in the config, so a record of another +# shape reaches this module too — measured on `validator-verdict-clean`, where +# `hk-plan`'s per-step lines were read as seven findings over a clean tree. +test_another_rows_record_is_not_read_as_a_finding if { + count(violation) == 0 with input as {"tree": { + "tool-verdict": {"hk-plan": {"batten-check": "included"}}, + "lines": {"Cargo.lock": [], "mise-tasks/sbom-actions.tsv": []}, + "missing": {}, + }} +} + +# A SOURCE THAT WOULD NOT PARSE is reported rather than skipped. +test_a_source_that_could_not_be_read_is_reported if { + some v in violation with input as {"tree": { + "tool-verdict": {}, + "lines": {}, + "missing": {"Cargo.lock": "Unparsed"}, + }} + v.verdict == "tool read broken" +} + +#MUTANT-SUITE crates/batten/tests/it/sbom_inventory.rs +#MUTANT package-drift-unread|s@^\tcount_of(format) != declared$@\tfalse@|a_drifted_cargo_count_is_refused_over_the_real_lockfile diff --git a/tests/hook-latency-drift.bats b/tests/hook-latency-drift.bats deleted file mode 100644 index 072764173..000000000 --- a/tests/hook-latency-drift.bats +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bats -# subject: mise-tasks/hook-latency-drift.sh -# CLOUD-509. The decision table for `hook-latency-drift`, the clock half of the -# two-tier gate. -# -# `hk` is stubbed throughout. The task's subject is a wall clock, so a suite that -# ran the real gate would take minutes per case and measure this container's mood -# rather than the task's arithmetic. A stub that sleeps a known amount makes the -# three verdicts reachable in seconds, and the thresholds are what this pins — -# the measurement itself is the runner's job, not the suite's. - -setup() { - TASK="$BATS_TEST_DIRNAME/../mise-tasks/hook-latency-drift.sh" - STUB="$BATS_TEST_TMPDIR/bin" - mkdir -p "$STUB" - PATH="$STUB:$PATH" - export PATH - # Three runs is the task's default and the suite's cost floor; every case - # pays it, so keep the stub's sleep small. - export BATTEN_HOOK_BUDGET_RUNS=3 -} - -# An `hk` that takes a known number of seconds and says nothing. -stub_hk() { - printf '#!/usr/bin/env bash\nsleep %s\nexit 0\n' "$1" >"$STUB/hk" - chmod +x "$STUB/hk" -} - -@test "a tier inside its budget passes" { - stub_hk 2 - BATTEN_HOOK_BUDGET_SECONDS=2 BATTEN_HOOK_BUDGET_SLACK=1 \ - BATTEN_HOOK_BUDGET_LOOSE_FACTOR=100 run "$TASK" - [ "$status" -eq 0 ] - [[ "$output" == *"within the 2s budget"* ]] -} - -@test "a tier over budget plus slack is drift-tight" { - # The direction that matters day to day: a step grew, or one joined the fast - # tier that belongs in the slow one. - stub_hk 2 - BATTEN_HOOK_BUDGET_SECONDS=0 BATTEN_HOOK_BUDGET_SLACK=0 run "$TASK" - [ "$status" -eq 1 ] - [[ "$output" == *"drift-tight"* ]] -} - -@test "slack absorbs a small overshoot rather than crying wolf" { - # A shared runner's wall clock moves run to run. A report that fires on that - # is a report nobody reads, which is worse than no report. - stub_hk 2 - BATTEN_HOOK_BUDGET_SECONDS=1 BATTEN_HOOK_BUDGET_SLACK=5 \ - BATTEN_HOOK_BUDGET_LOOSE_FACTOR=100 run "$TASK" - [ "$status" -eq 0 ] -} - -@test "a tier far under budget is drift-loose, not a silent pass" { - # The ratchet. Without this the budget rots upward: the tier gets faster, - # nobody re-derives the number, and the ceiling stops bounding anything. - stub_hk 0 - BATTEN_HOOK_BUDGET_SECONDS=30 BATTEN_HOOK_BUDGET_SLACK=5 \ - BATTEN_HOOK_BUDGET_LOOSE_FACTOR=3 run "$TASK" - [ "$status" -eq 1 ] - [[ "$output" == *"drift-loose"* ]] -} - -@test "a red gate is still timed, because cost is not correctness" { - # The task reports what the tier COSTS. A failing step still takes time, and - # treating a red gate as unmeasurable would blind the report exactly when a - # branch is broken. - printf '#!/usr/bin/env bash\nsleep 2\nexit 1\n' >"$STUB/hk" - chmod +x "$STUB/hk" - BATTEN_HOOK_BUDGET_SECONDS=2 BATTEN_HOOK_BUDGET_SLACK=1 \ - BATTEN_HOOK_BUDGET_LOOSE_FACTOR=100 run "$TASK" - [ "$status" -eq 0 ] -} - -@test "no hk on PATH is could-not-look, never a verdict" { - rm -f "$STUB/hk" - PATH="$STUB:/usr/bin:/bin" run "$TASK" - [ "$status" -eq 2 ] -} diff --git a/tests/sbom-check.bats b/tests/sbom-check.bats deleted file mode 100644 index 9040f7ba8..000000000 --- a/tests/sbom-check.bats +++ /dev/null @@ -1,340 +0,0 @@ -#!/usr/bin/env bats -# subject: mise-tasks/sbom-check.sh -# sbom-check's decision table (CLOUD-262): does the derived inventory describe the -# tree it claims to, and is it a function of the source rather than of the clock? -# -# Driven against a stubbed `syft` rather than the real one, for a reason the real -# tool cannot satisfy: two genuine scans of one tree always agree, so nothing would -# prove the normalizer is merely stripping the four volatile fields rather than -# stripping enough to make any two documents look identical. The stub can make two -# runs differ in a package NAME, which is exactly the case a too-wide normalizer -# would wave through. That is the negative self-test the acceptance asks for. -# -# The final case drops the stub and runs against the real repository, so the suite -# also asserts the committed toolchain and the real tree still satisfy the gate. - -setup() { - CHECK="$BATS_TEST_DIRNAME/../mise-tasks/sbom-check.sh" - STUB="$BATS_TEST_TMPDIR/bin" - mkdir -p "$STUB" - PATH="$STUB:$PATH" - export PATH - - # A minimal tree with the two files the gate reads: a manifest to take the - # version from, and a lockfile whose `[[package]]` count is the expectation. - ROOT="$BATS_TEST_TMPDIR/repo" - mkdir -p "$ROOT" - # `authors` as well as `version`: the workspace supplier is read from here - # (CLOUD-630), and a manifest declaring none leaves the document's own subject - # at NOASSERTION — which the supplier clause correctly refuses. - printf 'version = "9.9.9"\nauthors = ["Button Inc."]\n' >"$ROOT/Cargo.toml" - lockfile 1 - export SBOM_ROOT="$ROOT" - stub_syft - stub_cargo -} - -# `sbom.sh` reads `cargo metadata` for supplier and originator, and this gate -# re-runs it — so the synthetic tree needs an answer even though no case here -# asserts on those fields. It reports the one crate the syft stub catalogs, with an -# author, so the gate's originator-agreement clause is satisfied rather than -# bypassed. `renamed` is the drift fixture's alternate name and is declared too, or -# the drift case would fail the agreement clause instead of the stability one. -stub_cargo() { - cat >"$STUB/cargo" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail -# `fetch` is a no-op here: this synthetic tree has no crates to fetch. -[ "${1:-}" != "fetch" ] || exit 0 -[ "${1:-}" = "metadata" ] || exit 1 -cat <<'JSON' -{"packages":[ - {"name":"crate0","version":"1.0.0","source":"registry+https://github.com/rust-lang/crates.io-index","authors":["Someone"],"license":"Apache-2.0 OR MIT"}, - {"name":"renamed","version":"1.0.0","source":"registry+https://github.com/rust-lang/crates.io-index","authors":["Someone"],"license":"Apache-2.0 OR MIT"}, - {"name":"batten","version":"9.9.9","source":null,"authors":["Button Inc."],"license":"Apache-2.0"} -]} -JSON -EOF - chmod +x "$STUB/cargo" - # An unpacked registry cache for each package the stub declares. `sbom.sh` - # refuses to produce a document when a package the lockfile names has no - # unpacked source, so without this every case here would exercise that refusal - # instead of what it means to test. Empty directories, which yield `NONE` — no - # case in this suite asserts on a copyright value. - export CARGO_HOME="$BATS_TEST_TMPDIR/cargo" - mkdir -p "$CARGO_HOME/registry/src/index.crates.io-fixture" - mkdir -p "$CARGO_HOME/registry/src/index.crates.io-fixture/crate0-1.0.0" - mkdir -p "$CARGO_HOME/registry/src/index.crates.io-fixture/renamed-1.0.0" - mkdir -p "$CARGO_HOME/registry/src/index.crates.io-fixture/mystery-UNKNOWN" -} - -# A Cargo.lock declaring $1 SOURCED packages — the number the cargo purl count must -# equal (CLOUD-664). Every entry carries a `source`, because that is what makes it a -# registry dependency and so what predicts a purl in the document. A second -# argument adds one entry WITHOUT a source: a local workspace member, which syft -# 1.50.0+ deliberately gives no registry purl (anchore/syft#5105), so it must count -# toward `[[package]]` and not toward the expected purls. -lockfile() { - local n=$1 local_member="${2:-}" i - : >"$ROOT/Cargo.lock" - for ((i = 0; i < n; i++)); do - printf '[[package]]\nname = "crate%d"\nversion = "1.0.0"\nsource = "registry+https://example.invalid/index"\n\n' "$i" >>"$ROOT/Cargo.lock" - done - if [ -n "$local_member" ]; then - printf '[[package]]\nname = "batten"\nversion = "9.9.9"\n\n' >>"$ROOT/Cargo.lock" - fi -} - -# A `syft` that writes both documents, varying the four volatile fields on every -# call the way the real one does. Sentinels drive the failure shapes: -# syft.fails exit non-zero, so the gate cannot look -# syft.empty catalog nothing -# syft.drift alternate the package name per call — a REAL content change, and -# alternating rather than latching so that EVERY consecutive pair -# differs; latching from call 2 would leave a second gate run -# comparing two already-renamed documents and agreeing honestly -stub_syft() { - cat >"$STUB/syft" <"$BATS_TEST_TMPDIR/calls" - -spdx="" -cdx="" -want=0 -for arg in "\$@"; do - if [ "\$want" = 1 ]; then - case "\$arg" in - spdx-json=*) spdx="\${arg#spdx-json=}" ;; - cyclonedx-json=*) cdx="\${arg#cyclonedx-json=}" ;; - esac - want=0 - continue - fi - [ "\$arg" = "--output" ] && want=1 -done - -name=crate0 -if [ -f "$BATS_TEST_TMPDIR/syft.drift" ] && [ \$((n % 2)) -eq 0 ]; then - name=renamed -fi - -packages='{"SPDXID":"SPDXRef-Package-a","name":"'\$name'","versionInfo":"1.0.0","externalRefs":[{"referenceType":"purl","referenceLocator":"pkg:cargo/'\$name'@1.0.0"}]}' -components='{"bom-ref":"ref-a","name":"'\$name'","version":"1.0.0","purl":"pkg:cargo/'\$name'@1.0.0"}' - -# The three inflated shapes CLOUD-664 measured, each reachable on its own so a -# case can name which condition it means. They are appended as EXTRA components, -# because that is how syft produced them: a second entry for something already -# inventoried, or an entry for something that was never a dependency. -if [ -f "$BATS_TEST_TMPDIR/syft.duplicate" ]; then - packages="\$packages,"'{"SPDXID":"SPDXRef-Package-a-again","name":"'\$name'","versionInfo":"1.0.0","externalRefs":[{"referenceType":"purl","referenceLocator":"pkg:cargo/'\$name'@1.0.0"}]}' - components="\$components,"'{"bom-ref":"ref-a-again","name":"'\$name'","version":"1.0.0","purl":"pkg:cargo/'\$name'@1.0.0"}' -fi -if [ -f "$BATS_TEST_TMPDIR/syft.pathlike" ]; then - packages="\$packages,"'{"SPDXID":"SPDXRef-Package-local","name":"./action","versionInfo":"UNKNOWN","supplier":"Organization: ."}' - components="\$components,"'{"bom-ref":"ref-local","name":"./action","version":"UNKNOWN"}' -fi -if [ -f "$BATS_TEST_TMPDIR/syft.unversioned" ]; then - packages="\$packages,"'{"SPDXID":"SPDXRef-Package-nover","name":"mystery","versionInfo":"UNKNOWN"}' - components="\$components,"'{"bom-ref":"ref-nover","name":"mystery","version":"UNKNOWN"}' -fi - -# The document's own subject, and the relationship that identifies it. Present in -# every fixture because it is present in every real syft document, and because the -# gate reads it to decide what to EXEMPT: the subject shares its triple with the -# workspace member and must not be read as a duplicate of it. -subject='{"SPDXID":"SPDXRef-DocumentRoot-Directory-batten","name":"batten","versionInfo":"9.9.9"}' -relationships='{"spdxElementId":"SPDXRef-DOCUMENT","relatedSpdxElement":"SPDXRef-DocumentRoot-Directory-batten","relationshipType":"DESCRIBES"}' -if [ -f "$BATS_TEST_TMPDIR/syft.nodescribes" ]; then - relationships="" -fi - -if [ -f "$BATS_TEST_TMPDIR/syft.empty" ]; then - packages="" - components="" -fi -if [ -n "\$packages" ]; then - packages="\$subject,\$packages" -else - packages="\$subject" -fi - -mkdir -p "\$(dirname "\$spdx")" "\$(dirname "\$cdx")" -cat >"\$spdx" <"\$cdx" <"$BATS_TEST_TMPDIR/syft.drift" - run "$CHECK" - [ "$status" -eq 1 ] - [[ "$output" == *"sbom-unstable"* ]] -} - -@test "a cargo count that disagrees with Cargo.lock fails, naming both numbers" { - lockfile 3 - run "$CHECK" - [ "$status" -eq 1 ] - [[ "$output" == *"sbom-package-drift"* ]] - [[ "$output" == *"1 vs 3"* ]] -} - -@test "an SBOM that catalogs nothing must not report green" { - # Two empty documents agree perfectly, so every equality check below would - # pass. This is the vacuous green the gate has to refuse on its own. - : >"$BATS_TEST_TMPDIR/syft.empty" - lockfile 0 - run "$CHECK" - [ "$status" -eq 1 ] - [[ "$output" == *"sbom-empty"* ]] -} - -@test "output is pointer-only — no document body reaches the log" { - # rule 4. An SBOM is 300+ KB of dependency detail; the remedy is one command, - # so the body adds nothing and would bury the verdict. - : >"$BATS_TEST_TMPDIR/syft.drift" - run "$CHECK" - [ "$status" -eq 1 ] - [[ "$output" != *"referenceLocator"* ]] - [[ "$output" != *"pkg:cargo/"* ]] -} - -@test "the failure names an asset, not a scratch path" { - # The gate scans into mktemp dirs, and a pointer at one of those is noise a - # reader cannot act on. It points at the published asset name instead. - : >"$BATS_TEST_TMPDIR/syft.drift" - run "$CHECK" - [[ "$output" == *"batten.spdx.json:0"* ]] - [[ "$output" != *"$BATS_TEST_TMPDIR"* ]] -} - -@test "the gate leaves the tree it judges unmodified, and fails twice" { - # A gate that writes what it judges cannot fail twice: the second run would - # pass, laundering the drift into a clean result. - : >"$BATS_TEST_TMPDIR/syft.drift" - before="$(find "$ROOT" -type f | sort)" - run "$CHECK" - [ "$status" -eq 1 ] - [ "$(find "$ROOT" -type f | sort)" = "$before" ] - run "$CHECK" - [ "$status" -eq 1 ] -} - -@test "a syft that cannot run exits 2 — could not look is not a verdict" { - : >"$BATS_TEST_TMPDIR/syft.fails" - run "$CHECK" - [ "$status" -eq 2 ] - [[ "$output" == *"unverified"* ]] -} - -@test "a missing Cargo.lock exits 2 rather than passing vacuously" { - rm -f "$ROOT/Cargo.lock" - run "$CHECK" - [ "$status" -eq 2 ] - [[ "$output" == *"must not report green"* ]] -} - -@test "a lockfile whose local member has no source still matches: 1 purl, 1 sourced of 2" { - # CLOUD-664. syft 1.50.0 stopped emitting a registry purl for a local workspace - # package (anchore/syft#5105), so the count this clause compares against is the - # lockfile's SOURCED entries, not all of them. Comparing against all of them is - # the off-by-one that made #572's CI red. - lockfile 1 local - run "$CHECK" - [ "$status" -eq 0 ] - [[ "$output" == *"1 sourced entries of 2"* ]] -} - -@test "the inflated shapes syft produces are all absorbed before the gate judges" { - # The three shapes CLOUD-664 measured, driven through the whole path at once: - # a second entry for something already inventoried, a relative-path component - # that was never a dependency, and an entry with no usable version. The gate - # passes because `sbom.sh` normalises them — which is the integration this - # suite can assert and `tests/sbom.bats` asserts component by component. - # - # This case cannot show the clause FIRING, and that is a property of the design - # rather than a gap: the clause and the normaliser share one identity rule, so - # after a successful normalisation there is nothing left to find. The firing - # proof is the `#MUTANT` row on the normalise call in `sbom.sh`. - : >"$BATS_TEST_TMPDIR/syft.duplicate" - : >"$BATS_TEST_TMPDIR/syft.pathlike" - : >"$BATS_TEST_TMPDIR/syft.unversioned" - run "$CHECK" - [ "$status" -eq 0 ] - [[ "$output" == *"each a distinct thing"* ]] -} - -@test "a document that DESCRIBES nothing is could-not-look, not a clean inventory" { - # The subject is what the identity clause exempts, so without it every count is - # measured over the wrong set. Reporting green there would be a verdict reached - # by not looking — exit 2, the same answer this gate gives for a missing - # Cargo.lock. - : >"$BATS_TEST_TMPDIR/syft.nodescribes" - run "$CHECK" - [ "$status" -eq 2 ] - [[ "$output" == *"no DESCRIBES"* ]] -} - -@test "THE DRIFT DETECTOR: a pin with no table row fails, which is how a bump arrives" { - # The reason a committed table is defensible at all. A pinned action's license - # is immutable, so recording it is a property of this commit — but only while - # the table still describes the pins the workflows carry. This fires on the one - # event that breaks that, and it is the direction it will actually be hit: a - # renovate bump moves a sha, and the row that named the old one no longer - # matches. - mkdir -p "$ROOT/.github/workflows" - printf 'jobs:\n a:\n steps:\n - uses: some/action@%040d\n' 1 >"$ROOT/.github/workflows/w.yml" - printf 'some/action@%040d\tMIT\tCopyright (c) 2020 Someone\n' 2 >"$ROOT/actions.tsv" - SBOM_ACTIONS_TABLE="$ROOT/actions.tsv" run "$CHECK" - [ "$status" -eq 1 ] - [[ "$output" == *"sbom-action-unmapped"* ]] - # Pointer-only: the workflow file and line, never a license or a holder. - [[ "$output" == *".github/workflows/w.yml"* ]] - [[ "$output" != *"Copyright (c) 2020"* ]] -} - -@test "this repo's real tree satisfies the gate — with the real syft" { - # The self-consumption case. The stub proves the logic; this proves the logic - # is pointed at a tree and a toolchain that actually satisfy it, which is the - # only way the suite can also assert the committed pin works. - unset SBOM_ROOT - # And the real registry cache: `setup` points CARGO_HOME at a fixture holding - # only the stub's crates, which for the real tree would be an absent-source - # refusal rather than a verdict about the document. - unset CARGO_HOME - PATH="${PATH#"$STUB":}" - export PATH - cd "$BATS_TEST_DIRNAME/.." || return 1 - run "$CHECK" - [ "$status" -eq 0 ] - [[ "$output" == *"matching Cargo.lock"* ]] -}