From f82bb2637b36b297a61a4867e457ff6fc30ed4dd Mon Sep 17 00:00:00 2001 From: Jordan Gonzalez <30836115+duncanista@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:49:04 -0400 Subject: [PATCH 1/2] [SVLS-9977] ci: informational benchmark workflow --- .github/workflows/bench.yml | 163 ++++++++++++++++++++++++++++++++++++ bottlecap/benches/README.md | 34 ++++++++ scripts/ci/bench-measure.sh | 82 ++++++++++++++++++ 3 files changed, 279 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100755 scripts/ci/bench-measure.sh diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 000000000..d849ca455 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,163 @@ +# Criterion benchmarks (bottlecap/benches). Informational: no job here blocks +# a PR. See bottlecap/benches/README.md, section "CI". +# +# - bench-build: every PR, `cargo bench --no-run` with LTO off (compile only). +# - bench-measure: PRs with the `bench` label, nightly on main, or manual. +# Production profile. Nightly/full runs save the baseline +# artifact `bench-baseline-main`; a PR run compares against it +# with critcmp and writes the table to the step summary. +name: Benchmarks + +on: + pull_request: + # `labeled` so the `bench` label starts bench-measure. + types: [opened, synchronize, reopened, labeled] + paths: + - "bottlecap/**" + - "scripts/ci/bench-measure.sh" + - ".github/workflows/bench.yml" + schedule: + # Nightly on main, after the vulnerability scan. + - cron: "17 3 * * *" + workflow_dispatch: + inputs: + mode: + description: "smoke: short list, compare with the main baseline. full: whole suite, save the baseline (main only)." + type: choice + default: smoke + options: [smoke, full] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + PB_VERSION: 25.3 + PB_URL: https://github.com/protocolbuffers/protobuf/releases/download + PB_TARGET: linux-x86_64 + CARGO_INCREMENTAL: "0" + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + +jobs: + bench-build: + name: Bench Build (informational) + # Not on `labeled`: a label must not rebuild; it starts bench-measure. + if: github.event_name == 'pull_request' && github.event.action != 'labeled' + runs-on: ubuntu-22.04 + timeout-minutes: 45 + env: + # Compile check only: skip the fat-LTO link, which is the slow part. + CARGO_PROFILE_BENCH_LTO: "false" + CARGO_PROFILE_BENCH_CODEGEN_UNITS: "16" + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + # Nothing to compile before the criterion suite lands; a bench-profile + # build of the lib and bin harnesses would only fill the sccache cache. + - id: probe + working-directory: bottlecap + run: | + if cargo metadata --no-deps --format-version 1 --locked \ + | jq -e '.packages[].targets[] | select(.kind | index("bench"))' > /dev/null; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "present=false" >> "$GITHUB_OUTPUT" + echo "::notice title=No bench targets::no [[bench]] target in bottlecap; nothing to build." + fi + + # Same protoc install as rs_ci.yml. + - if: steps.probe.outputs.present == 'true' + run: sudo apt-get update + - if: steps.probe.outputs.present == 'true' + run: sudo apt-get install -y curl unzip cmake + - if: steps.probe.outputs.present == 'true' + name: Install protobuf ${{ env.PB_VERSION }} compiler from binary for ${{ env.PB_TARGET }} + run: | + curl -LO "${{ env.PB_URL }}/v${{ env.PB_VERSION }}/protoc-${{ env.PB_VERSION }}-${{ env.PB_TARGET }}.zip" + unzip "protoc-${{ env.PB_VERSION }}-${{ env.PB_TARGET }}.zip" -d "$HOME/.local" + export PATH="$PATH:$HOME/.local/bin" + + - if: steps.probe.outputs.present == 'true' + uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - id: build + if: steps.probe.outputs.present == 'true' + continue-on-error: true + working-directory: bottlecap + run: cargo bench --no-run --locked + - if: steps.build.outcome == 'failure' + run: | + echo "::warning title=Bench build failed::cargo bench --no-run failed; see the job log. Informational, does not block." + echo "### Bench build failed (informational)" >> "$GITHUB_STEP_SUMMARY" + echo "See the \`cargo bench --no-run\` step of this job." >> "$GITHUB_STEP_SUMMARY" + + bench-measure: + name: Bench Measure (informational) + if: >- + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'bench')) + runs-on: ubuntu-22.04 + timeout-minutes: 120 + permissions: + contents: read + # `gh run download` of the baseline artifact. + actions: read + env: + BENCH_MODE: ${{ (github.event_name == 'schedule' || inputs.mode == 'full') && 'full' || 'smoke' }} + BENCH_BASELINE_DIR: ${{ github.workspace }}/.bench-baseline + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - run: sudo apt-get update + - run: sudo apt-get install -y curl unzip cmake + - name: Install protobuf ${{ env.PB_VERSION }} compiler from binary for ${{ env.PB_TARGET }} + run: | + curl -LO "${{ env.PB_URL }}/v${{ env.PB_VERSION }}/protoc-${{ env.PB_VERSION }}-${{ env.PB_TARGET }}.zip" + unzip "protoc-${{ env.PB_VERSION }}-${{ env.PB_TARGET }}.zip" -d "$HOME/.local" + export PATH="$PATH:$HOME/.local/bin" + + - uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4 + with: + cache: false + - uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - name: Install critcmp + run: cargo install critcmp --locked --version 0.1.8 + + - name: Download the main baseline + if: env.BENCH_MODE == 'smoke' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: scripts/ci/bench-measure.sh download-baseline "$BENCH_BASELINE_DIR" + + - id: measure + continue-on-error: true + working-directory: bottlecap + run: ../scripts/ci/bench-measure.sh run + - if: steps.measure.outcome == 'failure' + run: | + echo "::warning title=Bench measure failed::a bench panicked or did not build; see the job log. Informational, does not block." + echo "### Bench measure failed (informational)" >> "$GITHUB_STEP_SUMMARY" + + - name: Save the main baseline + if: env.BENCH_MODE == 'full' && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: bench-baseline-main + path: bottlecap/.bench-export/main.json + retention-days: 90 + if-no-files-found: warn + - name: Upload the exported numbers + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: bench-export-${{ github.run_id }} + path: bottlecap/.bench-export/ + retention-days: 14 + if-no-files-found: ignore diff --git a/bottlecap/benches/README.md b/bottlecap/benches/README.md index b3f759b20..47b800bec 100644 --- a/bottlecap/benches/README.md +++ b/bottlecap/benches/README.md @@ -130,6 +130,40 @@ faster than production. The ratios between benches, and between main and a fix, The lambda-sim environment (`scripts/lambda-sim/`) can reuse this image and the fixture corpus to measure the whole extension under a CPU quota. +## CI + +`.github/workflows/bench.yml` runs a non-blocking part of this plan: `bench-build` compiles the +suite on every PR that touches `bottlecap/`, and `bench-measure` (`scripts/ci/bench-measure.sh`) +runs the smoke list of item 2 on PRs with the `bench` label, nightly on `main`, and on demand, +then compares it with the last `main` baseline. The list below is the full plan. + +Goal: catch a broken bench and a large regression on every PR without a flaky gate. + +1. **On every PR, blocking:** `cargo bench --no-run` with + `CARGO_PROFILE_BENCH_LTO=false CARGO_PROFILE_BENCH_CODEGEN_UNITS=16` (compile only, a few + minutes with the cache), and `cargo test --test fixtures_corpus` (the generators). This + keeps the suite compiling as the code changes. +2. **On every PR, non-blocking job `bench-smoke`:** run a fixed short list with the production + profile and `--quick` (about 2 minutes of measurement): `logs/processor_process`, + `logs/oom_check`, `traces/process_traces/default`, `traces/decode_v04`, + `invocation/trigger/from_value/sqs_event`, `enhanced_metrics/set_max`, + `config/get_config/typical`. Save the baseline `pr`, download the baseline artifact from the + last `main` run, and post `critcmp main pr -t 10` as a PR comment (or a job summary). The job + never fails on a number; it fails only if a bench panics. A shared GitHub runner is noisy, + so 10 % is the reporting threshold, and the comment says "informational". +3. **On `main`, nightly or per merge:** the full suite with the production profile, baseline + saved as an artifact (`critcmp --export main > main.json`) and kept 90 days. A second + nightly job runs `scripts/bench-docker-arm64.sh` on an arm64 runner for the Linux-only + `/proc` benches. Track the exported JSON over time (a small script or a Datadog metric per + bench id) and alert on a 20 % move that holds for 3 nights, which filters runner noise. +4. **Cost note:** `rs_ci.yml` runs `cargo clippy --all-targets` for `default`, `fips` and + `default,test-mode`, so the six bench targets compile three times per PR. The `fips` clippy + does not run on a Mac (libddwaf-sys aborts at load); check it on Linux before merging. +5. **Hardware note:** GitHub-hosted runners vary between runs. If the numbers matter more + than the trend, use a self-hosted or dedicated runner and pin the CPU set. Until then, the + nightly trend and the PR comment are hints; the developer confirms a regression locally + with `scripts/bench-compare.sh`. + ## Backlog to bench Review = `consolidated-findings-v3` (Confluence page 7384761389). Numbers are the first diff --git a/scripts/ci/bench-measure.sh b/scripts/ci/bench-measure.sh new file mode 100755 index 000000000..d0cc7659a --- /dev/null +++ b/scripts/ci/bench-measure.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Helper for .github/workflows/bench.yml (job bench-measure). +# +# bench-measure.sh download-baseline # latest nightly `bench-baseline-main` artifact (needs gh) +# bench-measure.sh run # from bottlecap/; BENCH_MODE=smoke|full +# +# smoke: a short fixed list with `--quick`, exported as pr.json, compared with +# /main.json when present (critcmp, 10 % threshold). +# full: the whole suite, baseline "main", exported as main.json. +# Exports land in .bench-export/. With no [[bench]] target in the checkout the +# run is a no-op, so the job is green before the suite lands on main. +set -euo pipefail + +export_dir=".bench-export" +# bottlecap/benches/README.md, "CI" item 2: one id per hot path. +smoke_list="logs:processor_process logs:oom_check traces:process_traces/default traces:decode_v04 invocation:trigger/from_value/sqs_event enhanced_metrics:set_max config:get_config/typical" + +summary() { + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then tee -a "$GITHUB_STEP_SUMMARY"; else cat; fi +} + +case "${1:-}" in +download-baseline) + dir="${2:?usage: bench-measure.sh download-baseline }" + mkdir -p "$dir" + run_id="$(gh run list --workflow bench.yml --branch main --event schedule --status success --limit 1 \ + --json databaseId --jq '.[0].databaseId // empty' || true)" + if [ -z "$run_id" ]; then + echo "no successful nightly bench run on main yet; no baseline" + exit 0 + fi + if gh run download "$run_id" --name bench-baseline-main --dir "$dir"; then + echo "baseline: run $run_id -> $dir/main.json" + else + echo "run $run_id has no bench-baseline-main artifact; no baseline" + fi + ;; + +run) + mode="${BENCH_MODE:-smoke}" + baseline_dir="${BENCH_BASELINE_DIR:-}" + targets="$(cargo metadata --no-deps --format-version 1 --locked \ + | jq -r '.packages[].targets[] | select(.kind | index("bench")) | .name')" + if [ -z "$targets" ]; then + echo "no [[bench]] targets in this checkout; nothing to measure" + exit 0 + fi + mkdir -p "$export_dir" + if [ "$mode" = "full" ]; then + cargo bench --locked -- --save-baseline main + critcmp --export main > "$export_dir/main.json" + { echo "### Bench full run (baseline main)"; echo '```'; critcmp main; echo '```'; } | summary + exit 0 + fi + for pair in $smoke_list; do + bench="${pair%%:*}" + filter="${pair#*:}" + if ! grep -qx "$bench" <<<"$targets"; then + echo "skip $bench/$filter: no such bench target" + continue + fi + cargo bench --locked --bench "$bench" -- --save-baseline pr --quick "$filter" + done + critcmp --export pr > "$export_dir/pr.json" + if [ -n "$baseline_dir" ] && [ -f "$baseline_dir/main.json" ]; then + cp "$baseline_dir/main.json" "$export_dir/main.json" + { + echo "### Bench smoke vs main (informational: shared runner, differences under 10 % hidden)" + echo '```' + critcmp "$export_dir/main.json" "$export_dir/pr.json" -t 10 + echo '```' + } | summary + else + { echo "### Bench smoke (no main baseline yet)"; echo '```'; critcmp "$export_dir/pr.json"; echo '```'; } | summary + fi + ;; + +*) + sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//' + exit 2 + ;; +esac From c70178f54795d9da1e4b405224d55aeebe8a4d11 Mon Sep 17 00:00:00 2001 From: Jordan Gonzalez <30836115+duncanista@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:19:20 -0400 Subject: [PATCH 2/2] [SVLS-9977] ci: address review feedback on the bench workflow - Upload the exports with include-hidden-files, so the .bench-export directory reaches the artifact. - download-baseline: walk back through the last 10 green nightlies to the newest one that has a bench-baseline-main artifact. - bench-measure: on a labeled event, run only for the bench label, and give other label events their own concurrency group so they do not cancel a running measurement. --- .github/workflows/bench.yml | 8 ++++++-- scripts/ci/bench-measure.sh | 23 ++++++++++++----------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index d849ca455..0dde52470 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -28,7 +28,8 @@ on: options: [smoke, full] concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + # An unrelated label gets its own group, so it does not cancel a running measurement. + group: ${{ github.workflow }}-${{ github.ref }}${{ github.event.action == 'labeled' && github.event.label.name != 'bench' && format('-{0}', github.run_id) || '' }} cancel-in-progress: true permissions: @@ -101,7 +102,8 @@ jobs: if: >- github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || - (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'bench')) + (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'bench') && + (github.event.action != 'labeled' || github.event.label.name == 'bench')) runs-on: ubuntu-22.04 timeout-minutes: 120 permissions: @@ -151,6 +153,7 @@ jobs: with: name: bench-baseline-main path: bottlecap/.bench-export/main.json + include-hidden-files: true retention-days: 90 if-no-files-found: warn - name: Upload the exported numbers @@ -159,5 +162,6 @@ jobs: with: name: bench-export-${{ github.run_id }} path: bottlecap/.bench-export/ + include-hidden-files: true retention-days: 14 if-no-files-found: ignore diff --git a/scripts/ci/bench-measure.sh b/scripts/ci/bench-measure.sh index d0cc7659a..b60f140cd 100755 --- a/scripts/ci/bench-measure.sh +++ b/scripts/ci/bench-measure.sh @@ -23,17 +23,18 @@ case "${1:-}" in download-baseline) dir="${2:?usage: bench-measure.sh download-baseline }" mkdir -p "$dir" - run_id="$(gh run list --workflow bench.yml --branch main --event schedule --status success --limit 1 \ - --json databaseId --jq '.[0].databaseId // empty' || true)" - if [ -z "$run_id" ]; then - echo "no successful nightly bench run on main yet; no baseline" - exit 0 - fi - if gh run download "$run_id" --name bench-baseline-main --dir "$dir"; then - echo "baseline: run $run_id -> $dir/main.json" - else - echo "run $run_id has no bench-baseline-main artifact; no baseline" - fi + # A nightly whose measure step failed is still green but has no artifact, so walk back to + # the newest run that has one. + run_ids="$(gh run list --workflow bench.yml --branch main --event schedule --status success --limit 10 \ + --json databaseId --jq '.[].databaseId' || true)" + for run_id in $run_ids; do + if gh run download "$run_id" --name bench-baseline-main --dir "$dir"; then + echo "baseline: run $run_id -> $dir/main.json" + exit 0 + fi + echo "run $run_id has no bench-baseline-main artifact; trying an older run" + done + echo "no bench-baseline-main artifact in the last 10 successful nightly runs on main; no baseline" ;; run)