From 51c55bb873254c3848221dc22f434e2eae1a39d3 Mon Sep 17 00:00:00 2001 From: Maxwell Elliott Date: Fri, 11 Sep 2026 14:09:41 -0400 Subject: [PATCH 1/2] ci: give the performance gate more time and retries CI runners were hitting the 45-minute job cap on a single hung measurement. Bound each attempt at 50 minutes, retry up to three times, and keep the job budget large enough for setup plus those attempts. Co-authored-by: Cursor --- .github/workflows/perf-gate.yaml | 36 ++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/workflows/perf-gate.yaml b/.github/workflows/perf-gate.yaml index 9f246a9..a908987 100644 --- a/.github/workflows/perf-gate.yaml +++ b/.github/workflows/perf-gate.yaml @@ -34,7 +34,8 @@ jobs: name: ${{ matrix.title }} runs-on: ${{ matrix.runner }} continue-on-error: ${{ matrix.continue_on_error }} - timeout-minutes: 45 + # Allows setup/build plus three bounded performance-gate attempts. + timeout-minutes: 180 strategy: fail-fast: false matrix: @@ -95,14 +96,31 @@ jobs: # workspace instead of the runfiles tree, and so Bazel is not holding the output-base # lock (or competing for cores) while timings are taken. run: | - python3 tools/perf_gate.py \ - --kotlin-binary bazel-bin/cli/bazel-diff \ - --rust-binary ${{ matrix.rust_binary }} \ - ${{ matrix.gate_workloads }} \ - --rounds 7 \ - --warmup-rounds 2 \ - --json perf-gate-${{ matrix.libc }}.json \ - | tee perf-gate-${{ matrix.libc }}.txt + for attempt in 1 2 3; do + echo "Performance gate attempt $attempt of 3" + rm -f perf-gate-${{ matrix.libc }}.json perf-gate-${{ matrix.libc }}.txt + set +e + timeout --signal=TERM --kill-after=30s 50m \ + python3 tools/perf_gate.py \ + --kotlin-binary bazel-bin/cli/bazel-diff \ + --rust-binary ${{ matrix.rust_binary }} \ + ${{ matrix.gate_workloads }} \ + --rounds 7 \ + --warmup-rounds 2 \ + --json perf-gate-${{ matrix.libc }}.json \ + 2>&1 | tee perf-gate-${{ matrix.libc }}.txt + status=${PIPESTATUS[0]} + set -e + + if [[ "$status" -eq 0 ]]; then + exit 0 + fi + if [[ "$attempt" -eq 3 ]]; then + exit "$status" + fi + echo "Performance gate failed with status $status; retrying in 15s..." + sleep 15 + done - name: Publish summary if: always() run: | From e3d8a5e78808fcb3ca213aefd699c9fc679c34c0 Mon Sep 17 00:00:00 2001 From: Maxwell Elliott Date: Fri, 11 Sep 2026 16:01:37 -0400 Subject: [PATCH 2/2] ci: run the performance gate nightly instead of on every PR The measurement is too long and runner-sensitive to block pull requests. Move it to a 07:00 UTC cron with a one-hour cap per attempt, keeping retries and on-demand dispatch. Co-authored-by: Cursor --- .github/workflows/perf-gate.yaml | 25 ++++++++++++------------- README.md | 2 +- docs/kotlin-rust-perf-gate.md | 8 ++++---- tools/readme_template.md | 2 +- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/.github/workflows/perf-gate.yaml b/.github/workflows/perf-gate.yaml index a908987..0f7f547 100644 --- a/.github/workflows/perf-gate.yaml +++ b/.github/workflows/perf-gate.yaml @@ -8,14 +8,14 @@ name: Performance Gate # built `-c opt`, because comparing a debug build against an optimized one measures the build # flags rather than the code. # -# Runs on every pull request: "Rust is faster" is a property of the code, and a regression is -# cheapest to catch in the change that caused it. The gate pairs its measurements (both binaries -# every round, alternating which goes first) and takes medians over several rounds rather than -# trusting a single timing. +# Nightly cron (plus on-demand from the Actions tab): the gate is too long and +# runner-sensitive to block pull requests. A regression still shows up within a day. +# The gate pairs its measurements (both binaries every round, alternating which goes +# first) and takes medians over several rounds rather than trusting a single timing. # # Two legs (see the matrix below): -# * glibc -- the host-native dynamic build. Stable on any runner, so it is the blocking gate -# that proves the Rust logic is faster than Kotlin. +# * glibc -- the host-native dynamic build. Stable on any runner, so it is the +# check that proves the Rust logic is faster than Kotlin. # * musl -- the statically-linked binary actually published for Linux. musl's single-arena # malloc would serialize the parallel proto decode, so this leg guards allocator scalability: # the release links mimalloc as its #[global_allocator], and this leg keeps it that way. It @@ -23,10 +23,9 @@ name: Performance Gate # runs on a larger runner over the allocator-sensitive workload. # See docs/kotlin-rust-perf-gate.md for the protocol and for what to do when this job fails. on: - push: - branches: [ master ] - pull_request: - branches: [ master ] + schedule: + # Daily, offset from the serve-stress crons (06:30 / 08:30 UTC). + - cron: "0 7 * * *" workflow_dispatch: jobs: @@ -34,8 +33,8 @@ jobs: name: ${{ matrix.title }} runs-on: ${{ matrix.runner }} continue-on-error: ${{ matrix.continue_on_error }} - # Allows setup/build plus three bounded performance-gate attempts. - timeout-minutes: 180 + # Setup/build plus three one-hour measurement attempts. + timeout-minutes: 210 strategy: fail-fast: false matrix: @@ -100,7 +99,7 @@ jobs: echo "Performance gate attempt $attempt of 3" rm -f perf-gate-${{ matrix.libc }}.json perf-gate-${{ matrix.libc }}.txt set +e - timeout --signal=TERM --kill-after=30s 50m \ + timeout --signal=TERM --kill-after=30s 60m \ python3 tools/perf_gate.py \ --kotlin-binary bazel-bin/cli/bazel-diff \ --rust-binary ${{ matrix.rust_binary }} \ diff --git a/README.md b/README.md index c55176c..996c0f6 100644 --- a/README.md +++ b/README.md @@ -1125,7 +1125,7 @@ Apple Silicon Mac. ### Performance gate -The Rust candidate is expected to be faster than Kotlin, and CI enforces it. `make perf-gate` +The Rust candidate is expected to be faster than Kotlin, and a nightly CI cron enforces it. `make perf-gate` runs both binaries over generated workloads -- a synthetic `streamed_proto` graph plus hash-file pairs, with no real workspace, Bazel server or Hyperfine involved -- and exits non-zero unless Rust wins on median wall time, on start-up-adjusted wall time, and in every paired round. diff --git a/docs/kotlin-rust-perf-gate.md b/docs/kotlin-rust-perf-gate.md index eed1ed9..00151c7 100644 --- a/docs/kotlin-rust-perf-gate.md +++ b/docs/kotlin-rust-perf-gate.md @@ -10,7 +10,7 @@ It is not the same tool as [`docs/kotlin-vs-rust-benchmark.md`](kotlin-vs-rust-b | --- | --- | --- | | Question | how much faster is Rust on *this* repository? | is Rust still faster than Kotlin at all? | | Inputs | a real checkout, Bazel, Hyperfine | none -- fixtures are generated | -| Runs in CI | no (manual, needs a workspace) | yes, on every pull request | +| Runs in CI | no (manual, needs a workspace) | nightly cron (and `workflow_dispatch`) | | Output | a report | a report **and an exit code** | ## Running it @@ -122,10 +122,10 @@ at the static-musl release on a many-core machine, or approximate it with a glib `MALLOC_ARENA_MAX=1`. The smaller graphs do not reach a high enough concurrent allocation rate to surface it, which is why the dense graph exists as a separate load. -The default CI run builds the host **glibc** binary, which is stable on any runner and never +The nightly CI run builds the host **glibc** binary, which is stable on any runner and never storms -- so it would not catch an allocator regression by itself. The gate therefore runs as -two legs (see `.github/workflows/perf-gate.yaml`): the glibc leg is the blocking "Rust is -faster" gate, and a second leg builds the published `--config=release-musl` binary and runs +two legs (see `.github/workflows/perf-gate.yaml`): the glibc leg is the "Rust is faster" +check, and a second leg builds the published `--config=release-musl` binary and runs the dense load against it. That leg needs a **many-core runner** -- the storm does not appear below ~8 cores, so a 2-vCPU runner would pass it regardless of the allocator. Locally, `make perf-gate-musl` reproduces it on a many-core host. diff --git a/tools/readme_template.md b/tools/readme_template.md index 603ad05..081ac5b 100644 --- a/tools/readme_template.md +++ b/tools/readme_template.md @@ -599,7 +599,7 @@ Apple Silicon Mac. ### Performance gate -The Rust candidate is expected to be faster than Kotlin, and CI enforces it. `make perf-gate` +The Rust candidate is expected to be faster than Kotlin, and a nightly CI cron enforces it. `make perf-gate` runs both binaries over generated workloads -- a synthetic `streamed_proto` graph plus hash-file pairs, with no real workspace, Bazel server or Hyperfine involved -- and exits non-zero unless Rust wins on median wall time, on start-up-adjusted wall time, and in every paired round.