From 3a258976b62f50ed88b67b01f28d3d82586585a6 Mon Sep 17 00:00:00 2001 From: PurHur Date: Fri, 18 Sep 2026 17:19:38 +0000 Subject: [PATCH] Bench: scale binary-trees/fannkuch so v2 VM finishes under 15 min wall (#36385) Depth 10 / n=8 timed out under the 25s VM cap and burned the full timeout with n/a columns. Scale to maxDepth=7 / n=6, honor @bench-skip-vm, and project wall from RESULTS in check-bench-readme-sync so the Done-when budget stays enforceable. Co-authored-by: Cursor --- benchmarks/v2/BASELINE.json | 18 ++-- benchmarks/v2/README.md | 13 +-- benchmarks/v2/RESULTS.json | 24 +++--- benchmarks/v2/binary-trees.php | 5 +- benchmarks/v2/fannkuch-redux.php | 5 +- docs/pages/bench.html | 2 +- script/bench.php | 66 +++++++++++---- script/check-bench-readme-sync.php | 130 ++++++++++++++++++++++++++++- test/unit/BenchGateTest.php | 26 ++++++ 9 files changed, 244 insertions(+), 45 deletions(-) diff --git a/benchmarks/v2/BASELINE.json b/benchmarks/v2/BASELINE.json index 11911579117..92a6411c364 100644 --- a/benchmarks/v2/BASELINE.json +++ b/benchmarks/v2/BASELINE.json @@ -1,14 +1,14 @@ { "version": 1, "suite": "v2", - "generated_at": "2026-09-09T16:31:20Z", + "generated_at": "2026-09-18T17:00:45Z", "arch": "x86_64-linux", "ratio_tolerance_percent": 20, "geomean_tolerance_percent": 10, - "geomean_ratio_aot_over_zend": 0.869234, + "geomean_ratio_aot_over_zend": 0.711107, "ir_tolerance_percent": 20, "regeneration": "PHP_8_2=$(command -v php) ./script/bench-gate.sh --v2 --update", - "note": "v2 gate subset (#36385). Per-case ratio +20%; suite geomean +10% so diffuse regressions fail.", + "note": "v2 gate subset (#36385). Per-case ratio +20%; suite geomean +10%. binary-trees maxDepth=7 / fannkuch n=6 so VM finishes under 25s wall cap.", "cases": { "call-heavy": { "ratio_aot_over_zend": 0.461441, @@ -109,16 +109,16 @@ "ir_defines": 555 }, "binary-trees": { - "ratio_aot_over_zend": 1.684036, - "zend_wall": 0.027164, - "aot_wall": 0.045745, + "ratio_aot_over_zend": 0.536744, + "zend_wall": 0.015482, + "aot_wall": 0.00831, "ir_lines": 280636, "ir_defines": 546 }, "fannkuch-redux": { - "ratio_aot_over_zend": 2.502303, - "zend_wall": 0.034162, - "aot_wall": 0.085484, + "ratio_aot_over_zend": 0.31601, + "zend_wall": 0.015905, + "aot_wall": 0.005026, "ir_lines": 199262, "ir_defines": 565 } diff --git a/benchmarks/v2/README.md b/benchmarks/v2/README.md index 23ab9e0a0f7..1b09eab5509 100644 --- a/benchmarks/v2/README.md +++ b/benchmarks/v2/README.md @@ -21,8 +21,11 @@ That also refreshes `docs/pages/bench.html` (ratio sparklines + web column). Set **Wall budget (#36385):** default `bench.php --v2` must finish in **< 15 min**. The harness uses a 25 s per-run VM/JIT cap (override `PHP_COMPILER_BENCH_TIMEOUT`), 3 averaged -iterations, skips JIT after a VM time-cap, and reuses a verify sample when it already -took ≥ 5 s — so `binary-trees` / `fannkuch-redux` VM timeouts no longer burn ~10 min each. +iterations, skips JIT after a VM time-cap, reuses a verify sample when it already +took ≥ 5 s, and honors `// @bench-skip-vm: <reason>` so a known time-cap case cannot +burn the full cap. `binary-trees` uses `maxDepth = 7` and `fannkuch-redux` uses `n = 6` +so VM/JIT columns stay measurable under that cap (depth 10 / n=8 timed out). Fast gate: +`php script/check-bench-readme-sync.php --check-wall` projects wall from committed `RESULTS.json`. Gate (AOT/Zend ratio + IR size vs committed baseline; subset that must match Zend under AOT): @@ -46,16 +49,16 @@ The chart page `docs/pages/bench.html` is generated by `script/generate-bench-ch -Environment: 8.2.32 · LLVM 9 available · 3 iterations averaged, wall time per run. +Environment: 8.2.33 · LLVM 9 available · 3 iterations averaged, wall time per run. | Test Name | Zend 8.2 (s)| bin/vm.php (s) | bin/jit.php (s) | phpc build (s) | native run (s) | |--------------------|-------------------|----------------|-----------------|----------------|----------------| | assoc-heavy | 0.0125 | 2.4465 | 2.5408 | 0.1881 | 0.0088 | -| binary-trees | 0.0238 | n/a | n/a | 0.1586 | 0.0437 | +| binary-trees | 0.0220 | 10.4585 | 10.1683 | 0.2830 | 0.0095 | | call-heavy | 0.0137 | 15.3373 | 15.7211 | 0.1701 | 0.0048 | | closure-heavy | 0.0167 | 2.6189 | 2.6044 | 2.2945 | 0.0070 | | exceptions | 0.0138 | 4.1803 | 4.3185 | 0.8953 | 0.0067 | -| fannkuch-redux | 0.0371 | n/a | n/a | 0.1758 | 0.0880 | +| fannkuch-redux | 0.0176 | 7.6686 | 7.3140 | 0.2859 | 0.0064 | | fasta | 0.0140 | 1.4628 | 1.4897 | 0.4918 | 0.0032 | | json-roundtrip | 0.0103 | 0.8345 | 0.8959 | 0.1475 | 0.0376 | | k-nucleotide | 0.0165 | 6.4666 | 6.7544 | 0.2238 | 0.0052 | diff --git a/benchmarks/v2/RESULTS.json b/benchmarks/v2/RESULTS.json index 142b14fa6fe..be7a48c6cc8 100644 --- a/benchmarks/v2/RESULTS.json +++ b/benchmarks/v2/RESULTS.json @@ -1,8 +1,8 @@ { "version": 1, "suite": "v2", - "generated_at": "2026-09-09T18:10:20Z", - "php_version": "8.2.32", + "generated_at": "2026-09-18T17:01:21Z", + "php_version": "8.2.33", "iterations": 3, "cases": { "assoc-heavy": { @@ -13,11 +13,11 @@ "aot": 0.008779684702555338 }, "binary-trees": { - "8.2": 0.02375332514444987, - "vm": null, - "jit": null, - "aotcompile": 0.15860867500305176, - "aot": 0.04365968704223633 + "8.2": 0.021964311599731445, + "vm": 10.458517074584961, + "jit": 10.1682710647583, + "aotcompile": 0.28298306465148926, + "aot": 0.009479999542236328 }, "call-heavy": { "8.2": 0.013673623402913412, @@ -41,11 +41,11 @@ "aot": 0.006700992584228516 }, "fannkuch-redux": { - "8.2": 0.037070274353027344, - "vm": null, - "jit": null, - "aotcompile": 0.17575367291768393, - "aot": 0.0879529317220052 + "8.2": 0.017571051915486652, + "vm": 7.6685521602630615, + "jit": 7.314037084579468, + "aotcompile": 0.2858619689941406, + "aot": 0.006447315216064453 }, "fasta": { "8.2": 0.014009316762288412, diff --git a/benchmarks/v2/binary-trees.php b/benchmarks/v2/binary-trees.php index f34f1c61c3e..ede32d19dbf 100644 --- a/benchmarks/v2/binary-trees.php +++ b/benchmarks/v2/binary-trees.php @@ -4,6 +4,9 @@ /** * Binary-trees (scaled) — object allocation / recursion (#36385). + * + * maxDepth stays ≤ 7 so default `bench.php --v2` VM verify finishes under the + * 25 s per-run cap (depth 10 timed out → burned the full cap with vm/jit n/a). */ final class TreeNode @@ -34,7 +37,7 @@ function itemCheck(TreeNode $node): int } $minDepth = 4; -$maxDepth = 10; +$maxDepth = 7; $stretchDepth = $maxDepth + 1; $stretch = bottomUpTree($stretchDepth); diff --git a/benchmarks/v2/fannkuch-redux.php b/benchmarks/v2/fannkuch-redux.php index 1a93c6c5b0a..12b1031c455 100644 --- a/benchmarks/v2/fannkuch-redux.php +++ b/benchmarks/v2/fannkuch-redux.php @@ -4,6 +4,9 @@ /** * Fannkuch-redux (scaled) — permutation / flip (#36385). + * + * n stays ≤ 6 so default `bench.php --v2` VM verify finishes under the 25 s + * per-run cap (n=8 timed out → burned the full cap with vm/jit n/a). */ function fannkuch(int $n): int @@ -66,4 +69,4 @@ function fannkuch(int $n): int } } -echo fannkuch(8), "\n"; +echo fannkuch(6), "\n"; diff --git a/docs/pages/bench.html b/docs/pages/bench.html index 1c324ad84be..3ecd29cb64b 100644 --- a/docs/pages/bench.html +++ b/docs/pages/bench.html @@ -44,7 +44,7 @@

Web-request (MiniWebApp)

FastCGI soak RSS (#36388)

- +