Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions benchmarks/v2/BASELINE.json
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
{
"version": 1,
"suite": "v2",
"generated_at": "2026-09-09T16:31:20Z",
"generated_at": "2026-09-18T17:00:45Z",
"arch": "x86_64-linux",
"ratio_tolerance_percent": 20,
"geomean_tolerance_percent": 10,
"geomean_ratio_aot_over_zend": 0.869234,
"geomean_ratio_aot_over_zend": 0.711107,
"ir_tolerance_percent": 20,
"regeneration": "PHP_8_2=$(command -v php) ./script/bench-gate.sh --v2 --update",
"note": "v2 gate subset (#36385). Per-case ratio +20%; suite geomean +10% so diffuse regressions fail.",
"note": "v2 gate subset (#36385). Per-case ratio +20%; suite geomean +10%. binary-trees maxDepth=7 / fannkuch n=6 so VM finishes under 25s wall cap.",
"cases": {
"call-heavy": {
"ratio_aot_over_zend": 0.461441,
Expand Down Expand Up @@ -109,16 +109,16 @@
"ir_defines": 555
},
"binary-trees": {
"ratio_aot_over_zend": 1.684036,
"zend_wall": 0.027164,
"aot_wall": 0.045745,
"ratio_aot_over_zend": 0.536744,
"zend_wall": 0.015482,
"aot_wall": 0.00831,
"ir_lines": 280636,
"ir_defines": 546
},
"fannkuch-redux": {
"ratio_aot_over_zend": 2.502303,
"zend_wall": 0.034162,
"aot_wall": 0.085484,
"ratio_aot_over_zend": 0.31601,
"zend_wall": 0.015905,
"aot_wall": 0.005026,
"ir_lines": 199262,
"ir_defines": 565
}
Expand Down
13 changes: 8 additions & 5 deletions benchmarks/v2/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@ That also refreshes `docs/pages/bench.html` (ratio sparklines + web column). Set

**Wall budget (#36385):** default `bench.php --v2` must finish in **< 15 min**. The harness
uses a 25 s per-run VM/JIT cap (override `PHP_COMPILER_BENCH_TIMEOUT`), 3 averaged
iterations, skips JIT after a VM time-cap, and reuses a verify sample when it already
took ≥ 5 s — so `binary-trees` / `fannkuch-redux` VM timeouts no longer burn ~10 min each.
iterations, skips JIT after a VM time-cap, reuses a verify sample when it already
took ≥ 5 s, and honors `// @bench-skip-vm: <reason>` so a known time-cap case cannot
burn the full cap. `binary-trees` uses `maxDepth = 7` and `fannkuch-redux` uses `n = 6`
so VM/JIT columns stay measurable under that cap (depth 10 / n=8 timed out). Fast gate:
`php script/check-bench-readme-sync.php --check-wall` projects wall from committed `RESULTS.json`.

Gate (AOT/Zend ratio + IR size vs committed baseline; subset that must match Zend under AOT):

Expand All @@ -46,16 +49,16 @@ The chart page `docs/pages/bench.html` is generated by `script/generate-bench-ch

<!-- v2 benchmark table start -->

Environment: 8.2.32 · LLVM 9 available · 3 iterations averaged, wall time per run.
Environment: 8.2.33 · LLVM 9 available · 3 iterations averaged, wall time per run.

| Test Name | Zend 8.2 (s)| bin/vm.php (s) | bin/jit.php (s) | phpc build (s) | native run (s) |
|--------------------|-------------------|----------------|-----------------|----------------|----------------|
| assoc-heavy | 0.0125 | 2.4465 | 2.5408 | 0.1881 | 0.0088 |
| binary-trees | 0.0238 | n/a | n/a | 0.1586 | 0.0437 |
| binary-trees | 0.0220 | 10.4585 | 10.1683 | 0.2830 | 0.0095 |
| call-heavy | 0.0137 | 15.3373 | 15.7211 | 0.1701 | 0.0048 |
| closure-heavy | 0.0167 | 2.6189 | 2.6044 | 2.2945 | 0.0070 |
| exceptions | 0.0138 | 4.1803 | 4.3185 | 0.8953 | 0.0067 |
| fannkuch-redux | 0.0371 | n/a | n/a | 0.1758 | 0.0880 |
| fannkuch-redux | 0.0176 | 7.6686 | 7.3140 | 0.2859 | 0.0064 |
| fasta | 0.0140 | 1.4628 | 1.4897 | 0.4918 | 0.0032 |
| json-roundtrip | 0.0103 | 0.8345 | 0.8959 | 0.1475 | 0.0376 |
| k-nucleotide | 0.0165 | 6.4666 | 6.7544 | 0.2238 | 0.0052 |
Expand Down
24 changes: 12 additions & 12 deletions benchmarks/v2/RESULTS.json
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
{
"version": 1,
"suite": "v2",
"generated_at": "2026-09-09T18:10:20Z",
"php_version": "8.2.32",
"generated_at": "2026-09-18T17:01:21Z",
"php_version": "8.2.33",
"iterations": 3,
"cases": {
"assoc-heavy": {
Expand All @@ -13,11 +13,11 @@
"aot": 0.008779684702555338
},
"binary-trees": {
"8.2": 0.02375332514444987,
"vm": null,
"jit": null,
"aotcompile": 0.15860867500305176,
"aot": 0.04365968704223633
"8.2": 0.021964311599731445,
"vm": 10.458517074584961,
"jit": 10.1682710647583,
"aotcompile": 0.28298306465148926,
"aot": 0.009479999542236328
},
"call-heavy": {
"8.2": 0.013673623402913412,
Expand All @@ -41,11 +41,11 @@
"aot": 0.006700992584228516
},
"fannkuch-redux": {
"8.2": 0.037070274353027344,
"vm": null,
"jit": null,
"aotcompile": 0.17575367291768393,
"aot": 0.0879529317220052
"8.2": 0.017571051915486652,
"vm": 7.6685521602630615,
"jit": 7.314037084579468,
"aotcompile": 0.2858619689941406,
"aot": 0.006447315216064453
},
"fasta": {
"8.2": 0.014009316762288412,
Expand Down
5 changes: 4 additions & 1 deletion benchmarks/v2/binary-trees.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@

/**
* Binary-trees (scaled) — object allocation / recursion (#36385).
*
* maxDepth stays ≤ 7 so default `bench.php --v2` VM verify finishes under the
* 25 s per-run cap (depth 10 timed out → burned the full cap with vm/jit n/a).
*/

final class TreeNode
Expand Down Expand Up @@ -34,7 +37,7 @@ function itemCheck(TreeNode $node): int
}

$minDepth = 4;
$maxDepth = 10;
$maxDepth = 7;
$stretchDepth = $maxDepth + 1;

$stretch = bottomUpTree($stretchDepth);
Expand Down
5 changes: 4 additions & 1 deletion benchmarks/v2/fannkuch-redux.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@

/**
* Fannkuch-redux (scaled) — permutation / flip (#36385).
*
* n stays ≤ 6 so default `bench.php --v2` VM verify finishes under the 25 s
* per-run cap (n=8 timed out → burned the full cap with vm/jit n/a).
*/

function fannkuch(int $n): int
Expand Down Expand Up @@ -66,4 +69,4 @@ function fannkuch(int $n): int
}
}

echo fannkuch(8), "\n";
echo fannkuch(6), "\n";
2 changes: 1 addition & 1 deletion docs/pages/bench.html

Large diffs are not rendered by default.

66 changes: 52 additions & 14 deletions script/bench.php
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,18 @@
const ITERATIONS_V2 = 3;
/**
* Fail-fast per-run cap for v2 VM/JIT (seconds). Legacy default remains 300s.
* Override with PHP_COMPILER_BENCH_TIMEOUT. binary-trees / fannkuch-redux still
* time out under VM; a 300s cap burned ~20 min on those two alone.
* Override with PHP_COMPILER_BENCH_TIMEOUT. Workloads that still exceed this
* under VM should shrink (see binary-trees / fannkuch-redux) or declare
* `@bench-skip-vm: <reason>` so the harness does not burn the full cap (#36385).
*/
const V2_DEFAULT_TIMEOUT_SEC = 25;
/**
* When a successful VM/JIT verify already took this long, reuse that sample as
* the column time instead of averaging more runs (#36385 wall budget).
*/
const V2_REUSE_VERIFY_SEC = 5.0;
/** Done-when wall budget for a full `bench.php --v2` regen (#36385). */
const V2_WALL_BUDGET_SEC = 900.0;

$root = dirname(__DIR__);
$argvList = $argv ?? [];
Expand Down Expand Up @@ -178,8 +181,8 @@
}

$suiteWall = microtime(true) - $suiteStarted;
printf("v2 suite wall: %.1fs (budget 900s / 15 min, #36385)\n", $suiteWall);
if ($suiteWall > 900.0) {
printf("v2 suite wall: %.1fs (budget %.0fs / 15 min, #36385)\n", $suiteWall, V2_WALL_BUDGET_SEC);
if ($suiteWall > V2_WALL_BUDGET_SEC) {
fwrite(STDERR, "bench.php --v2: exceeded 15 min wall (#36385 Done-when)\n");
exit(1);
}
Expand Down Expand Up @@ -227,18 +230,29 @@ function bench(
die("Failure for Zend {$name}: \"{$got}\" != \"{$expected}\"\n");
}
}

$skipVmReason = $v2 ? benchSkipVmReason($file) : null;
$vmCmd = escapeshellcmd($harnessPhp)
.' -d error_reporting=1 -d log_errors=1 -d display_errors=stderr'
.' '.escapeshellarg($root.'/bin/vm.php').' '.escapeshellarg($file);
$vmStarted = microtime(true);
$vmOut = trim(capture($vmCmd, $vmRc, null, $v2));
$vmVerifySec = microtime(true) - $vmStarted;
$vmOk = $vmOut === $expected;
$vmTimedOut = !$vmOk && capture_timed_out($vmRc);
if (!$vmOk) {
echo $vmTimedOut
? " vm.php exceeded the time cap — vm column n/a\n"
: " vm.php output mismatch — vm column n/a\n";
$vmOk = false;
$vmTimedOut = false;
$vmVerifySec = 0.0;
$vmRc = null;
if (null !== $skipVmReason) {
echo " skip vm ({$skipVmReason}) — vm column n/a\n";
$vmTimedOut = true; // also skip JIT (same workload)
} else {
$vmStarted = microtime(true);
$vmOut = trim(capture($vmCmd, $vmRc, null, $v2));
$vmVerifySec = microtime(true) - $vmStarted;
$vmOk = $vmOut === $expected;
$vmTimedOut = !$vmOk && capture_timed_out($vmRc);
if (!$vmOk) {
echo $vmTimedOut
? " vm.php exceeded the time cap — vm column n/a\n"
: " vm.php output mismatch — vm column n/a\n";
}
}

$jitCmd = $llvmEnv.' '.escapeshellcmd($harnessPhp).' '.escapeshellarg($root.'/bin/jit.php').' '.escapeshellarg($file);
Expand All @@ -249,7 +263,9 @@ function bench(
echo " no LLVM available — jit column n/a\n";
} elseif ($v2 && $vmTimedOut) {
// Same workload will hit the cap under JIT; do not burn another full timeout (#36385).
echo " skip jit after vm time-cap — jit column n/a\n";
echo null !== $skipVmReason
? " skip jit after @bench-skip-vm — jit column n/a\n"
: " skip jit after vm time-cap — jit column n/a\n";
} else {
$jitStarted = microtime(true);
$jitOk = trim(capture($jitCmd, $jitRc, null, $v2)) === $expected;
Expand Down Expand Up @@ -350,3 +366,25 @@ function buildCapSeconds(): int

return is_string($v) && ctype_digit($v) ? (int) $v : 1800;
}

/**
* Optional file annotation: `// @bench-skip-vm: <reason>` (#36385).
* Skips VM+JIT verify immediately so a known time-cap case cannot burn the
* full per-run timeout (AOT columns still measured).
*/
function benchSkipVmReason(string $file): ?string
{
$src = @file_get_contents($file);
if (!is_string($src) || '' === $src) {
return null;
}
// Only the leading docblock / first ~4 KiB — keep annotation cheap to scan.
$head = substr($src, 0, 4096);
if (1 === preg_match('/@bench-skip-vm:\s*(.+)$/m', $head, $m)) {
$reason = trim($m[1]);

return '' !== $reason ? $reason : 'annotated';
}

return null;
}
Loading
Loading