From 14f5a05590039a521ff0659acaeea263e684fe52 Mon Sep 17 00:00:00 2001 From: "ondrej@mirtes.cz" Date: Fri, 13 Nov 2026 22:41:01 +0000 Subject: [PATCH 1/2] Run the benchmarks with Tally The benchmarks in tests/bench/data are now cases of AnalysisBench, run by phpstan/tally. The expected numbers moved from expected.neon into the comment at the top of each file, and the workflow runs tally run --shard and tally compare instead of the scripts in tests/bench. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d525-9c4c-73cf-8edc-000aa8e78378 --- .github/workflows/benchmarks.yml | 36 +- CLAUDE.md | 2 +- Makefile | 2 +- build/ignore-by-php-version.neon.php | 1 - build/memory-reset-peak-usage.neon | 8 - build/phpstan.neon | 7 +- composer.json | 1 + composer.lock | 62 ++- phpcs.xml | 1 + tally.php | 15 + .../Benchmark/BenchmarkComparatorTest.php | 111 ---- .../Benchmark/BenchmarkProfileParserTest.php | 81 --- .../PHPStan/Benchmark/BenchmarkReportTest.php | 52 -- .../PHPStan/Benchmark/BenchmarkShardsTest.php | 36 -- tests/bench/AnalysedFileExpectations.php | 42 ++ tests/bench/AnalysedFileHeader.php | 76 +++ tests/bench/AnalysisBench.php | 137 +++++ tests/bench/BenchmarkComparator.php | 163 ------ tests/bench/BenchmarkProfileParser.php | 70 --- tests/bench/BenchmarkReport.php | 397 -------------- tests/bench/BenchmarkShards.php | 52 -- tests/bench/compare.php | 78 --- .../data/and-chain-resolve-type-blowup.php | 6 + tests/bench/data/and-chain-truthy-blowup.php | 6 + .../data/array-values-literal-keyed-map.php | 6 + tests/bench/data/big-constant-int-union.php | 6 + .../big-constant-string-union-operators.php | 6 + .../bench/data/big-constant-string-union.php | 6 + tests/bench/data/bug-10147.php | 6 + tests/bench/data/bug-10538.php | 6 + tests/bench/data/bug-10772.php | 6 + tests/bench/data/bug-10979.php | 6 + tests/bench/data/bug-11263.php | 6 + tests/bench/data/bug-11283.php | 6 + tests/bench/data/bug-11297.php | 6 + tests/bench/data/bug-11913.php | 6 + tests/bench/data/bug-12159.php | 6 + tests/bench/data/bug-12671.php | 6 + tests/bench/data/bug-12787.php | 6 + tests/bench/data/bug-12800.php | 6 + tests/bench/data/bug-13218.php | 6 + tests/bench/data/bug-13310.php | 6 + tests/bench/data/bug-13352.php | 6 + tests/bench/data/bug-13685.php | 6 + tests/bench/data/bug-1388.php | 6 + tests/bench/data/bug-13933.php | 6 + tests/bench/data/bug-14207-and.php | 6 + tests/bench/data/bug-14207.php | 6 + tests/bench/data/bug-14319.php | 6 + tests/bench/data/bug-14452.php | 6 + tests/bench/data/bug-14462.php | 6 + tests/bench/data/bug-1447.php | 6 + tests/bench/data/bug-14475.php | 6 + tests/bench/data/bug-14590.php | 6 + tests/bench/data/bug-14624.php | 6 + tests/bench/data/bug-14674.php | 6 + tests/bench/data/bug-14869-enum.php | 6 + tests/bench/data/bug-14869.php | 6 + tests/bench/data/bug-14972-concat.php | 6 + tests/bench/data/bug-14972.php | 6 + tests/bench/data/bug-14996.php | 6 + tests/bench/data/bug-15061.php | 6 + tests/bench/data/bug-15305.php | 6 + tests/bench/data/bug-15348.php | 6 + tests/bench/data/bug-15452.php | 6 + tests/bench/data/bug-15498.php | 6 + tests/bench/data/bug-15591.php | 6 + tests/bench/data/bug-3686.php | 6 + tests/bench/data/bug-4300.php | 6 + tests/bench/data/bug-4308.php | 6 + tests/bench/data/bug-5081.php | 6 + tests/bench/data/bug-5231.php | 6 + tests/bench/data/bug-5231_2.php | 6 + tests/bench/data/bug-5390.php | 6 + tests/bench/data/bug-6265.php | 6 + tests/bench/data/bug-6442.php | 6 + tests/bench/data/bug-6936.php | 6 + tests/bench/data/bug-6948.php | 6 + tests/bench/data/bug-7140.php | 6 + tests/bench/data/bug-7214.php | 6 + tests/bench/data/bug-7581.php | 6 + tests/bench/data/bug-7637.php | 6 + tests/bench/data/bug-7901.php | 6 + tests/bench/data/bug-7903.php | 6 + tests/bench/data/bug-8146a.php | 6 + tests/bench/data/bug-8146b.php | 6 + tests/bench/data/bug-8147.php | 6 + tests/bench/data/bug-8215.php | 6 + tests/bench/data/bug-8503.php | 6 + tests/bench/data/bug-9690.php | 6 + tests/bench/data/callable-union-describe.php | 6 + .../data/closure-variable-invocations.php | 6 + .../conditional-expression-infinite-loop.php | 6 + tests/bench/data/enum-case-union-removal.php | 6 + tests/bench/data/enum-value-keyed-map.php | 6 + .../finite-types-optional-keys-blowup.php | 6 + .../data/finite-types-optional-keys-count.php | 6 + .../flatten-types-optional-keys-blowup.php | 6 + tests/bench/data/hash-key-lookup.php | 6 + .../data/implode-optional-keys-blowup.php | 6 + tests/bench/data/impure-call-columns.php | 6 + .../bench/data/in-array-intersect-blowup.php | 6 + .../data/literal-key-union-offset-write.php | 6 + tests/bench/data/long-statement-list.php | 6 + .../data/loop-widening-optional-keys.php | 6 + tests/bench/data/loose-string-comparison.php | 6 + tests/bench/data/nested-context-snapshots.php | 6 + .../data/nested-loops-wrapping-array.php | 6 + tests/bench/data/nullsafe-chain-walk.php | 6 + tests/bench/data/or-chain-falsey-blowup.php | 6 + .../data/or-chain-resolve-type-blowup.php | 6 + .../bench/data/phpdoc-inheritance-speedup.php | 6 + .../bench/data/phpdoc-intersection-blowup.php | 6 + .../process-called-method-infinite-loop.php | 6 + tests/bench/data/sealed-shape-if-chain.php | 6 + tests/bench/data/union-fast-path.php | 6 + .../data/unrolled-foreach-list-guards.php | 6 + tests/bench/data/wordpress-user.php | 6 + tests/bench/expected.neon | 483 ------------------ tests/bench/measure.php | 138 ----- tests/bench/run.php | 153 ------ tests/bench/shard.php | 45 -- 122 files changed, 934 insertions(+), 1891 deletions(-) delete mode 100644 build/memory-reset-peak-usage.neon create mode 100644 tally.php delete mode 100644 tests/PHPStan/Benchmark/BenchmarkComparatorTest.php delete mode 100644 tests/PHPStan/Benchmark/BenchmarkProfileParserTest.php delete mode 100644 tests/PHPStan/Benchmark/BenchmarkReportTest.php delete mode 100644 tests/PHPStan/Benchmark/BenchmarkShardsTest.php create mode 100644 tests/bench/AnalysedFileExpectations.php create mode 100644 tests/bench/AnalysedFileHeader.php create mode 100644 tests/bench/AnalysisBench.php delete mode 100644 tests/bench/BenchmarkComparator.php delete mode 100644 tests/bench/BenchmarkProfileParser.php delete mode 100644 tests/bench/BenchmarkReport.php delete mode 100644 tests/bench/BenchmarkShards.php delete mode 100644 tests/bench/compare.php delete mode 100644 tests/bench/expected.neon delete mode 100644 tests/bench/measure.php delete mode 100644 tests/bench/run.php delete mode 100644 tests/bench/shard.php diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index a0909b18f78..fc792a1630c 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -8,6 +8,7 @@ on: - 'stubs/**' - 'composer.lock' - 'tests/bench/**' + - 'tally.php' - '.github/workflows/benchmarks.yml' push: branches: @@ -18,6 +19,7 @@ on: - 'stubs/**' - 'composer.lock' - 'tests/bench/**' + - 'tally.php' - '.github/workflows/benchmarks.yml' concurrency: @@ -31,7 +33,7 @@ env: CODSPEED_VERSION: "5.6.3" # of codspeed-runner-x86_64-unknown-linux-musl.tar.gz in that release CODSPEED_SHA256: "f29532a5cbcb3462bc5d6bcf5570ddb8c8cbfffed579cb48c35d3910ed5f7f00" - # tests/bench/measure.php calls it over FFI to measure only the analysis of the benchmark + # Tally calls it over FFI to count only the benchmark, not the warm-up before it INSTRUMENT_HOOKS_COMMIT: "8fd5b878499f819561b33219cdafaa6f8c28b4e4" jobs: @@ -39,13 +41,13 @@ jobs: name: "Benchmarks shard ${{ matrix.shard }}" # Pinned because the instruction counts depend on the PHP build and the system libraries runs-on: ubuntu-24.04 - # The slowest shard takes about 20 minutes, and run.php stops a benchmark after 25 + # The slowest shard takes about 20 minutes, and tally.php stops a benchmark after 25 timeout-minutes: 40 strategy: fail-fast: false matrix: - # Each shard is one runner, see tests/bench/shard.php + # Each shard is one runner, tally run --shard balances them by the expected instructions shard: [1, 2, 3, 4] steps: @@ -64,7 +66,7 @@ jobs: with: coverage: "none" php-version: "8.5" - extensions: ds,ffi,mbstring + extensions: ds,ffi,mbstring,posix ini-file: production ini-values: memory_limit=-1 @@ -91,23 +93,19 @@ jobs: - name: "Run benchmarks" env: INSTRUMENT_HOOKS_LIBRARY: ${{ runner.temp }}/instrument-hooks/libinstrument_hooks.so - SHARD: ${{ matrix.shard }} - SHARDS: ${{ strategy.job-total }} # Valgrind runs each benchmark on one CPU, so the shard runs as many at once as the runner has - run: | - mkdir -p tmp/bench-results - php tests/bench/shard.php "$SHARDS" "$SHARD" | xargs -P "$(nproc)" -I {} php tests/bench/run.php "tests/bench/data/{}" "tmp/bench-results/{}.neon" + run: "php vendor/bin/tally run --shard=${{ matrix.shard }}/${{ strategy.job-total }} --output=tmp/tally-results" - name: "Upload results" if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: benchmark-results-${{ matrix.shard }} - path: tmp/bench-results/*.neon + name: tally-results-${{ matrix.shard }} + path: tmp/tally-results if-no-files-found: ignore compare: - name: "Compare with expected.neon" + name: "Compare all shards" needs: run if: ${{ !cancelled() }} runs-on: ubuntu-latest @@ -135,16 +133,18 @@ jobs: - name: "Download results" uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - pattern: benchmark-results-* + pattern: tally-results-* merge-multiple: true - path: tmp/bench-results + path: tmp/tally-results - name: "Compare" - run: "php tests/bench/compare.php tmp/bench-results tmp/expected.neon" + run: "php vendor/bin/tally compare --table tmp/tally-results" - - name: "Upload measured metrics" + # vendor/bin/tally accept writes the numbers into the files in tests/bench/data + - name: "Upload results of all shards" if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: benchmark-measured - path: tmp/expected.neon + name: tally-results + path: tmp/tally-results + if-no-files-found: ignore diff --git a/CLAUDE.md b/CLAUDE.md index 6bfbaa5d77f..36263a6c57a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -294,7 +294,7 @@ New introduced classes in these categories should be shadowed with a native C++ - **Always verify the test fails before the fix.** Stating "tests pass" is not enough — that only proves the fix doesn't break things. `git stash` the source change, run the test, see it fail for the right reason; `git stash pop`, run again, see it pass. Only then is the fix confirmed. The `regression-test` skill bakes this in for bug-issue workflows. - **Reproducers come from the OP's playground link** in the issue body, not from later comments. Copy the snippet as close to verbatim as practical. - **Integration tests**: `AnalyserIntegrationTest` runs full analysis on test files and checks error output. -- **Benchmarks**: every file in `tests/bench/data` is analysed in the Benchmarks workflow under CodSpeed's CPU simulation, which counts instructions instead of measuring time, so the numbers are the same on every runner. `tests/bench/measure.php` boots PHPStan and analyses a warm-up file first, so only the analysis of the benchmark file itself is counted. `tests/bench/compare.php` compares them against `tests/bench/expected.neon`; when a change moves them on purpose or adds a benchmark, replace that file with the workflow's `benchmark-measured` artifact. +- **Benchmarks**: every file in `tests/bench/data` is analysed by [Tally](https://github.com/phpstan/tally) in the Benchmarks workflow, under CodSpeed's CPU simulation, which counts instructions instead of measuring time, so the numbers are the same on every runner. `tests/bench/AnalysisBench.php` boots PHPStan and analyses `tests/bench/Warmup.php` uncounted in a `#[Before]` method, so only the analysis of the benchmark file itself is counted. The expected instructions, cycles and memory are in the comment at the top of each data file (`tests/bench/AnalysedFileHeader.php`); a new file without the comment passes and gets its numbers from CI. When a change moves them on purpose or adds a benchmark, download the workflow's `tally-results` artifact and run `vendor/bin/tally accept `. The configuration is `tally.php`. ### Adding support for new PHP versions diff --git a/Makefile b/Makefile index 15e63882923..1c773c96aad 100644 --- a/Makefile +++ b/Makefile @@ -143,7 +143,7 @@ lint: --exclude tests/PHPStan/Rules/Variables/data/bug-14352.php \ --exclude tests/PHPStan/Rules/Variables/data/bug-14351.php \ --exclude tests/PHPStan/Rules/Properties/data/bug-14457.php \ - src tests + src tests tally.php install-paratest: composer install --working-dir tests diff --git a/build/ignore-by-php-version.neon.php b/build/ignore-by-php-version.neon.php index 77c8caf7c5e..debc6ba56c9 100644 --- a/build/ignore-by-php-version.neon.php +++ b/build/ignore-by-php-version.neon.php @@ -27,7 +27,6 @@ if (PHP_VERSION_ID < 80200) { $includes[] = __DIR__ . '/randomizer.neon'; - $includes[] = __DIR__ . '/memory-reset-peak-usage.neon'; } if (PHP_VERSION_ID >= 80000) { diff --git a/build/memory-reset-peak-usage.neon b/build/memory-reset-peak-usage.neon deleted file mode 100644 index e7591bd6c9b..00000000000 --- a/build/memory-reset-peak-usage.neon +++ /dev/null @@ -1,8 +0,0 @@ -parameters: - ignoreErrors: - - - # tests/bench/measure.php runs only on the PHP 8.5 of the Benchmarks workflow - message: '#^(Used function|Function) memory_reset_peak_usage not found\.$#' - identifier: function.notFound - count: 2 - path: ../tests/bench/measure.php diff --git a/build/phpstan.neon b/build/phpstan.neon index 586595aed6c..3347041d2d8 100644 --- a/build/phpstan.neon +++ b/build/phpstan.neon @@ -17,6 +17,7 @@ parameters: - PHPStan - ../src - ../tests + - ../tally.php bootstrapFiles: - ../tests/phpstan-bootstrap.php scanFiles: @@ -90,7 +91,6 @@ parameters: paths: - ../src/File/FsEventsFileMonitor.php - ../src/File/InotifyFileMonitor.php - - ../tests/bench/measure.php - identifier: shipmonk.deadMethod message: '#^Unused .*?::__construct#' # likely used in DIC @@ -109,6 +109,11 @@ parameters: - ../tests/PHPStan/Tests - ../tests/e2e reportUnmatched: false + - + # Tally calls the benchmark methods, their data providers and #[Before] methods + identifier: shipmonk.deadMethod + path: ../tests/bench/AnalysisBench.php + reportUnmatched: false - identifier: shipmonk.deadConstant paths: diff --git a/composer.json b/composer.json index e92a2c4a72e..71b5378f622 100644 --- a/composer.json +++ b/composer.json @@ -71,6 +71,7 @@ "phpstan/phpstan-nette": "^2.0", "phpstan/phpstan-phpunit": "^2.1", "phpstan/phpstan-strict-rules": "^2.1", + "phpstan/tally": "dev-main", "phpunit/phpunit": "^11.5.23", "shipmonk/composer-dependency-analyser": "^1.5", "shipmonk/dead-code-detector": "^1.4.2", diff --git a/composer.lock b/composer.lock index 7c105cafa7d..11af18bbf31 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "10873ca6b3f5192aa55c383ff2206c12", + "content-hash": "2903b076c8da1462199b1d569eefe18e", "packages": [ { "name": "clue/ndjson-react", @@ -4203,6 +4203,63 @@ "time": "2026-11-02T08:16:15+00:00", "published-time": "2026-11-02T10:55:07+00:00" }, + { + "name": "phpstan/tally", + "version": "dev-main", + "source": { + "type": "git", + "url": "https://github.com/phpstan/tally.git", + "reference": "0a1ec271d83f1430607b9a6b36e1d619ff0928d6" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phpstan/tally/zipball/0a1ec271d83f1430607b9a6b36e1d619ff0928d6", + "reference": "0a1ec271d83f1430607b9a6b36e1d619ff0928d6", + "shasum": "" + }, + "require": { + "php": "^8.2" + }, + "require-dev": { + "infection/infection": "^0.35.10", + "php-parallel-lint/php-parallel-lint": "^1.4", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-deprecation-rules": "^2.0", + "phpstan/phpstan-phpunit": "^2.0", + "phpstan/phpstan-strict-rules": "^2.0", + "phpunit/phpunit": "^11.5 || ^12.5", + "shipmonk/dead-code-detector": "^1.4" + }, + "suggest": { + "ext-ffi": "To count instructions under CodSpeed's CPU simulation" + }, + "default-branch": true, + "bin": [ + "bin/tally" + ], + "type": "library", + "autoload": { + "psr-4": { + "Tally\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "PHP benchmarking framework that doesn't care about wall-clock time or how fast your CPU is.", + "keywords": [ + "benchmark", + "codspeed", + "performance" + ], + "support": { + "issues": "https://github.com/phpstan/tally/issues", + "source": "https://github.com/phpstan/tally/tree/main" + }, + "time": "2026-11-13T22:28:28+00:00", + "published-time": "2026-11-13T22:28:33+00:00" + }, { "name": "phpunit/php-code-coverage", "version": "11.0.12", @@ -5882,7 +5939,8 @@ "aliases": [], "minimum-stability": "dev", "stability-flags": { - "jetbrains/phpstorm-stubs": 20 + "jetbrains/phpstorm-stubs": 20, + "phpstan/tally": 20 }, "prefer-stable": true, "prefer-lowest": false, diff --git a/phpcs.xml b/phpcs.xml index 98e0caae54d..b150454ae32 100644 --- a/phpcs.xml +++ b/phpcs.xml @@ -12,6 +12,7 @@ build src tests + tally.php compiler/src compiler/tests apigen/src diff --git a/tally.php b/tally.php new file mode 100644 index 00000000000..586e8c54b82 --- /dev/null +++ b/tally.php @@ -0,0 +1,15 @@ +withPaths([__DIR__ . '/tests/bench/AnalysisBench.php']) + // bin/phpstan restarts PHP with these OPcache settings, the benchmarks run without the restart + ->withIniEntries([...OpcacheSettings::getIniEntries(), 'memory_limit=-1']) + // The slowest benchmark takes a few seconds natively and about 13 minutes under the simulation. + // One stuck in a loop fails in the native run, one that regresses far beyond that fails as not + // measured instead of holding up its shard. + ->withTimeLimits(30, 25 * 60); diff --git a/tests/PHPStan/Benchmark/BenchmarkComparatorTest.php b/tests/PHPStan/Benchmark/BenchmarkComparatorTest.php deleted file mode 100644 index e83284617ff..00000000000 --- a/tests/PHPStan/Benchmark/BenchmarkComparatorTest.php +++ /dev/null @@ -1,111 +0,0 @@ - ['instructions' => $instructions, 'estimatedCycles' => $cycles, 'peakMemory' => 100_000_000], - 'fast.php' => ['instructions' => $instructions, 'estimatedCycles' => $cycles, 'peakMemory' => 100_000_000], - 'removed.php' => ['instructions' => $instructions], - ]; - $actual = [ - 'slow.php' => [ - // exactly at the tolerance - 'instructions' => $instructions + self::countFor('instructions', 200), - // just above it - 'estimatedCycles' => $cycles + self::countFor('estimatedCycles', 200) + 1, - // within the peak memory one - 'peakMemory' => 108_000_000, - ], - 'fast.php' => [ - 'instructions' => $instructions - self::countFor('instructions', 100), - 'estimatedCycles' => $cycles - self::countFor('estimatedCycles', 201), - 'peakMemory' => 89_000_000, - ], - 'added.php' => ['instructions' => 700], - ]; - - $statuses = []; - foreach (BenchmarkComparator::compare($expected, $actual) as $result) { - $statuses[] = [$result['benchmark'], $result['metric'], $result['status']]; - } - $this->assertSame([ - ['added.php', 'instructions', 'not expected'], - ['fast.php', 'instructions', 'ok'], - ['fast.php', 'estimatedCycles', 'faster'], - ['fast.php', 'peakMemory', 'faster'], - ['removed.php', 'instructions', 'not measured'], - ['slow.php', 'instructions', 'ok'], - ['slow.php', 'estimatedCycles', 'slower'], - ['slow.php', 'peakMemory', 'ok'], - ], $statuses); - } - - /** - * @return iterable - */ - public static function dataAllowedChange(): iterable - { - yield 'quickest range, allowed 50 % of its 100 ms' => [10, 50 / 10]; - yield 'just below 100 ms' => [99, 50 / 99]; - yield '25 % would be less than the 50 ms allowed below 100 ms' => [150, 50 / 150]; - yield '25 %' => [300, 0.25]; - yield '20 % would be less than the 125 ms allowed at 500 ms' => [600, 125 / 600]; - yield '20 %' => [1000, 0.2]; - yield '10 % would be less than the 400 ms allowed at 2 s' => [2500, 400 / 2500]; - yield '10 %' => [5000, 0.1]; - } - - #[DataProvider('dataAllowedChange')] - public function testAllowedChange(int $milliseconds, float $allowedChange): void - { - foreach (['instructions', 'estimatedCycles'] as $metric) { - $expected = self::countFor($metric, $milliseconds); - $this->assertEqualsWithDelta($allowedChange, BenchmarkComparator::allowedChange($metric, $expected), 1e-9, $metric); - - $allowed = (int) round($expected * $allowedChange); - $results = BenchmarkComparator::compare(['a.php' => [$metric => $expected]], ['a.php' => [$metric => $expected + $allowed]]); - $this->assertSame('ok', $results[0]['status'], $metric); - $results = BenchmarkComparator::compare(['a.php' => [$metric => $expected]], ['a.php' => [$metric => $expected + $allowed + 2]]); - $this->assertSame('slower', $results[0]['status'], $metric); - } - - $this->assertSame(0.1, BenchmarkComparator::allowedChange('peakMemory', 1_000_000)); - } - - public function testFails(): void - { - $instructions = self::countFor('instructions', 1000); - $expected = ['a.php' => ['instructions' => $instructions, 'peakMemory' => 100_000_000]]; - - $this->assertFalse(BenchmarkComparator::fails(BenchmarkComparator::compare($expected, ['a.php' => ['instructions' => (int) ($instructions * 1.2), 'peakMemory' => 110_000_000]]))); - $this->assertTrue(BenchmarkComparator::fails(BenchmarkComparator::compare($expected, ['a.php' => ['instructions' => (int) ($instructions * 1.21), 'peakMemory' => 100_000_000]]))); - $this->assertTrue(BenchmarkComparator::fails(BenchmarkComparator::compare($expected, ['a.php' => ['instructions' => $instructions, 'peakMemory' => 111_000_000]]))); - $this->assertTrue(BenchmarkComparator::fails(BenchmarkComparator::compare($expected, []))); - // faster and new benchmarks only ask for expected.neon to be updated - $this->assertFalse(BenchmarkComparator::fails(BenchmarkComparator::compare($expected, [ - 'a.php' => ['instructions' => (int) ($instructions / 2), 'peakMemory' => 50_000_000], - 'b.php' => ['instructions' => 100], - ]))); - } - - /** - * @param 'instructions'|'estimatedCycles' $metric - */ - private static function countFor(string $metric, int $milliseconds): int - { - return $milliseconds * BenchmarkComparator::PER_MILLISECOND[$metric]; - } - -} diff --git a/tests/PHPStan/Benchmark/BenchmarkProfileParserTest.php b/tests/PHPStan/Benchmark/BenchmarkProfileParserTest.php deleted file mode 100644 index 08922c5ffba..00000000000 --- a/tests/PHPStan/Benchmark/BenchmarkProfileParserTest.php +++ /dev/null @@ -1,81 +0,0 @@ -assertSame([ - 'instructions' => 5_605_163_255, - // 5605163255 + 10 * (32176288 + 34788426 + 12883708) + 100 * (89670 + 1339330 + 2320891) - 'estimatedCycles' => 6_778_636_575, - ], BenchmarkProfileParser::parseCallgrindOutput(self::PROFILE, 'tests/bench/data/bug-10147.php')); - } - - public function testParseCallgrindOutputOfOtherBenchmark(): void - { - $this->assertNull(BenchmarkProfileParser::parseCallgrindOutput(self::PROFILE, 'tests/bench/data/bug-1014.php')); - } - - public function testParseCallgrindOutputOfProcessWithoutMeasurement(): void - { - $contents = <<<'CALLGRIND' - # callgrind format - cmd: /usr/bin/stty -a - part: 1 - desc: Trigger: Program termination - events: Ir Dr Dw I1mr D1mr D1mw ILmr DLmr DLmw - totals: 21618 5618 2229 497 177 70 491 177 70 - CALLGRIND; - - $this->assertNull(BenchmarkProfileParser::parseCallgrindOutput($contents, 'tests/bench/data/bug-10147.php')); - } - -} diff --git a/tests/PHPStan/Benchmark/BenchmarkReportTest.php b/tests/PHPStan/Benchmark/BenchmarkReportTest.php deleted file mode 100644 index 8720df904e3..00000000000 --- a/tests/PHPStan/Benchmark/BenchmarkReportTest.php +++ /dev/null @@ -1,52 +0,0 @@ - ['instructions' => 1_000_000_000, 'peakMemory' => 100 * 1024 * 1024], - 'b-mixed.php' => ['instructions' => 2_000_000_000, 'peakMemory' => 100 * 1024 * 1024], - 'c-faster.php' => ['instructions' => 2_000_000_000], - 'e-small.php' => ['instructions' => 10_000_000], - ], - [ - 'a-ok.php' => ['instructions' => 1_010_000_000, 'peakMemory' => 100 * 1024 * 1024], - // slower in instructions, lower in memory: listed once, as slower, with both metrics - 'b-mixed.php' => ['instructions' => 3_000_000_000, 'peakMemory' => 80 * 1024 * 1024], - 'c-faster.php' => ['instructions' => 1_000_000_000], - 'd-new.php' => ['instructions' => 1], - // 30 % more, but within the 50 ms allowed below 100 ms - 'e-small.php' => ['instructions' => 13_000_000], - ], - ); - - $this->assertSame( - "::error file=tests/bench/data/b-mixed.php,line=1,title=b-mixed.php is slower than expected::instructions +50.0 %25 (3.00 G, expected 2.00 G, allowed ±25 %25)%0ApeakMemory -20.0 %25 (80.0 MB, expected 100.0 MB, allowed ±10 %25)\n" - . "::warning file=tests/bench/data/d-new.php,line=1,title=d-new.php has no expected numbers::expected.neon has no numbers for these yet. This does not fail the build, but accept the new numbers.\n" - . "::warning file=tests/bench/data/c-faster.php,line=1,title=c-faster.php is faster than expected::instructions -50.0 %25 (1.00 G, expected 2.00 G, allowed ±25 %25)\n", - BenchmarkReport::renderAnnotations($results), - ); - } - - public function testRenderTerminalWhenAllMatch(): void - { - $results = BenchmarkComparator::compare( - ['a.php' => ['instructions' => 1_000_000_000], 'b.php' => ['instructions' => 1_000_000_000]], - ['a.php' => ['instructions' => 1_010_000_000], 'b.php' => ['instructions' => 990_000_000]], - ); - - $this->assertSame( - "Benchmarks passed: all 2 match tests/bench/expected.neon\n" - . "Allowed difference from tests/bench/expected.neon: instructions and estimatedCycles by the time they take at 5.8 M instructions or 7.4 M cycles per ms: ±50 ms below 100 ms, ±25 % from 100 ms, ±20 % from 500 ms, ±10 % from 2000 ms, each at least what the range below allows; ±10 % peakMemory.\n", - BenchmarkReport::renderTerminal($results, 'accept', false), - ); - } - -} diff --git a/tests/PHPStan/Benchmark/BenchmarkShardsTest.php b/tests/PHPStan/Benchmark/BenchmarkShardsTest.php deleted file mode 100644 index b98c49c844f..00000000000 --- a/tests/PHPStan/Benchmark/BenchmarkShardsTest.php +++ /dev/null @@ -1,36 +0,0 @@ - 7, 'a.php' => 10, 'e.php' => 5, 'b.php' => 8, 'd.php' => 6, 'f.php' => 4, 'g.php' => 2]; - - $this->assertSame([ - ['a.php', 'd.php', 'f.php', 'g.php'], - ['b.php', 'c.php', 'e.php'], - ], BenchmarkShards::split($costs, 2)); - } - - public function testSplitBreaksTiesByName(): void - { - $this->assertSame([ - ['a.php', 'd.php'], - ['b.php'], - ['c.php'], - ], BenchmarkShards::split(['d.php' => 1, 'c.php' => 1, 'b.php' => 1, 'a.php' => 1], 3)); - } - - public function testSplitIntoMoreShardsThanBenchmarks(): void - { - $this->assertSame([['a.php'], ['b.php']], BenchmarkShards::split(['a.php' => 3, 'b.php' => 1], 4)); - } - -} diff --git a/tests/bench/AnalysedFileExpectations.php b/tests/bench/AnalysedFileExpectations.php new file mode 100644 index 00000000000..a6d2c1de5c9 --- /dev/null +++ b/tests/bench/AnalysedFileExpectations.php @@ -0,0 +1,42 @@ +metrics; + } + + #[Override] + public function file(): string + { + return $this->file; + } + + #[Override] + public function rewrite(string $code, Metrics $measured): string + { + return $this->header->write($code, $measured); + } + + #[Override] + public function location(): SourceLocation + { + return new SourceLocation($this->file, $this->metrics === null ? 1 : AnalysedFileHeader::LINE); + } + +} diff --git a/tests/bench/AnalysedFileHeader.php b/tests/bench/AnalysedFileHeader.php new file mode 100644 index 00000000000..d15bea1e012 --- /dev/null +++ b/tests/bench/AnalysedFileHeader.php @@ -0,0 +1,76 @@ +format($metrics, Metric::Instructions), + $this->format($metrics, Metric::Cycles), + $this->format($metrics, Metric::Memory), + ); + if (preg_match(self::PATTERN, $code) === 1) { + // the replacement reads $ and \ + return (string) preg_replace(self::PATTERN, '$1' . "\n" . addcslashes($header, '$\\'), $code); + } + if (preg_match('~\A<\?php[^\n]*\n~', $code, $matches) !== 1) { + throw new ExpectationException('The benchmark does not start with a format()->format($metrics->get($metric) ?? 0); + } + +} diff --git a/tests/bench/AnalysisBench.php b/tests/bench/AnalysisBench.php new file mode 100644 index 00000000000..565da51eedf --- /dev/null +++ b/tests/bench/AnalysisBench.php @@ -0,0 +1,137 @@ + 0.5, 100 => 0.25, 500 => 0.2, 2000 => 0.1], + instructionsPerMillisecond: 5_800_000, + cyclesPerMillisecond: 7_400_000, + memory: 0.1, +)] +final class AnalysisBench +{ + + private const WARMUP_FILE = __DIR__ . '/Warmup.php'; + + /** @var (Closure(string): int)|null analyses a file and returns the number of errors */ + private ?Closure $analyse = null; + + public function __construct(private ScratchDirectory $scratch) + { + } + + /** + * @return iterable + * @throws InvalidQuantityException + */ + public static function files(): iterable + { + $header = new AnalysedFileHeader(); + foreach (glob(__DIR__ . '/data/*.php') ?: [] as $file) { + $name = basename($file); + yield $name => DataSet::named($name, [$file]) + ->withExpectations(new AnalysedFileExpectations($header, $file, $header->read((string) file_get_contents($file)))); + } + } + + /** + * @throws InceptionNotSuccessfulException + */ + #[Before] + public function boot(string $file): void + { + // like bin/phpstan + error_reporting(E_ALL & ~E_DEPRECATED); + gc_disable(); + + $configFile = $this->scratch->path() . '/phpstan.neon'; + FileSystem::write($configFile, Neon::encode([ + 'includes' => [__DIR__ . '/phpstan.neon'], + 'parameters' => ['tmpDir' => $this->scratch->path() . '/cache'], + ], true)); + + $inceptionResult = CommandHelper::begin( + new ArrayInput([]), + new StreamOutput(STDERR), + [self::WARMUP_FILE, $file], + '-1', + null, + [dirname(__DIR__, 2)], + $configFile, + null, + null, + false, + true, + null, + null, + false, + ); + // also tells the parser which files are analysed, the others are parsed without function bodies + $inceptionResult->getFiles(); + $container = $inceptionResult->getContainer(); + $analyser = $container->getByType(Analyser::class); + $finalizer = $container->getByType(AnalyserResultFinalizer::class); + $this->analyse = static function (string $file) use ($analyser, $finalizer): int { + $result = $analyser->analyse([$file], null, null, true, [$file]); + + return count($finalizer->finalize($result, false, true)->getErrors()); + }; + + // A different file than the benchmark: PHPStan memoizes types and reflection in memory, + // so analysing the benchmark twice would hide work from the counted analysis + ($this->analyse)(self::WARMUP_FILE); + } + + #[DataProvider(method: 'files')] + public function benchAnalyse(string $file): int + { + if ($this->analyse === null) { + throw new LogicException('boot() did not run.'); + } + + return ($this->analyse)($file); + } + +} diff --git a/tests/bench/BenchmarkComparator.php b/tests/bench/BenchmarkComparator.php deleted file mode 100644 index fc1c5f03c4c..00000000000 --- a/tests/bench/BenchmarkComparator.php +++ /dev/null @@ -1,163 +0,0 @@ - 5_800_000, - 'estimatedCycles' => 7_400_000, - ]; - - /** Allowed relative difference of a counted metric, keyed by the expected milliseconds from which it applies */ - public const DURATION_TOLERANCES = [ - 0 => 0.5, - 100 => 0.25, - 500 => 0.2, - 2000 => 0.1, - ]; - - /** Allowed relative difference of the peak memory */ - public const PEAK_MEMORY_TOLERANCE = 0.1; - - /** - * @param array> $expected metric values keyed by benchmark file name and metric name - * @param array> $actual the same shape, as measured - * @return list, expected: int|null, actual: int|null, change: float|null, allowedChange: float|null, status: self::STATUS_*}> sorted by benchmark, then in the order of METRICS - */ - public static function compare(array $expected, array $actual): array - { - $benchmarks = array_unique([...array_keys($expected), ...array_keys($actual)]); - sort($benchmarks, SORT_STRING); - - $results = []; - foreach ($benchmarks as $benchmark) { - foreach (self::METRICS as $metric) { - $expectedValue = $expected[$benchmark][$metric] ?? null; - $actualValue = $actual[$benchmark][$metric] ?? null; - if ($expectedValue === null && $actualValue === null) { - continue; - } - - $change = null; - $allowedChange = null; - if ($expectedValue === null) { - $status = self::STATUS_NOT_EXPECTED; - } elseif ($actualValue === null) { - $status = self::STATUS_NOT_MEASURED; - } else { - $change = ($actualValue - $expectedValue) / (float) $expectedValue; - $allowedChange = self::allowedChange($metric, $expectedValue); - if (abs($change) <= $allowedChange) { - $status = self::STATUS_OK; - } elseif ($change > 0) { - $status = self::STATUS_SLOWER; - } else { - $status = self::STATUS_FASTER; - } - } - - $results[] = [ - 'benchmark' => $benchmark, - 'metric' => $metric, - 'expected' => $expectedValue, - 'actual' => $actualValue, - 'change' => $change, - 'allowedChange' => $allowedChange, - 'status' => $status, - ]; - } - } - - return $results; - } - - /** - * @param value-of $metric - */ - public static function allowedChange(string $metric, int $expectedValue): float - { - if ($metric === 'peakMemory') { - return self::PEAK_MEMORY_TOLERANCE; - } - - $expectedMilliseconds = $expectedValue / self::PER_MILLISECOND[$metric]; - $starts = array_keys(self::DURATION_TOLERANCES); - $allowedMilliseconds = 0.0; - foreach ($starts as $i => $start) { - if ($start > $expectedMilliseconds) { - break; - } - $end = $starts[$i + 1] ?? INF; - $tolerance = self::DURATION_TOLERANCES[$start]; - // the quickest range is allowed its margin at its upper end, the others at least that - $allowedMilliseconds = max($allowedMilliseconds, $tolerance * ($i === 0 ? $end : min($expectedMilliseconds, $end))); - } - - return $allowedMilliseconds / $expectedMilliseconds; - } - - /** - * @param list $results from compare() - */ - public static function fails(array $results): bool - { - foreach ($results as $result) { - if (in_array($result['status'], self::FAILING_STATUSES, true)) { - return true; - } - } - - return false; - } - -} diff --git a/tests/bench/BenchmarkProfileParser.php b/tests/bench/BenchmarkProfileParser.php deleted file mode 100644 index 9141d6ffbcf..00000000000 --- a/tests/bench/BenchmarkProfileParser.php +++ /dev/null @@ -1,70 +0,0 @@ - $instructions, - 'estimatedCycles' => $instructions + 10 * $firstLevelMisses + 100 * $lastLevelMisses, - ]; - } - } - - return null; - } - - /** - * @return list - */ - private static function splitFields(string $fields): array - { - return explode(' ', trim($fields)); - } - -} diff --git a/tests/bench/BenchmarkReport.php b/tests/bench/BenchmarkReport.php deleted file mode 100644 index 1794e84583c..00000000000 --- a/tests/bench/BenchmarkReport.php +++ /dev/null @@ -1,397 +0,0 @@ -, expected: int|null, actual: int|null, change: float|null, allowedChange: float|null, status: BenchmarkComparator::STATUS_*} - */ -final class BenchmarkReport -{ - - /** The sections in the order they are shown, the first one a benchmark has a metric in wins */ - private const SECTIONS = [ - BenchmarkComparator::STATUS_SLOWER => [ - 'title' => 'Slower than expected', - 'alert' => 'CAUTION', - 'color' => '31', - 'annotation' => 'error', - 'annotationTitle' => '%s is slower than expected', - 'text' => 'This change makes PHPStan do more work on these files. Find out why before merging. If it is the price of a fix, accept the new numbers.', - ], - BenchmarkComparator::STATUS_NOT_MEASURED => [ - 'title' => 'Not measured', - 'alert' => 'WARNING', - 'color' => '33', - 'annotation' => 'error', - 'annotationTitle' => '%s was not measured', - 'text' => 'These benchmarks failed or timed out, see the log of their shard. A benchmark removed from data/ also has to be removed from expected.neon.', - ], - BenchmarkComparator::STATUS_NOT_EXPECTED => [ - 'title' => 'New benchmarks', - 'alert' => 'NOTE', - 'color' => '34', - 'annotation' => 'warning', - 'annotationTitle' => '%s has no expected numbers', - 'text' => 'expected.neon has no numbers for these yet. This does not fail the build, but accept the new numbers.', - ], - BenchmarkComparator::STATUS_FASTER => [ - 'title' => 'Faster than expected', - 'alert' => 'TIP', - 'color' => '32', - 'annotation' => 'warning', - 'annotationTitle' => '%s is faster than expected', - 'text' => 'Nice! This does not fail the build, but accept the new numbers, so a later slowdown back to the old ones does not go unnoticed.', - ], - ]; - - private const HEADER = ['Benchmark', 'Metric', 'Expected', 'Measured', 'Change', 'Allowed']; - - /** - * @param list $results - * @param string $acceptCommand shell commands that replace expected.neon with the measured numbers - */ - public static function renderTerminal(array $results, string $acceptCommand, bool $colors): string - { - $style = static fn (string $code, string $text): string => $colors ? "\e[" . $code . 'm' . $text . "\e[0m" : $text; - $sections = self::groupBySection($results); - - $output = $style('1', self::headline($results, $sections)) . "\n"; - $output .= self::toleranceLine() . "\n"; - foreach ($sections as $status => $rows) { - $section = self::SECTIONS[$status]; - $output .= "\n" . $style('1;' . $section['color'], $section['title']) . "\n"; - $output .= $section['text'] . "\n\n"; - $output .= self::renderTextTable(self::tableRows($rows), ' '); - } - if ($sections === []) { - return $output; - } - - $output .= "\n" . $style('1', 'Reproduce locally') . ', on this branch and on the base branch:' . "\n"; - $output .= ' ' . self::reproduceCommand($sections) . "\n"; - $output .= "\n" . $style('1', 'Accept the new numbers') . "\n"; - $output .= ' ' . str_replace("\n", "\n ", $acceptCommand) . "\n"; - - return $output; - } - - /** - * Every benchmark with its metrics, for the collapsed part of the log. - * - * @param list $results - */ - public static function renderTerminalTable(array $results): string - { - return self::renderTextTable([ - [...self::HEADER, 'Status'], - ...array_map(static fn (array $result): array => [...self::tableRow($result), $result['status']], $results), - ], ''); - } - - /** - * @param list $results - */ - public static function renderMarkdown(array $results, string $acceptCommand): string - { - $sections = self::groupBySection($results); - - $output = '## ' . self::headline($results, $sections) . "\n\n"; - $output .= self::toleranceLine() . "\n"; - foreach ($sections as $status => $rows) { - $section = self::SECTIONS[$status]; - $output .= sprintf("\n> [!%s]\n> **%s**\n> %s\n\n", $section['alert'], $section['title'], $section['text']); - $output .= self::renderMarkdownTable(self::tableRows($rows)); - } - if ($sections !== []) { - $output .= "\n### Reproduce locally\n\nOn this branch and on the base branch:\n\n```\n" . self::reproduceCommand($sections) . "\n```\n"; - $output .= "\n### Accept the new numbers\n\n```\n" . $acceptCommand . "\n```\n"; - } - - $output .= "\n
All benchmarks\n\n"; - $output .= self::renderMarkdownTable([ - [...self::HEADER, 'Status'], - ...array_map(static fn (array $result): array => [...self::tableRow($result), $result['status']], $results), - ]); - - return $output . "\n
\n"; - } - - /** - * GitHub Actions workflow commands that annotate the benchmark files, one per benchmark. - * - * @param list $results - */ - public static function renderAnnotations(array $results): string - { - $output = ''; - foreach (self::groupBySection($results) as $status => $rows) { - $byBenchmark = []; - foreach ($rows as $row) { - $byBenchmark[$row['benchmark']][] = $row; - } - foreach ($byBenchmark as $benchmark => $benchmarkRows) { - $changes = []; - foreach ($benchmarkRows as $row) { - if ($row['change'] === null) { - continue; - } - $changes[] = sprintf( - '%s %s (%s, expected %s, allowed ±%s %%)', - $row['metric'], - self::formatChange($row['change']), - self::formatShort($row['metric'], $row['actual']), - self::formatShort($row['metric'], $row['expected']), - self::formatPercent($row['allowedChange'] ?? 0.0), - ); - } - $output .= sprintf( - "::%s file=tests/bench/data/%s,line=1,title=%s::%s\n", - self::SECTIONS[$status]['annotation'], - $benchmark, - self::escapeAnnotation(sprintf(self::SECTIONS[$status]['annotationTitle'], $benchmark)), - self::escapeAnnotation($changes === [] ? self::SECTIONS[$status]['text'] : implode("\n", $changes)), - ); - } - } - - return $output; - } - - /** - * @param list $results - * @return array, non-empty-list> only the sections with a benchmark, in the order of SECTIONS - */ - private static function groupBySection(array $results): array - { - $statusesByBenchmark = []; - foreach ($results as $result) { - $statusesByBenchmark[$result['benchmark']][$result['status']] = true; - } - - $sections = []; - foreach ($results as $result) { - if ($result['status'] === BenchmarkComparator::STATUS_OK) { - continue; - } - foreach (array_keys(self::SECTIONS) as $status) { - if (isset($statusesByBenchmark[$result['benchmark']][$status])) { - $sections[$status][] = $result; - break; - } - } - } - - $ordered = []; - foreach (array_keys(self::SECTIONS) as $status) { - if (!isset($sections[$status])) { - continue; - } - - $ordered[$status] = $sections[$status]; - } - - return $ordered; - } - - /** - * @param list $results - * @param array, non-empty-list> $sections - */ - private static function headline(array $results, array $sections): string - { - $benchmarkCounts = []; - foreach ($sections as $status => $rows) { - $benchmarkCounts[$status] = count(array_unique(array_map(static fn (array $row): string => $row['benchmark'], $rows))); - } - $allBenchmarks = count(array_unique(array_map(static fn (array $result): string => $result['benchmark'], $results))); - $unchanged = $allBenchmarks - array_sum($benchmarkCounts); - if ($benchmarkCounts === []) { - return sprintf('Benchmarks passed: all %d match tests/bench/expected.neon', $allBenchmarks); - } - - $parts = []; - foreach ($benchmarkCounts as $status => $benchmarkCount) { - $parts[] = sprintf('%d %s', $benchmarkCount, $status === BenchmarkComparator::STATUS_NOT_EXPECTED ? 'new' : $status); - } - $parts[] = sprintf('%d unchanged', $unchanged); - - return (BenchmarkComparator::fails($results) ? 'Benchmarks failed: ' : 'Benchmarks passed: ') . implode(', ', $parts); - } - - private static function toleranceLine(): string - { - $starts = array_keys(BenchmarkComparator::DURATION_TOLERANCES); - $ranges = []; - foreach ($starts as $i => $start) { - $tolerance = BenchmarkComparator::DURATION_TOLERANCES[$start]; - $ranges[] = $i === 0 - ? sprintf('±%d ms below %d ms', (int) round($tolerance * $starts[1]), $starts[1]) - : sprintf('±%d %% from %d ms', (int) round($tolerance * 100), $start); - } - - return sprintf( - 'Allowed difference from tests/bench/expected.neon: instructions and estimatedCycles by the time they take at %s instructions or %s cycles per ms: %s, each at least what the range below allows; ±%d %% peakMemory.', - number_format(BenchmarkComparator::PER_MILLISECOND['instructions'] / 1e6, 1) . ' M', - number_format(BenchmarkComparator::PER_MILLISECOND['estimatedCycles'] / 1e6, 1) . ' M', - implode(', ', $ranges), - (int) round(BenchmarkComparator::PEAK_MEMORY_TOLERANCE * 100), - ); - } - - /** - * @param array, non-empty-list> $sections - */ - private static function reproduceCommand(array $sections): string - { - $worst = null; - foreach ($sections[array_key_first($sections) ?? BenchmarkComparator::STATUS_SLOWER] ?? [] as $row) { - if ($worst !== null && abs($row['change'] ?? 0.0) <= abs($worst['change'] ?? 0.0)) { - continue; - } - - $worst = $row; - } - if ($worst === null) { - return ''; - } - - return 'bin/phpstan analyse -c tests/bench/phpstan.neon --debug tests/bench/data/' . $worst['benchmark']; - } - - /** - * @param list $rows - * @return non-empty-list> - */ - private static function tableRows(array $rows): array - { - return [self::HEADER, ...array_map(static fn (array $row): array => self::tableRow($row), $rows)]; - } - - /** - * @param Result $result - * @return list - */ - private static function tableRow(array $result): array - { - return [ - $result['benchmark'], - $result['metric'], - self::formatValue($result['metric'], $result['expected']), - self::formatValue($result['metric'], $result['actual']), - $result['change'] === null ? '-' : self::formatChange($result['change']), - $result['allowedChange'] === null ? '-' : '±' . self::formatPercent($result['allowedChange']) . ' %', - ]; - } - - /** - * Left-aligns the first two columns and right-aligns the numbers. - * - * @param list> $rows the first one is the header - */ - private static function renderTextTable(array $rows, string $indent): string - { - $widths = []; - foreach ($rows as $row) { - foreach ($row as $i => $cell) { - $widths[$i] = max($widths[$i] ?? 0, mb_strlen($cell)); - } - } - - $lines = []; - foreach ($rows as $row) { - $cells = []; - foreach ($row as $i => $cell) { - $cells[] = mb_str_pad($cell, $widths[$i], ' ', $i < 2 || $i > 5 ? STR_PAD_RIGHT : STR_PAD_LEFT); - } - $lines[] = rtrim($indent . implode(' ', $cells)); - } - - return implode("\n", $lines) . "\n"; - } - - /** - * @param non-empty-list> $rows the first one is the header - */ - private static function renderMarkdownTable(array $rows): string - { - $header = array_shift($rows); - $output = '| ' . implode(' | ', $header) . " |\n"; - $output .= '|' . implode('|', array_map(static fn (int $i): string => $i < 2 || $i > 5 ? '---' : '--:', array_keys($header))) . "|\n"; - foreach ($rows as $row) { - $row[0] = '`' . $row[0] . '`'; - $output .= '| ' . implode(' | ', $row) . " |\n"; - } - - return $output; - } - - private static function formatValue(string $metric, ?int $value): string - { - if ($value === null) { - return '-'; - } - if ($metric === 'peakMemory') { - return number_format($value / 1024 / 1024, 1) . ' MB'; - } - - return number_format($value); - } - - private static function formatShort(string $metric, ?int $value): string - { - if ($value === null || $metric === 'peakMemory') { - return self::formatValue($metric, $value); - } - - if ($value < 1_000_000_000) { - return number_format($value / 1e6) . ' M'; - } - - return number_format($value / 1e9, 2) . ' G'; - } - - private static function formatPercent(float $fraction): string - { - $percent = $fraction * 100; - - return $percent >= 10 || $percent === round($percent) ? (string) (int) round($percent) : sprintf('%.1f', $percent); - } - - private static function formatChange(float $change): string - { - return sprintf('%+.1f %%', $change * 100); - } - - /** https://github.com/actions/toolkit/blob/main/packages/core/src/command.ts */ - private static function escapeAnnotation(string $text): string - { - return str_replace(['%', "\r", "\n"], ['%25', '%0D', '%0A'], $text); - } - -} diff --git a/tests/bench/BenchmarkShards.php b/tests/bench/BenchmarkShards.php deleted file mode 100644 index c3a2426aaf5..00000000000 --- a/tests/bench/BenchmarkShards.php +++ /dev/null @@ -1,52 +0,0 @@ - $costs expected cost keyed by benchmark - * @return list> benchmarks of each shard from the most expensive one, without empty shards - */ - public static function split(array $costs, int $count): array - { - // ties are broken by name, so the shards stay the same while the costs do - ksort($costs, SORT_STRING); - arsort($costs, SORT_NUMERIC); - - $shards = []; - $totals = []; - foreach (range(0, $count - 1) as $shard) { - $shards[$shard] = []; - $totals[$shard] = 0; - } - - foreach ($costs as $benchmark => $cost) { - $cheapestShard = array_keys($totals, min($totals), true)[0]; - $shards[$cheapestShard][] = $benchmark; - $totals[$cheapestShard] += $cost; - } - - return array_values(array_filter($shards, static fn (array $benchmarks): bool => count($benchmarks) > 0)); - } - -} diff --git a/tests/bench/compare.php b/tests/bench/compare.php deleted file mode 100644 index 7e17d0ee5c3..00000000000 --- a/tests/bench/compare.php +++ /dev/null @@ -1,78 +0,0 @@ - [] - * - * The written file has the format of expected.neon, so it can replace it after an intended change. - */ - -use Nette\Neon\Neon; -use function count; -use function file_get_contents; -use function file_put_contents; -use function fwrite; -use function getenv; -use function glob; -use function is_file; -use function ksort; -use function sprintf; -use function stream_isatty; -use const FILE_APPEND; -use const SORT_STRING; -use const STDERR; -use const STDOUT; - -require_once __DIR__ . '/../../vendor/autoload.php'; - -if (!isset($argv) || (count($argv) !== 2 && count($argv) !== 3)) { - fwrite(STDERR, "Usage: php tests/bench/compare.php []\n"); - exit(2); -} - -$expectedFile = __DIR__ . '/expected.neon'; -$expected = is_file($expectedFile) ? Neon::decode((string) file_get_contents($expectedFile)) : []; - -$actual = []; -foreach (glob($argv[1] . '/*.neon') ?: [] as $resultFile) { - $actual += Neon::decode((string) file_get_contents($resultFile)); -} -ksort($actual, SORT_STRING); - -if (isset($argv[2])) { - $header = "# Expected metrics of the benchmarks in data/, compared by compare.php with the tolerances in BenchmarkComparator.\n" - . "# To update it, replace it with the benchmark-measured artifact of the Benchmarks workflow.\n\n"; - file_put_contents($argv[2], $header . Neon::encode($actual, true)); -} - -$results = BenchmarkComparator::compare($expected, $actual); - -$onGitHub = getenv('GITHUB_ACTIONS') === 'true'; -$runId = getenv('GITHUB_RUN_ID'); -$repository = getenv('GITHUB_REPOSITORY'); -$acceptCommand = sprintf( - "gh run download %s%s -n benchmark-measured -D tmp/benchmark-measured\nmv tmp/benchmark-measured/expected.neon tests/bench/expected.neon", - $runId !== false && $runId !== '' ? $runId : '', - $repository !== false && $repository !== '' ? ' -R ' . $repository : '', -); - -echo BenchmarkReport::renderTerminal($results, $acceptCommand, $onGitHub || stream_isatty(STDOUT)); -echo "\n"; -echo $onGitHub ? "::group::All benchmarks\n" : "All benchmarks\n\n"; -echo BenchmarkReport::renderTerminalTable($results); -if ($onGitHub) { - echo "::endgroup::\n"; - echo BenchmarkReport::renderAnnotations($results); -} - -$stepSummaryFile = getenv('GITHUB_STEP_SUMMARY'); -if ($stepSummaryFile !== false && $stepSummaryFile !== '') { - file_put_contents($stepSummaryFile, BenchmarkReport::renderMarkdown($results, $acceptCommand), FILE_APPEND); -} - -exit(BenchmarkComparator::fails($results) ? 1 : 0); diff --git a/tests/bench/data/and-chain-resolve-type-blowup.php b/tests/bench/data/and-chain-resolve-type-blowup.php index a80e6aa3d0a..83ee80d234a 100644 --- a/tests/bench/data/and-chain-resolve-type-blowup.php +++ b/tests/bench/data/and-chain-resolve-type-blowup.php @@ -1,5 +1,11 @@ = 8.1 +/* + * instructions: 4.344B + * cycles: 5.682B + * memory: 94.20MiB + */ + namespace Bug10772; use PrinsFrank\Standards\Country\CountryAlpha2; diff --git a/tests/bench/data/bug-10979.php b/tests/bench/data/bug-10979.php index e244f5a7f26..4d3383b2d03 100644 --- a/tests/bench/data/bug-10979.php +++ b/tests/bench/data/bug-10979.php @@ -1,5 +1,11 @@ = 8.1 +/* + * instructions: 4.081B + * cycles: 5.558B + * memory: 42.37MiB + */ + declare(strict_types=1); namespace Bug10979; diff --git a/tests/bench/data/bug-11263.php b/tests/bench/data/bug-11263.php index 938527eda61..bb35537ddb5 100644 --- a/tests/bench/data/bug-11263.php +++ b/tests/bench/data/bug-11263.php @@ -1,5 +1,11 @@ = 8.1 +/* + * instructions: 1.927B + * cycles: 2.275B + * memory: 42.12MiB + */ + declare(strict_types = 1); namespace Bug11263; diff --git a/tests/bench/data/bug-11283.php b/tests/bench/data/bug-11283.php index 26c83f50b7b..8756488c8a1 100644 --- a/tests/bench/data/bug-11283.php +++ b/tests/bench/data/bug-11283.php @@ -1,5 +1,11 @@ = 8.0 +/* + * instructions: 1.208B + * cycles: 1.591B + * memory: 40.16MiB + */ + declare(strict_types=1); namespace Bug11283; diff --git a/tests/bench/data/bug-11297.php b/tests/bench/data/bug-11297.php index 5bc767581ce..af51ef84c3c 100644 --- a/tests/bench/data/bug-11297.php +++ b/tests/bench/data/bug-11297.php @@ -1,5 +1,11 @@ = 8.1 +/* + * instructions: 9.284B + * cycles: 10.27B + * memory: 65.86MiB + */ + namespace Bug11297; class ClassA { diff --git a/tests/bench/data/bug-11913.php b/tests/bench/data/bug-11913.php index 865023c7252..1fc3dc7190a 100644 --- a/tests/bench/data/bug-11913.php +++ b/tests/bench/data/bug-11913.php @@ -1,5 +1,11 @@ = 8.3 +/* + * instructions: 1.811B + * cycles: 2.245B + * memory: 58.08MiB + */ + namespace Bug12159; $func = static fn( int $Point ) : int => TestMatrix::Values[ $Point ][ 0 ]; diff --git a/tests/bench/data/bug-12671.php b/tests/bench/data/bug-12671.php index 80663164001..4139e86d1bc 100644 --- a/tests/bench/data/bug-12671.php +++ b/tests/bench/data/bug-12671.php @@ -1,5 +1,11 @@ = 8.0 +/* + * instructions: 350.9M + * cycles: 436.8M + * memory: 39.21MiB + */ + declare(strict_types = 1); namespace Bug13218; diff --git a/tests/bench/data/bug-13310.php b/tests/bench/data/bug-13310.php index f5c56bffdbf..5a498f9d605 100644 --- a/tests/bench/data/bug-13310.php +++ b/tests/bench/data/bug-13310.php @@ -1,5 +1,11 @@ = 8.0 +/* + * instructions: 12.87B + * cycles: 17.65B + * memory: 175.9MiB + */ + namespace Bug13352; use Exception; diff --git a/tests/bench/data/bug-13685.php b/tests/bench/data/bug-13685.php index 8679971ff4b..fc34cacab0b 100644 --- a/tests/bench/data/bug-13685.php +++ b/tests/bench/data/bug-13685.php @@ -1,5 +1,11 @@ = 8.0 +/* + * instructions: 338.9M + * cycles: 432.9M + * memory: 37.44MiB + */ + namespace BenchBug14319; function foo(string $a, int $b): array|object diff --git a/tests/bench/data/bug-14452.php b/tests/bench/data/bug-14452.php index 71a4852ccad..023b21535a2 100644 --- a/tests/bench/data/bug-14452.php +++ b/tests/bench/data/bug-14452.php @@ -1,5 +1,11 @@ } */ diff --git a/tests/bench/data/bug-1447.php b/tests/bench/data/bug-1447.php index 53cffc3366b..7039a547e9e 100644 --- a/tests/bench/data/bug-1447.php +++ b/tests/bench/data/bug-1447.php @@ -1,5 +1,11 @@ = 8.1 +/* + * instructions: 1.622B + * cycles: 1.942B + * memory: 52.35MiB + */ + namespace Bug14869Enum; /** diff --git a/tests/bench/data/bug-14869.php b/tests/bench/data/bug-14869.php index 6fe64a30559..a4038ce02f7 100644 --- a/tests/bench/data/bug-14869.php +++ b/tests/bench/data/bug-14869.php @@ -1,5 +1,11 @@ >> $pl */ diff --git a/tests/bench/data/bug-15591.php b/tests/bench/data/bug-15591.php index 60b6f3820f8..8c8a5753d32 100644 --- a/tests/bench/data/bug-15591.php +++ b/tests/bench/data/bug-15591.php @@ -1,5 +1,11 @@ = 8.0 +/* + * instructions: 27.17M + * cycles: 39.59M + * memory: 35.87MiB + */ + namespace Bug4308; class Test diff --git a/tests/bench/data/bug-5081.php b/tests/bench/data/bug-5081.php index 25c35dcc2b0..14030da9602 100644 --- a/tests/bench/data/bug-5081.php +++ b/tests/bench/data/bug-5081.php @@ -1,5 +1,11 @@ = 8.0 + +/* + * instructions: 53.47M + * cycles: 75.86M + * memory: 36.05MiB + */ declare(strict_types=1); namespace ConditionalExpressionInfiniteLoop; diff --git a/tests/bench/data/enum-case-union-removal.php b/tests/bench/data/enum-case-union-removal.php index 1d9d30c5ac9..2aabd21928f 100644 --- a/tests/bench/data/enum-case-union-removal.php +++ b/tests/bench/data/enum-case-union-removal.php @@ -1,5 +1,11 @@ = 8.1 +/* + * instructions: 1.923B + * cycles: 2.330B + * memory: 42.18MiB + */ + declare(strict_types = 1); namespace BenchEnumCaseUnionRemoval; diff --git a/tests/bench/data/enum-value-keyed-map.php b/tests/bench/data/enum-value-keyed-map.php index 58bf2ab0c0b..49319efc071 100644 --- a/tests/bench/data/enum-value-keyed-map.php +++ b/tests/bench/data/enum-value-keyed-map.php @@ -1,5 +1,11 @@ = 8.1 +/* + * instructions: 697.9M + * cycles: 877.6M + * memory: 53.48MiB + */ + namespace BenchEnumValueKeyedMap; enum Code: string diff --git a/tests/bench/data/finite-types-optional-keys-blowup.php b/tests/bench/data/finite-types-optional-keys-blowup.php index 7190155e5bc..5c672402da7 100644 --- a/tests/bench/data/finite-types-optional-keys-blowup.php +++ b/tests/bench/data/finite-types-optional-keys-blowup.php @@ -1,5 +1,11 @@ = 8.0 + +/* + * instructions: 3.496B + * cycles: 4.608B + * memory: 66.92MiB + */ declare(strict_types = 1); namespace BenchHashKeyLookup; diff --git a/tests/bench/data/implode-optional-keys-blowup.php b/tests/bench/data/implode-optional-keys-blowup.php index b8a3e122465..8d95ae08d10 100644 --- a/tests/bench/data/implode-optional-keys-blowup.php +++ b/tests/bench/data/implode-optional-keys-blowup.php @@ -1,5 +1,11 @@ = 8.0 +/* + * instructions: 3.264B + * cycles: 4.504B + * memory: 104.9MiB + */ + namespace ImpureCallColumnsBench; class Repro diff --git a/tests/bench/data/in-array-intersect-blowup.php b/tests/bench/data/in-array-intersect-blowup.php index 573673f59bd..ed69d503454 100644 --- a/tests/bench/data/in-array-intersect-blowup.php +++ b/tests/bench/data/in-array-intersect-blowup.php @@ -1,4 +1,10 @@ = 8.0 + +/* + * instructions: 149.4M + * cycles: 182.0M + * memory: 36.69MiB + */ declare(strict_types = 1); namespace BenchInArrayIntersectBlowup; diff --git a/tests/bench/data/literal-key-union-offset-write.php b/tests/bench/data/literal-key-union-offset-write.php index cbb8633dcda..bc233e4b9da 100644 --- a/tests/bench/data/literal-key-union-offset-write.php +++ b/tests/bench/data/literal-key-union-offset-write.php @@ -1,5 +1,11 @@ = 8.0 +/* + * instructions: 819.9M + * cycles: 1.051B + * memory: 39.43MiB + */ + namespace UnionFastPath; /** diff --git a/tests/bench/data/unrolled-foreach-list-guards.php b/tests/bench/data/unrolled-foreach-list-guards.php index 8abeb7e5145..91f637da575 100644 --- a/tests/bench/data/unrolled-foreach-list-guards.php +++ b/tests/bench/data/unrolled-foreach-list-guards.php @@ -1,5 +1,11 @@ = 8.0 +/* + * instructions: 2.781B + * cycles: 3.536B + * memory: 45.12MiB + */ + namespace BenchComplexUnion; class WP_User { diff --git a/tests/bench/expected.neon b/tests/bench/expected.neon deleted file mode 100644 index 10d0bc1afd9..00000000000 --- a/tests/bench/expected.neon +++ /dev/null @@ -1,483 +0,0 @@ -# Expected metrics of the benchmarks in data/, compared by compare.php with the tolerances in BenchmarkComparator. -# To update it, replace it with the benchmark-measured artifact of the Benchmarks workflow. - -and-chain-resolve-type-blowup.php: - instructions: 3118976850 - estimatedCycles: 3398267520 - peakMemory: 46647360 - -and-chain-truthy-blowup.php: - instructions: 260863142 - estimatedCycles: 339059372 - peakMemory: 41364296 - -array-values-literal-keyed-map.php: - instructions: 3331053894 - estimatedCycles: 3466340454 - peakMemory: 42788536 - -big-constant-int-union.php: - instructions: 398919563 - estimatedCycles: 464217973 - peakMemory: 42361376 - -big-constant-string-union-operators.php: - instructions: 2866282352 - estimatedCycles: 3572647042 - peakMemory: 47451696 - -big-constant-string-union.php: - instructions: 1710009691 - estimatedCycles: 1930846771 - peakMemory: 43002880 - -bug-10147.php: - instructions: 11277321 - estimatedCycles: 18112651 - peakMemory: 37541184 - -bug-10538.php: - instructions: 14438399977 - estimatedCycles: 17315768997 - peakMemory: 56158040 - -bug-10772.php: - instructions: 4344296988 - estimatedCycles: 5681586818 - peakMemory: 98774032 - -bug-10979.php: - instructions: 4080991526 - estimatedCycles: 5558220036 - peakMemory: 44427496 - -bug-11263.php: - instructions: 1927238062 - estimatedCycles: 2274701462 - peakMemory: 44164064 - -bug-11283.php: - instructions: 1208462202 - estimatedCycles: 1591071832 - peakMemory: 42106128 - -bug-11297.php: - instructions: 9283508750 - estimatedCycles: 10268936180 - peakMemory: 69061168 - -bug-11913.php: - instructions: 994141076 - estimatedCycles: 1273772136 - peakMemory: 55989984 - -bug-12159.php: - instructions: 1811347354 - estimatedCycles: 2245447024 - peakMemory: 60898040 - -bug-12671.php: - instructions: 4330116170 - estimatedCycles: 5280107900 - peakMemory: 69597048 - -bug-12787.php: - instructions: 44328171 - estimatedCycles: 65522761 - peakMemory: 38159776 - -bug-12800.php: - instructions: 669384996 - estimatedCycles: 798860606 - peakMemory: 41455352 - -bug-13218.php: - instructions: 350905536 - estimatedCycles: 436795786 - peakMemory: 41110480 - -bug-13310.php: - instructions: 60433683 - estimatedCycles: 85906553 - peakMemory: 38154424 - -bug-13352.php: - instructions: 12872149086 - estimatedCycles: 17645659486 - peakMemory: 184464688 - -bug-13685.php: - instructions: 142966791 - estimatedCycles: 194887211 - peakMemory: 38983816 - -bug-1388.php: - instructions: 317236770 - estimatedCycles: 412990560 - peakMemory: 40174600 - -bug-13933.php: - instructions: 1037981764 - estimatedCycles: 1320984934 - peakMemory: 52148848 - -bug-14207-and.php: - instructions: 279509672 - estimatedCycles: 356570762 - peakMemory: 41438336 - -bug-14207.php: - instructions: 447139867 - estimatedCycles: 571792877 - peakMemory: 43230776 - -bug-14319.php: - instructions: 338943118 - estimatedCycles: 432923978 - peakMemory: 39259168 - -bug-14452.php: - instructions: 154206190 - estimatedCycles: 206947750 - peakMemory: 40134776 - -bug-14462.php: - instructions: 949947181 - estimatedCycles: 1181533721 - peakMemory: 40252816 - -bug-1447.php: - instructions: 104045329 - estimatedCycles: 143087049 - peakMemory: 39005504 - -bug-14475.php: - instructions: 146363547 - estimatedCycles: 192493237 - peakMemory: 39084328 - -bug-14590.php: - instructions: 969347556 - estimatedCycles: 1251238886 - peakMemory: 38478272 - -bug-14624.php: - instructions: 5262333615 - estimatedCycles: 6425685605 - peakMemory: 53960128 - -bug-14674.php: - instructions: 7278531300 - estimatedCycles: 9257088560 - peakMemory: 45986928 - -bug-14869-enum.php: - instructions: 1622307286 - estimatedCycles: 1941780476 - peakMemory: 54889400 - -bug-14869.php: - instructions: 2012006872 - estimatedCycles: 2388282422 - peakMemory: 49264928 - -bug-14972-concat.php: - instructions: 7865041309 - estimatedCycles: 10957120539 - peakMemory: 185727192 - -bug-14972.php: - instructions: 9444093255 - estimatedCycles: 11141678195 - peakMemory: 142411224 - -bug-14996.php: - instructions: 9708256090 - estimatedCycles: 11949370430 - peakMemory: 46756008 - -bug-15061.php: - instructions: 3166173143 - estimatedCycles: 3596156503 - peakMemory: 41640520 - -bug-15305.php: - instructions: 3140000769 - estimatedCycles: 3927966729 - peakMemory: 39361248 - -bug-15348.php: - instructions: 8112253881 - estimatedCycles: 8825000791 - peakMemory: 47350112 - -bug-15452.php: - instructions: 1728727927 - estimatedCycles: 2306301017 - peakMemory: 69576560 - -bug-15498.php: - instructions: 646763456 - estimatedCycles: 817752296 - peakMemory: 39190880 - -bug-15591.php: - instructions: 42690903 - estimatedCycles: 63414123 - peakMemory: 37781672 - -bug-3686.php: - instructions: 60930098 - estimatedCycles: 86877308 - peakMemory: 38193264 - -bug-4300.php: - instructions: 26985267 - estimatedCycles: 41333207 - peakMemory: 37749288 - -bug-4308.php: - instructions: 27170602 - estimatedCycles: 39594262 - peakMemory: 37616880 - -bug-5081.php: - instructions: 4256909596 - estimatedCycles: 4965341716 - peakMemory: 117740424 - -bug-5231.php: - instructions: 401941781 - estimatedCycles: 506156601 - peakMemory: 41104336 - -bug-5231_2.php: - instructions: 138832280 - estimatedCycles: 191315080 - peakMemory: 39292544 - -bug-5390.php: - instructions: 20090450 - estimatedCycles: 31853290 - peakMemory: 38053992 - -bug-6265.php: - instructions: 630251088 - estimatedCycles: 828945918 - peakMemory: 41330184 - -bug-6442.php: - instructions: 36170708 - estimatedCycles: 53168128 - peakMemory: 38640352 - -bug-6936.php: - instructions: 227610761 - estimatedCycles: 300896961 - peakMemory: 40375368 - -bug-6948.php: - instructions: 222515067 - estimatedCycles: 282956707 - peakMemory: 38824392 - -bug-7140.php: - instructions: 154332859 - estimatedCycles: 209441949 - peakMemory: 38925648 - -bug-7214.php: - instructions: 52175358 - estimatedCycles: 73222238 - peakMemory: 38818224 - -bug-7581.php: - instructions: 3168460718 - estimatedCycles: 3768865088 - peakMemory: 46579680 - -bug-7637.php: - instructions: 189179088 - estimatedCycles: 238995378 - peakMemory: 39753152 - -bug-7901.php: - instructions: 5142568969 - estimatedCycles: 6707120249 - peakMemory: 93531400 - -bug-7903.php: - instructions: 5763025228 - estimatedCycles: 7269732218 - peakMemory: 51449568 - -bug-8146a.php: - instructions: 883700057 - estimatedCycles: 1128401907 - peakMemory: 41573968 - -bug-8146b.php: - instructions: 6458828961 - estimatedCycles: 8608803391 - peakMemory: 119253800 - -bug-8147.php: - instructions: 2951209974 - estimatedCycles: 3897899204 - peakMemory: 71177488 - -bug-8215.php: - instructions: 5619919565 - estimatedCycles: 7463578535 - peakMemory: 136605448 - -bug-8503.php: - instructions: 972485165 - estimatedCycles: 1258370605 - peakMemory: 45581368 - -bug-9690.php: - instructions: 396856020 - estimatedCycles: 522778460 - peakMemory: 41089904 - -callable-union-describe.php: - instructions: 19982351354 - estimatedCycles: 24745356204 - peakMemory: 64848832 - -closure-variable-invocations.php: - instructions: 2865101528 - estimatedCycles: 3920387128 - peakMemory: 76746696 - -conditional-expression-infinite-loop.php: - instructions: 53473120 - estimatedCycles: 75860390 - peakMemory: 37796328 - -enum-case-union-removal.php: - instructions: 1922936607 - estimatedCycles: 2329574177 - peakMemory: 44226224 - -enum-value-keyed-map.php: - instructions: 697896028 - estimatedCycles: 877574558 - peakMemory: 56075608 - -finite-types-optional-keys-blowup.php: - instructions: 37389089 - estimatedCycles: 49226249 - peakMemory: 37662336 - -finite-types-optional-keys-count.php: - instructions: 4335905622 - estimatedCycles: 5332033992 - peakMemory: 81122592 - -flatten-types-optional-keys-blowup.php: - instructions: 33765874 - estimatedCycles: 45920344 - peakMemory: 37770144 - -hash-key-lookup.php: - instructions: 3496481720 - estimatedCycles: 4608404080 - peakMemory: 70175336 - -implode-optional-keys-blowup.php: - instructions: 36435800 - estimatedCycles: 50220030 - peakMemory: 37806992 - -impure-call-columns.php: - instructions: 3264129324 - estimatedCycles: 4504091164 - peakMemory: 110020192 - -in-array-intersect-blowup.php: - instructions: 149372943 - estimatedCycles: 182045283 - peakMemory: 38474472 - -literal-key-union-offset-write.php: - instructions: 10710007650 - estimatedCycles: 14647147890 - peakMemory: 227556976 - -long-statement-list.php: - instructions: 4204046727 - estimatedCycles: 5751714957 - peakMemory: 73423064 - -loop-widening-optional-keys.php: - instructions: 1102199915 - estimatedCycles: 1423094205 - peakMemory: 43355248 - -loose-string-comparison.php: - instructions: 10221646438 - estimatedCycles: 12245021718 - peakMemory: 90156816 - -nested-context-snapshots.php: - instructions: 6644393415 - estimatedCycles: 8373080615 - peakMemory: 63016312 - -nested-loops-wrapping-array.php: - instructions: 1671558123 - estimatedCycles: 2088059083 - peakMemory: 42935856 - -nullsafe-chain-walk.php: - instructions: 18144239401 - estimatedCycles: 22755782431 - peakMemory: 103660256 - -or-chain-falsey-blowup.php: - instructions: 247645613 - estimatedCycles: 313978833 - peakMemory: 41573008 - -or-chain-resolve-type-blowup.php: - instructions: 3083958002 - estimatedCycles: 3354453502 - peakMemory: 45895032 - -phpdoc-inheritance-speedup.php: - instructions: 8887388700 - estimatedCycles: 9937094400 - peakMemory: 50524152 - -phpdoc-intersection-blowup.php: - instructions: 49360480 - estimatedCycles: 65471590 - peakMemory: 37794704 - -process-called-method-infinite-loop.php: - instructions: 367171013 - estimatedCycles: 460954543 - peakMemory: 41350104 - -sealed-shape-if-chain.php: - instructions: 1630694890 - estimatedCycles: 1979409200 - peakMemory: 61600112 - -union-fast-path.php: - instructions: 819895295 - estimatedCycles: 1051466255 - peakMemory: 41342824 - -unrolled-foreach-list-guards.php: - instructions: 1689884152 - estimatedCycles: 2013112302 - peakMemory: 41587880 - -wordpress-user.php: - instructions: 2781089351 - estimatedCycles: 3536387361 - peakMemory: 47310656 - diff --git a/tests/bench/measure.php b/tests/bench/measure.php deleted file mode 100644 index 08e7afb774e..00000000000 --- a/tests/bench/measure.php +++ /dev/null @@ -1,138 +0,0 @@ - - * - * The INSTRUMENT_HOOKS_LIBRARY environment variable points to CodSpeed's instrument-hooks - * library (https://github.com/CodSpeedHQ/instrument-hooks). Without it, the analysis runs - * unmeasured, which run.php uses to fill the caches in tmpDir before the measured run. - * - * The peak memory is the highest memory usage during the benchmark's analysis, including what - * the booted container and the warm-up keep allocated. - */ - -use FFI; -use PHPStan\Analyser\Analyser; -use PHPStan\Analyser\AnalyserResultFinalizer; -use PHPStan\Command\CommandHelper; -use Symfony\Component\Console\Input\ArrayInput; -use Symfony\Component\Console\Output\StreamOutput; -use function basename; -use function count; -use function dirname; -use function error_reporting; -use function file_put_contents; -use function fwrite; -use function gc_disable; -use function getenv; -use function getmypid; -use function memory_get_peak_usage; -use function memory_reset_peak_usage; -use function realpath; -use function sprintf; -use const E_ALL; -use const E_DEPRECATED; -use const STDERR; - -// Like bin/phpstan -error_reporting(E_ALL & ~E_DEPRECATED); -gc_disable(); - -$rootDir = dirname(__DIR__, 2); -require_once $rootDir . '/vendor/autoload.php'; - -if (!isset($argv) || count($argv) !== 4) { - fwrite(STDERR, "Usage: php -d ffi.enable=1 tests/bench/measure.php \n"); - exit(2); -} - -[, $configFile, $benchmarkFile, $peakMemoryFile] = $argv; -$benchmarkFile = realpath($benchmarkFile); -$warmupFile = __DIR__ . '/Warmup.php'; -if ($benchmarkFile === false) { - fwrite(STDERR, "The benchmark file does not exist.\n"); - exit(2); -} - -$hooks = null; -$hooksLibrary = getenv('INSTRUMENT_HOOKS_LIBRARY'); -if ($hooksLibrary !== false && $hooksLibrary !== '') { - $hooks = FFI::cdef(' - typedef struct InstrumentHooks InstrumentHooks; - InstrumentHooks *instrument_hooks_init(void); - bool instrument_hooks_is_instrumented(InstrumentHooks *); - uint8_t instrument_hooks_start_benchmark(InstrumentHooks *); - uint8_t instrument_hooks_stop_benchmark(InstrumentHooks *); - uint8_t instrument_hooks_set_executed_benchmark(InstrumentHooks *, int32_t pid, const char *uri); - uint8_t instrument_hooks_set_integration(InstrumentHooks *, const char *name, const char *version); - ', $hooksLibrary); -} - -$output = new StreamOutput(STDERR); -$inceptionResult = CommandHelper::begin( - new ArrayInput([]), - $output, - [$warmupFile, $benchmarkFile], - '-1', - null, - [$rootDir], - $configFile, - null, - null, - false, - true, - null, - null, - false, -); -// Also tells the parser which files are analysed, the others are parsed without function bodies -$inceptionResult->getFiles(); -$container = $inceptionResult->getContainer(); -$analyser = $container->getByType(Analyser::class); -$finalizer = $container->getByType(AnalyserResultFinalizer::class); - -$analyse = static function (string $file) use ($analyser, $finalizer): int { - $result = $analyser->analyse([$file], null, null, true, [$file]); - - return count($finalizer->finalize($result, false, true)->getErrors()); -}; - -// A different file than the benchmark: PHPStan memoizes types and reflection in memory, -// so analysing the benchmark twice would hide work from the measured run -$analyse($warmupFile); - -$instance = null; -if ($hooks !== null) { - $instance = $hooks->instrument_hooks_init(); - if ($instance === null || !$hooks->instrument_hooks_is_instrumented($instance)) { - fwrite(STDERR, "Not running under CodSpeed's CPU simulation.\n"); - exit(1); - } - $hooks->instrument_hooks_set_integration($instance, 'phpstan-bench', '1.0.0'); -} - -memory_reset_peak_usage(); -if ($hooks !== null && $hooks->instrument_hooks_start_benchmark($instance) !== 0) { - fwrite(STDERR, "Cannot start the measurement.\n"); - exit(1); -} - -$errors = $analyse($benchmarkFile); - -if ($hooks !== null) { - if ($hooks->instrument_hooks_stop_benchmark($instance) !== 0) { - fwrite(STDERR, "Cannot stop the measurement.\n"); - exit(1); - } - // run.php finds the measured part of the profile by this name - $hooks->instrument_hooks_set_executed_benchmark($instance, getmypid(), 'tests/bench/data/' . basename($benchmarkFile)); -} - -file_put_contents($peakMemoryFile, (string) memory_get_peak_usage()); -fwrite(STDERR, sprintf("Analysed %s, %d errors.\n", $benchmarkFile, $errors)); diff --git a/tests/bench/run.php b/tests/bench/run.php deleted file mode 100644 index 084fcfe212c..00000000000 --- a/tests/bench/run.php +++ /dev/null @@ -1,153 +0,0 @@ - - */ - -use Nette\Neon\Neon; -use Nette\Utils\FileSystem; -use PHPStan\Process\Restart\OpcacheSettings; -use Symfony\Component\Process\Exception\ProcessTimedOutException; -use Symfony\Component\Process\Process; -use function basename; -use function count; -use function dirname; -use function file_get_contents; -use function file_put_contents; -use function fwrite; -use function getenv; -use function getmypid; -use function glob; -use function is_dir; -use function is_file; -use function mkdir; -use function realpath; -use function rmdir; -use function sprintf; -use function str_ends_with; -use function sys_get_temp_dir; -use function unlink; -use const PHP_BINARY; -use const STDERR; - -require_once __DIR__ . '/../../vendor/autoload.php'; - -// The slowest benchmark takes a few seconds when PHPStan runs natively, so a benchmark -// stuck in an infinite loop fails in the native run, before the much slower simulation -const NATIVE_RUN_SECONDS_LIMIT = 30; - -// The slowest benchmark takes about 13 minutes under CodSpeed's CPU simulation. A benchmark -// that regresses far beyond that fails as not measured instead of holding up its shard. -const SIMULATION_SECONDS_LIMIT = 25 * 60; - -if (!isset($argv) || count($argv) !== 3) { - fwrite(STDERR, "Usage: php tests/bench/run.php \n"); - exit(2); -} - -[, $benchmarkFile, $outputFile] = $argv; -$benchmark = basename($benchmarkFile); -$benchmarkFile = realpath($benchmarkFile); -if ($benchmarkFile === false) { - fwrite(STDERR, sprintf("Benchmark %s does not exist.\n", $argv[1])); - exit(2); -} - -// Each benchmark gets its own tmpDir, so what earlier benchmarks left in the cache does not change its numbers -$tmpDir = dirname(__DIR__, 2) . '/tmp/bench/' . $benchmark; -FileSystem::delete($tmpDir); -FileSystem::createDir($tmpDir); -$configFile = $tmpDir . '/phpstan.neon'; -FileSystem::write($configFile, Neon::encode([ - 'includes' => [__DIR__ . '/phpstan.neon'], - 'parameters' => ['tmpDir' => $tmpDir . '/cache'], -], true)); - -// measure.php does not restart PHP like bin/phpstan, so it gets the OPcache settings the restart -// would add. Valgrind also breaks PHP's detection of the stack size, so the stack limit check is -// turned off. -$command = [PHP_BINARY, '-d', 'ffi.enable=1', '-d', 'memory_limit=-1', '-d', 'zend.max_allowed_stack_size=-1']; -foreach (OpcacheSettings::getIniEntries() as $iniEntry) { - $command[] = '-d'; - $command[] = $iniEntry; -} -$peakMemoryFile = $tmpDir . '/peak-memory'; -$command = [...$command, 'tests/bench/measure.php', $configFile, $benchmarkFile, $peakMemoryFile]; - -$hooksLibrary = getenv('INSTRUMENT_HOOKS_LIBRARY'); -if ($hooksLibrary === false || !is_file($hooksLibrary)) { - fwrite(STDERR, "Set INSTRUMENT_HOOKS_LIBRARY to the path of CodSpeed's instrument-hooks library.\n"); - exit(2); -} - -// Compiles the DI container and fills the caches in tmpDir, so the measured run reads them -$warmUp = new Process($command, __DIR__ . '/../..', ['INSTRUMENT_HOOKS_LIBRARY' => false]); -$warmUp->setTimeout(NATIVE_RUN_SECONDS_LIMIT); -try { - $warmUp->run(); -} catch (ProcessTimedOutException) { - fwrite(STDERR, sprintf("PHPStan did not finish in %d seconds.\n", NATIVE_RUN_SECONDS_LIMIT)); - exit(1); -} -if (!$warmUp->isSuccessful()) { - fwrite(STDERR, $warmUp->getOutput() . $warmUp->getErrorOutput()); - fwrite(STDERR, sprintf("PHPStan failed with exit code %s.\n", $warmUp->getExitCode())); - exit(1); -} - -$profileDir = sys_get_temp_dir() . '/phpstan-benchmark-' . getmypid(); -if (!is_dir($profileDir) && !mkdir($profileDir, 0777, true)) { - fwrite(STDERR, sprintf("Cannot create %s.\n", $profileDir)); - exit(1); -} - -$codspeed = new Process([ - 'codspeed', - 'run', - '--mode=simulation', - // The comparison against expected.neon happens in compare.php - '--skip-upload', - '--profile-folder=' . $profileDir, - '--', - ...$command, -], __DIR__ . '/../..', ['INSTRUMENT_HOOKS_LIBRARY' => $hooksLibrary], timeout: SIMULATION_SECONDS_LIMIT); -try { - $codspeed->run(); -} catch (ProcessTimedOutException) { - fwrite(STDERR, sprintf("CodSpeed did not finish in %d minutes.\n", SIMULATION_SECONDS_LIMIT / 60)); - exit(1); -} - -// Every process gets its own file, only the one of measure.php has the measured part -$metrics = []; -foreach (glob($profileDir . '/*') ?: [] as $profileFile) { - if (str_ends_with($profileFile, '.out')) { - $profile = BenchmarkProfileParser::parseCallgrindOutput((string) file_get_contents($profileFile), 'tests/bench/data/' . $benchmark); - if ($profile !== null) { - $metrics[] = $profile; - } - } - unlink($profileFile); -} -rmdir($profileDir); - -$peakMemory = @file_get_contents($peakMemoryFile); -if (!$codspeed->isSuccessful() || count($metrics) !== 1 || $peakMemory === false) { - fwrite(STDERR, $codspeed->getOutput() . $codspeed->getErrorOutput()); - fwrite(STDERR, sprintf("CodSpeed exited with code %s and measured %d benchmarks instead of one.\n", $codspeed->getExitCode(), count($metrics))); - exit(1); -} - -$result = [$benchmark => $metrics[0] + ['peakMemory' => (int) $peakMemory]]; -if (file_put_contents($outputFile, Neon::encode($result, true)) === false) { - fwrite(STDERR, sprintf("Cannot write %s.\n", $outputFile)); - exit(1); -} - -echo Neon::encode($result, true); diff --git a/tests/bench/shard.php b/tests/bench/shard.php deleted file mode 100644 index 50cef127df0..00000000000 --- a/tests/bench/shard.php +++ /dev/null @@ -1,45 +0,0 @@ - - */ - -use Nette\Neon\Neon; -use function basename; -use function count; -use function file_get_contents; -use function fwrite; -use function glob; -use function is_file; -use function max; -use const STDERR; - -require_once __DIR__ . '/../../vendor/autoload.php'; - -if (!isset($argv) || count($argv) !== 3 || (int) $argv[1] < 1 || (int) $argv[2] < 1 || (int) $argv[2] > (int) $argv[1]) { - fwrite(STDERR, "Usage: php tests/bench/shard.php \n"); - exit(2); -} - -$expectedFile = __DIR__ . '/expected.neon'; -$expected = is_file($expectedFile) ? Neon::decode((string) file_get_contents($expectedFile)) : []; -$expectedInstructions = []; -foreach ($expected as $benchmark => $metrics) { - $expectedInstructions[$benchmark] = $metrics['instructions']; -} - -$costs = []; -foreach (glob(__DIR__ . '/data/*.php') ?: [] as $file) { - $benchmark = basename($file); - // a benchmark without expected metrics yet is usually a reproducer of something slow - $costs[$benchmark] = $expectedInstructions[$benchmark] ?? ($expectedInstructions === [] ? 1 : max($expectedInstructions)); -} - -foreach (BenchmarkShards::split($costs, (int) $argv[1])[(int) $argv[2] - 1] ?? [] as $benchmark) { - echo $benchmark . "\n"; -} From bc279941211742242fd73c97c468b57997d4461f Mon Sep 17 00:00:00 2001 From: "ondrej@mirtes.cz" Date: Fri, 13 Nov 2026 22:51:51 +0000 Subject: [PATCH 2/2] Keep the benchmarks out of lint and analysis on PHP < 8.1 PHP 7.4 reads #[ as a comment until the end of the line, so the attribute of AnalysisBench is on one line. Tally's sources use enums and readonly properties, which PHPStan does not parse for PHP < 8.1, so the benchmarks are not analysed there. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d525-9c4c-73cf-8edc-000aa8e78378 --- build/ignore-by-php-version.neon.php | 1 + build/tally-pre-8.1.neon | 7 +++++++ tests/bench/AnalysisBench.php | 9 +++------ 3 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 build/tally-pre-8.1.neon diff --git a/build/ignore-by-php-version.neon.php b/build/ignore-by-php-version.neon.php index debc6ba56c9..8ea017fa594 100644 --- a/build/ignore-by-php-version.neon.php +++ b/build/ignore-by-php-version.neon.php @@ -10,6 +10,7 @@ $includes[] = __DIR__ . '/enums.neon'; $includes[] = __DIR__ . '/readonly-property.neon'; $includes[] = __DIR__ . '/turbo-declaration-generator-pre-8.1.neon'; + $includes[] = __DIR__ . '/tally-pre-8.1.neon'; } // PHP 8.1's serialize() does not trim its buffer, so the rule's cache keys cost 4 KB each diff --git a/build/tally-pre-8.1.neon b/build/tally-pre-8.1.neon new file mode 100644 index 00000000000..e1e01e4f694 --- /dev/null +++ b/build/tally-pre-8.1.neon @@ -0,0 +1,7 @@ +parameters: + excludePaths: + # Tally's sources use enums and readonly properties, which PHPStan does not parse for PHP < 8.1 + - ../tally.php + - ../tests/bench/AnalysedFileExpectations.php + - ../tests/bench/AnalysedFileHeader.php + - ../tests/bench/AnalysisBench.php diff --git a/tests/bench/AnalysisBench.php b/tests/bench/AnalysisBench.php index 565da51eedf..668fb54f323 100644 --- a/tests/bench/AnalysisBench.php +++ b/tests/bench/AnalysisBench.php @@ -43,13 +43,10 @@ * is converted to milliseconds by how many of it the analysis runs per millisecond (the median * over the benchmarks, natively on PHP 8.5 and an Intel Xeon @ 2.60 GHz), and the quicker the * benchmark, the larger the share of it that may change. + * + * The attribute is on one line because PHP 7.4 reads #[ as a comment until the end of the line. */ -#[DurationTolerance( - tolerances: [0 => 0.5, 100 => 0.25, 500 => 0.2, 2000 => 0.1], - instructionsPerMillisecond: 5_800_000, - cyclesPerMillisecond: 7_400_000, - memory: 0.1, -)] +#[DurationTolerance(tolerances: [0 => 0.5, 100 => 0.25, 500 => 0.2, 2000 => 0.1], instructionsPerMillisecond: 5_800_000, cyclesPerMillisecond: 7_400_000, memory: 0.1)] final class AnalysisBench {