From 39307501debb09dbf86494ba70aff2912aface24 Mon Sep 17 00:00:00 2001 From: Dave Ventesca Date: Thu, 20 Aug 2026 11:28:10 -0700 Subject: [PATCH 1/3] microchaos: add process-fanout --concurrency and release 4.2.0 --burst stays sequential pacing. --concurrency=N launches N loadtest processes so requests overlap on Pressable; combined Throughput is not a Phase 4 RPS. --- tools/microchaos-cli/README.md | 44 +- tools/microchaos-cli/build.js | 1 + tools/microchaos-cli/dist/microchaos-cli.php | 562 +++++++++++++++++- tools/microchaos-cli/microchaos-cli.php | 2 +- tools/microchaos-cli/microchaos/bootstrap.php | 1 + .../microchaos/core/commands.php | 38 +- .../microchaos/core/concurrency-runner.php | 469 +++++++++++++++ .../microchaos/core/constants.php | 11 + .../orchestrators/loadtest-orchestrator.php | 58 +- .../tests/Unit/ConcurrencyRunnerTest.php | 87 +++ .../tests/Unit/ConstantsTest.php | 7 + .../tests/Unit/LoadTestOrchestratorTest.php | 9 + tools/microchaos-cli/tests/bootstrap.php | 3 + 13 files changed, 1260 insertions(+), 32 deletions(-) create mode 100644 tools/microchaos-cli/microchaos/core/concurrency-runner.php create mode 100644 tools/microchaos-cli/tests/Unit/ConcurrencyRunnerTest.php diff --git a/tools/microchaos-cli/README.md b/tools/microchaos-cli/README.md index 17f03bc..80b74ed 100644 --- a/tools/microchaos-cli/README.md +++ b/tools/microchaos-cli/README.md @@ -1,6 +1,6 @@ # ⚡️ MicroChaos CLI Load Tester -v4.1.0 — "Ground Truth" +v4.2.0 — "Overlap" Welcome to **MicroChaos**—a precision-built WP-CLI load testing tool forged in the fires of real-world WordPress hosting constraints. @@ -23,6 +23,26 @@ Built for staging environments like **Pressable**, MicroChaos simulates traffic --- +## 🆕 What's New in v4.2.0 "Overlap" + +v4.1.0 told the truth about what one process can measure: `--burst` is pacing, +and a single WP-CLI process fires requests one at a time. That is still true, +and it is still the sizing input. + +v4.2.0 adds the overlap run that v3 removed for the wrong mechanism. The old +`--concurrency` was curl_multi inside one process; Pressable loopback serializes +that. The new `--concurrency=N` launches N sequential `loadtest` processes at +the same instant — the method that produced real overlap on Pressable (cache +stampede races, HIT/regen splits on an already-warm URL). + +- **`--concurrency=N`** (default 1, cap 8). `--count` and `--duration` are per + process. `--count=1 --concurrency=4` is a four-request stampede. +- **`--results-json=`** so workers can hand results back without + interleaving summaries. The parent prints one merged report. +- Combined Throughput from an overlap run is **not** a Phase 4 RPS. Size on a + sequential `--cache-bust` run. The summary says so. +- Sequential default is unchanged. Existing 4.1.0 commands keep their numbers. + ## 🆕 What's New in v4.1.0 "Ground Truth" v4.1.0 is a correctness release. Every change below fixes a case where MicroChaos @@ -108,11 +128,14 @@ v3.0.0 is a **radical simplification**. We removed features that didn't work rel These features were removed because they don't work on Pressable due to loopback rate limiting: -- ❌ `wp microchaos parallel` - Parallel test execution +- ❌ `wp microchaos parallel` - Parallel test execution (one-process curl_multi) - ❌ `wp microchaos progressive` - Progressive load testing -- ❌ `--concurrency` flag - True concurrent requests +- ❌ `--concurrency` flag as curl_multi-in-one-process (did not overlap on Pressable) -If you need these features, use an external load testing tool (k6, Artillery, etc.) that can hit your site from outside. +v4.2.0 brings `--concurrency` back as **process fan-out**, which is a different +mechanism and the one that actually overlaps on this platform. External tools +(k6, Artillery) remain the right choice for hundreds of VUs from outside the +container. --- @@ -128,7 +151,7 @@ If you need these features, use an external load testing tool (k6, Artillery, et ## 🚨 Platform Considerations (Pressable & Managed Hosts) -MicroChaos is built specifically for **Pressable** and similar managed WordPress hosts where loopback requests are rate-limited (~10 concurrent max). The tool uses **serial execution** which works perfectly within these constraints. +MicroChaos is built specifically for **Pressable** and similar managed WordPress hosts where loopback requests are rate-limited (~10 concurrent max). The default path is still **serial execution** inside one process. `--concurrency=N` overlaps by launching N of those serial processes; stay at or under 8, and treat 4 as the starting point on a 5-worker staging box. ### Understanding the `--burst` Flag @@ -608,8 +631,10 @@ wp microchaos loadtest --endpoint=checkout --count=50 --auth=admin@example.com - - `--endpoints=` Comma-separated list of endpoints to rotate through - `--count=` Total requests to send (default: 100) - `--duration=` Run test for specified duration instead of fixed request count -- `--burst=` Requests per burst (default: 10) +- `--burst=` Requests per burst (default: 10). Pacing, not concurrency. - `--delay=` Delay between bursts (default: 2) +- `--concurrency=` Sequential processes launched together (default: 1, cap: 8). `--count` and `--duration` are per process. +- `--results-json=` Write machine-readable results to a JSON file ### Request Configuration @@ -658,6 +683,13 @@ Load test the homepage with cache warmup and log output wp microchaos loadtest --endpoint=home --count=100 --warm-cache --log-to=uploads/home-log.txt ``` +Four-request stampede (one request per process, launched together). Combined Throughput is not a sizing RPS. + +```bash +wp microchaos loadtest --endpoint=custom:/fresh-page/ --count=1 --cache-bust --concurrency=4 +wp microchaos loadtest --endpoint=home --count=1 --warm-cache --cache-headers --concurrency=4 +``` + Test multiple endpoints with random rotation ```bash diff --git a/tools/microchaos-cli/build.js b/tools/microchaos-cli/build.js index 39b59c1..c55931e 100755 --- a/tools/microchaos-cli/build.js +++ b/tools/microchaos-cli/build.js @@ -39,6 +39,7 @@ const sources = { path.join(__dirname, "microchaos/core/reporting-engine.php"), // Load orchestrators (before commands which depends on them) path.join(__dirname, "microchaos/core/orchestrators/loadtest-orchestrator.php"), + path.join(__dirname, "microchaos/core/concurrency-runner.php"), // Load commands last (depends on orchestrators) path.join(__dirname, "microchaos/core/commands.php"), ], diff --git a/tools/microchaos-cli/dist/microchaos-cli.php b/tools/microchaos-cli/dist/microchaos-cli.php index 95fde90..14118ef 100644 --- a/tools/microchaos-cli/dist/microchaos-cli.php +++ b/tools/microchaos-cli/dist/microchaos-cli.php @@ -3,7 +3,7 @@ * Plugin Name: MicroChaos CLI Load Tester * Description: Internal WP-CLI based WordPress load tester for staging environments where * external load testing is restricted (like Pressable). - * Version: 4.1.0 + * Version: 4.2.0 * Author: Phill * License: GPL-3.0-or-later * @@ -15,7 +15,7 @@ // Bootstrap MicroChaos components /** - * COMPILED SINGLE-FILE VERSION - MicroChaos 4.1.0 + * COMPILED SINGLE-FILE VERSION - MicroChaos 4.2.0 * * This is an automatically generated file - DO NOT EDIT DIRECTLY * Make changes to the modular version and rebuild with: node build.js @@ -24,7 +24,7 @@ if (defined('WP_CLI') && WP_CLI) { if (!defined('MICROCHAOS_VERSION')) { - define('MICROCHAOS_VERSION', '4.1.0'); + define('MICROCHAOS_VERSION', '4.2.0'); } class MicroChaos_Constants { @@ -53,6 +53,17 @@ class MicroChaos_Constants { */ const DEFAULT_WORKERS = 3; + /** + * Default --concurrency. 1 is the sequential path; nothing fans out. + */ + const DEFAULT_CONCURRENCY = 1; + + /** + * Hard cap on --concurrency. Staging boxes are 5 workers; past 8 the + * generators themselves start to be the thing under test. + */ + const MAX_CONCURRENCY = 8; + /** * Time conversion constants */ @@ -3002,6 +3013,9 @@ private function normalize_config(array $config): array { 'save_baseline' => null, 'compare_baseline' => null, 'log_path' => null, + 'concurrency' => MicroChaos_Constants::DEFAULT_CONCURRENCY, + 'results_json' => null, + 'worker_id' => null, ], $config); } @@ -3156,12 +3170,21 @@ public function execute(): array { ); } - // Display reports - $reporting_engine->report_summary($perf_baseline, null, $use_thresholds, $execution_metrics); + if (!empty($config['results_json'])) { + $this->write_results_json($config['results_json'], $reporting_engine, $loop_result, $cache_analyzer); + } - $this->report_timeout_censoring($reporting_engine, $request_generator->get_timeout()); + // A worker writing results-json is collected by the parent; printing + // a second summary here would interleave with its siblings. + if (empty($config['results_json'])) { + $reporting_engine->report_summary($perf_baseline, null, $use_thresholds, $execution_metrics); + } - if ($config['resource_logging']) { + if (empty($config['results_json'])) { + $this->report_timeout_censoring($reporting_engine, $request_generator->get_timeout()); + } + + if ($config['resource_logging'] && empty($config['results_json'])) { $resource_monitor->report_summary($resource_baseline, null, $use_thresholds); if ($config['resource_trends']) { @@ -3170,7 +3193,7 @@ public function execute(): array { } // Save baseline if specified - if ($config['save_baseline']) { + if ($config['save_baseline'] && empty($config['results_json'])) { $reporting_engine->save_baseline($config['save_baseline']); if ($config['resource_logging']) { $resource_monitor->save_baseline($config['save_baseline']); @@ -3179,7 +3202,7 @@ public function execute(): array { } // Report cache headers if enabled - if ($config['collect_cache_headers']) { + if ($config['collect_cache_headers'] && empty($config['results_json'])) { $cache_analyzer->report_summary($reporting_engine->get_request_count()); } @@ -3205,6 +3228,40 @@ public function execute(): array { ]; } + /** + * Write this process's raw results for a parent overlap run to merge. + * + * @param string $path Absolute path + * @param MicroChaos_Reporting_Engine $reporting_engine + * @param array $loop_result + * @param MicroChaos_Cache_Analyzer $cache_analyzer + */ + private function write_results_json(string $path, MicroChaos_Reporting_Engine $reporting_engine, array $loop_result, MicroChaos_Cache_Analyzer $cache_analyzer): void { + $payload = [ + 'worker_id' => $this->config['worker_id'], + 'results' => $reporting_engine->get_results(), + 'completed' => $loop_result['completed'], + 'run_by_duration' => $loop_result['run_by_duration'], + 'test_start_timestamp' => $loop_result['test_start_timestamp'], + 'test_end_timestamp' => $loop_result['test_end_timestamp'], + 'cache_headers' => $cache_analyzer->get_cache_headers(), + ]; + + $dir = dirname($path); + if (!is_dir($dir) && !@mkdir($dir, 0700, true) && !is_dir($dir)) { + MicroChaos_Log::error("Could not write results JSON directory: {$dir}"); + } + + $json = function_exists('wp_json_encode') + ? wp_json_encode($payload, JSON_UNESCAPED_SLASHES) + : json_encode($payload, JSON_UNESCAPED_SLASHES); + + $written = @file_put_contents($path, $json); + if ($written === false) { + MicroChaos_Log::error("Could not write results JSON: {$path}"); + } + } + /** * Warn that the timing distribution is right-censored, if any request timed out * @@ -3684,6 +3741,455 @@ private function format_duration(float $seconds): string { } } +class MicroChaos_Concurrency_Runner { + + /** + * Clamp a requested process count to the supported range. + * + * @param int $requested Requested process count + * @return int Clamped count, at least 1 and at most MAX_CONCURRENCY + */ + public static function clamp(int $requested): int { + if ($requested < 1) { + return 1; + } + + if ($requested > MicroChaos_Constants::MAX_CONCURRENCY) { + return MicroChaos_Constants::MAX_CONCURRENCY; + } + + return $requested; + } + + /** + * Build the assoc args one child process should receive. + * + * Forces concurrency=1 so a child cannot fan out again. Points the child + * at a results JSON file so the parent can merge without reading interleaved + * stdout. Drops warm-cache: the parent warms once, then the children hit. + * + * @param array $assoc_args Parent CLI args + * @param int $worker_id 1-based worker id + * @param string $results_json Absolute path the child should write + * @return array Child assoc args + */ + public static function child_assoc_args(array $assoc_args, int $worker_id, string $results_json): array { + $child = $assoc_args; + $child['concurrency'] = 1; + $child['results-json'] = $results_json; + $child['worker-id'] = $worker_id; + unset($child['warm-cache']); + + return $child; + } + + /** + * Format assoc args as a shell-safe WP-CLI flag string. + * + * @param array $assoc_args Flags to format + * @return string Leading-space flag string, or empty + */ + public static function format_assoc_args(array $assoc_args): string { + $parts = []; + + foreach ($assoc_args as $key => $value) { + if ($value === false || $value === null) { + continue; + } + if ($value === true) { + $parts[] = '--' . $key; + continue; + } + $parts[] = '--' . $key . '=' . escapeshellarg((string) $value); + } + + return $parts === [] ? '' : ' ' . implode(' ', $parts); + } + + /** + * Prefix used to re-invoke WP-CLI as a child. + * + * Prefers the current argv[0] so a `wp` wrapper stays a `wp` wrapper. + * Falls back to `wp` when argv is missing (tests). + * + * @return string Shell-escaped command prefix, no trailing space + */ + public static function command_prefix(): string { + $invoker = $GLOBALS['argv'][0] ?? 'wp'; + + if (self::invoker_is_php_script($invoker)) { + return escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg($invoker); + } + + return escapeshellarg($invoker); + } + + /** + * Whether the invoker is a PHP script that must be launched via PHP_BINARY. + * + * @param string $invoker Path from argv[0] + * @return bool + */ + public static function invoker_is_php_script(string $invoker): bool { + $base = strtolower(basename($invoker)); + + return str_ends_with($base, '.php') || str_ends_with($base, '.phar'); + } + + /** + * Fan out, wait, merge, and report. Returns the same shape as execute(). + * + * @param array $assoc_args Parent CLI args + * @param array $config Parsed config + * @return array{completed: int, count: int, run_by_duration: bool, actual_minutes: float} + */ + public function run(array $assoc_args, array $config): array { + $n = self::clamp((int) $config['concurrency']); + + if ($n !== (int) $config['concurrency']) { + MicroChaos_Log::warning( + "⚠️ --concurrency={$config['concurrency']} exceeds the cap of " + . MicroChaos_Constants::MAX_CONCURRENCY + . "; running {$n} processes." + ); + } + + MicroChaos_Log::log("🔀 Overlap run: {$n} sequential processes launched together."); + MicroChaos_Log::log(" Combined Throughput is not a Phase 4 RPS — size on a sequential --cache-bust run."); + + if (!empty($config['warm_cache'])) { + $this->warm_once($config); + } + + $tmp = $this->make_temp_dir(); + $handles = $this->launch($assoc_args, $n, $tmp); + $this->wait($handles); + $payloads = $this->read_payloads($handles); + + $this->report_merged($payloads, $n); + + $completed = 0; + $run_by_duration = false; + foreach ($payloads as $payload) { + $completed += (int) ($payload['completed'] ?? 0); + $run_by_duration = $run_by_duration || !empty($payload['run_by_duration']); + } + + return [ + 'completed' => $completed, + 'count' => $config['count'] * $n, + 'run_by_duration' => $run_by_duration, + 'actual_minutes' => $this->merged_minutes($payloads), + ]; + } + + /** + * Fire one warm request per endpoint in the parent, before children launch. + * + * @param array $config Parsed config + */ + private function warm_once(array $config): void { + MicroChaos_Log::log("🧤 Warming once in the parent, then launching children without --warm-cache."); + + $generator = new MicroChaos_Request_Generator([ + 'collect_cache_headers' => !empty($config['collect_cache_headers']), + 'timeout' => $config['timeout'] ?? MicroChaos_Constants::DEFAULT_REQUEST_TIMEOUT, + ]); + $endpoints = $this->endpoints_for_warm($generator, $config); + foreach ($endpoints as $endpoint_item) { + $generator->fire_request($endpoint_item['url'], null, null, $config['method'] ?? 'GET', null); + MicroChaos_Log::log(" Warmed {$endpoint_item['slug']}"); + } + } + + /** + * Resolve endpoints for the parent warm pass. + * + * @param MicroChaos_Request_Generator $generator Request generator + * @param array $config Parsed config + * @return array + */ + private function endpoints_for_warm(MicroChaos_Request_Generator $generator, array $config): array { + $orchestrator = new MicroChaos_LoadTest_Orchestrator($config); + $method = new \ReflectionMethod($orchestrator, 'resolve_endpoints'); + + return $method->invoke($orchestrator, $generator, $config); + } + + /** + * @return string Absolute temp directory + */ + private function make_temp_dir(): string { + $tmp = rtrim(sys_get_temp_dir(), '/') . '/microchaos-' . uniqid('', true); + if (!@mkdir($tmp, 0700) && !is_dir($tmp)) { + MicroChaos_Log::error("Could not create temp dir for overlap workers: {$tmp}"); + } + + return $tmp; + } + + /** + * Launch N children. Returns handles the waiter understands. + * + * @param array $assoc_args Parent CLI args + * @param int $n Process count + * @param string $tmp Temp directory + * @return array + */ + private function launch(array $assoc_args, int $n, string $tmp): array { + if (!function_exists('proc_open')) { + MicroChaos_Log::error( + "proc_open is disabled, so --concurrency cannot launch workers. " + . "Run the sequential test, or background N `wp microchaos loadtest --count=1` processes yourself." + ); + } + + $globals = $this->runtime_globals(); + $prefix = self::command_prefix(); + $handles = []; + + for ($i = 1; $i <= $n; $i++) { + $json = $tmp . '/worker-' . $i . '.json'; + $child = array_merge($globals, self::child_assoc_args($assoc_args, $i, $json)); + $cmd = $prefix . ' microchaos loadtest' . self::format_assoc_args($child); + $out = $tmp . '/worker-' . $i . '.out'; + $err = $tmp . '/worker-' . $i . '.err'; + + $descriptors = [ + 0 => ['file', '/dev/null', 'r'], + 1 => ['file', $out, 'w'], + 2 => ['file', $err, 'w'], + ]; + + $proc = proc_open($cmd, $descriptors, $pipes, null, null); + $handles[] = [ + 'proc' => $proc, + 'json' => $json, + 'id' => $i, + 'cmd' => $cmd, + 'err' => $err, + ]; + } + + return $handles; + } + + /** + * WP-CLI global flags the children must inherit or they hit the wrong site. + * + * @return array + */ + private function runtime_globals(): array { + $globals = []; + + if (!class_exists('WP_CLI')) { + return $globals; + } + + $config = \WP_CLI::get_runner()->config; + foreach (['path', 'url', 'user'] as $key) { + if (!empty($config[$key])) { + $globals[$key] = $config[$key]; + } + } + if (!empty($config['allow-root'])) { + $globals['allow-root'] = true; + } + + return $globals; + } + + /** + * Wait for every child, then terminate stragglers. + * + * @param array $handles + */ + private function wait(array $handles): void { + $deadline = time() + MicroChaos_Constants::DEFAULT_PARALLEL_TIMEOUT; + + while (time() < $deadline) { + $alive = 0; + foreach ($handles as $handle) { + if ($handle['proc'] === false) { + continue; + } + $status = proc_get_status($handle['proc']); + if (!empty($status['running'])) { + $alive++; + } + } + if ($alive === 0) { + break; + } + usleep(100000); + } + + foreach ($handles as $handle) { + if ($handle['proc'] === false) { + continue; + } + $status = proc_get_status($handle['proc']); + if (!empty($status['running'])) { + proc_terminate($handle['proc']); + MicroChaos_Log::warning("⚠️ Worker {$handle['id']} exceeded the overlap timeout and was terminated."); + } + proc_close($handle['proc']); + } + } + + /** + * @param array $handles + * @return array> + */ + private function read_payloads(array $handles): array { + $payloads = []; + + foreach ($handles as $handle) { + if (!is_readable($handle['json'])) { + $tail = ''; + if (!empty($handle['err']) && is_readable($handle['err'])) { + $tail = trim((string) file_get_contents($handle['err'])); + if (strlen($tail) > 300) { + $tail = substr($tail, -300); + } + } + MicroChaos_Log::warning( + "⚠️ Worker {$handle['id']} produced no results file." + . ($tail !== '' ? " stderr: {$tail}" : '') + ); + continue; + } + + $decoded = json_decode((string) file_get_contents($handle['json']), true); + if (!is_array($decoded) || !isset($decoded['results']) || !is_array($decoded['results'])) { + MicroChaos_Log::warning("⚠️ Worker {$handle['id']} wrote unreadable results JSON."); + continue; + } + + $payloads[] = $decoded; + } + + if ($payloads === []) { + MicroChaos_Log::error("No overlap workers returned results. The sequential path is unchanged; re-run without --concurrency."); + } + + return $payloads; + } + + /** + * Merge worker results and print one summary. No serial-ceiling projection: + * those numbers describe one-at-a-time cost and are a lie under overlap. + * + * @param array> $payloads + * @param int $n Process count + */ + private function report_merged(array $payloads, int $n): void { + $engine = new MicroChaos_Reporting_Engine(); + $starts = []; + $ends = []; + $completed = 0; + + foreach ($payloads as $payload) { + $engine->add_results($payload['results']); + $completed += (int) ($payload['completed'] ?? count($payload['results'])); + if (isset($payload['test_start_timestamp'])) { + $starts[] = (float) $payload['test_start_timestamp']; + } + if (isset($payload['test_end_timestamp'])) { + $ends[] = (float) $payload['test_end_timestamp']; + } + } + + $start = $starts === [] ? microtime(true) : min($starts); + $end = $ends === [] ? $start : max($ends); + $duration = max(0.0, $end - $start); + $rps = $duration > 0 ? round($completed / $duration, 2) : 0.0; + + $metrics = [ + 'started_at' => date('Y-m-d H:i:s', (int) $start), + 'started_at_iso' => date('c', (int) $start), + 'ended_at' => date('Y-m-d H:i:s', (int) $end), + 'ended_at_iso' => date('c', (int) $end), + 'duration_seconds' => round($duration, 2), + 'duration_formatted' => $this->format_duration($duration), + 'total_requests' => $completed, + 'throughput_rps' => $rps, + ]; + + MicroChaos_Log::log("📊 Overlap summary ({$n} processes)"); + MicroChaos_Log::log(" Do not size workers from this Throughput figure."); + $engine->report_summary(null, null, null, $metrics); + + $this->report_merged_cache($payloads, $completed); + } + + /** + * Rebuild the cache-header tally from workers so a stampede still shows + * HIT vs regen instead of losing it behind --results-json. + * + * @param array> $payloads + * @param int $completed Combined request count + */ + private function report_merged_cache(array $payloads, int $completed): void { + $analyzer = new MicroChaos_Cache_Analyzer(); + $has_cache = false; + + foreach ($payloads as $payload) { + if (empty($payload['cache_headers']) || !is_array($payload['cache_headers'])) { + continue; + } + foreach ($payload['cache_headers'] as $header => $values) { + if (!is_array($values)) { + continue; + } + foreach ($values as $value => $header_count) { + $has_cache = true; + $analyzer->collect_headers([$header => (string) $value], (int) $header_count); + } + } + } + + if ($has_cache) { + $analyzer->report_summary($completed); + } + } + + /** + * @param array> $payloads + */ + private function merged_minutes(array $payloads): float { + $starts = []; + $ends = []; + foreach ($payloads as $payload) { + if (isset($payload['test_start_timestamp'])) { + $starts[] = (float) $payload['test_start_timestamp']; + } + if (isset($payload['test_end_timestamp'])) { + $ends[] = (float) $payload['test_end_timestamp']; + } + } + if ($starts === [] || $ends === []) { + return 0.0; + } + + return round((max($ends) - min($starts)) / MicroChaos_Constants::SECONDS_PER_MINUTE, 1); + } + + /** + * @param float $seconds Duration in seconds + */ + private function format_duration(float $seconds): string { + $minutes = floor($seconds / 60); + $secs = (int) round($seconds % 60); + + if ($minutes > 0) { + return "{$minutes}m {$secs}s"; + } + + return "{$secs}s"; + } +} + class MicroChaos_Commands { /** * Register WP-CLI commands @@ -3739,7 +4245,8 @@ public static function register(): void { * When specified, this takes precedence over --count option. * * [--burst=] - * : Number of concurrent requests to fire per burst. Default: 10 + * : Number of sequential requests to fire back-to-back before the delay. + * This is pacing, not concurrency. Default: 10 * * [--delay=] * : Delay between bursts in seconds. Default: 2 @@ -3784,8 +4291,19 @@ public static function register(): void { * : Shorthand for GraphQL testing. Sets method=POST and endpoint=/graphql if not specified. * Use with --body to provide your GraphQL query. * + * [--concurrency=] + * : Launch this many sequential loadtest processes at the same instant so + * requests actually overlap. Default: 1 (no fan-out). Capped at 8. + * --count and --duration are per process: `--count=1 --concurrency=4` + * is a four-request stampede. Combined Throughput is not a Phase 4 RPS; + * size on a sequential `--cache-bust` run. + * + * [--results-json=] + * : Write this process's raw results as JSON and skip the printed summary. + * Used by `--concurrency` workers; also useful on its own for scripts. + * * [--rampup] - * : Gradually increase the number of concurrent requests from 1 up to the burst limit. + * : Gradually increase the number of back-to-back requests from 1 up to the burst limit. * * [--timeout=] * : Seconds to wait for a response before abandoning the request. Default: 30. @@ -3891,6 +4409,15 @@ public static function register(): void { * # subtracted from the per-site CPU reading before sizing workers from it. * wp microchaos loadtest --endpoint=home --duration=10 --resource-logging * + * # Overlap: four one-shot processes against a never-seen URL (stampede). + * # All regenerating means no cache lock. Size workers from a sequential + * # --cache-bust run, not from this Throughput figure. + * wp microchaos loadtest --endpoint=custom:/fresh-page/ --count=1 --cache-bust --concurrency=4 + * + * # Control: same N against a URL you already warmed. A HIT/regen split + * # is a race in the cache-validity check, not launch noise. + * wp microchaos loadtest --endpoint=home --count=1 --warm-cache --cache-headers --concurrency=4 + * * # Give a slow endpoint room to answer, so the tail is measured rather * # than cut off and recounted as an error. * wp microchaos loadtest --endpoint=custom:/checkout/ --duration=5 --timeout=60 @@ -3920,9 +4447,13 @@ public function loadtest(array $args, array $assoc_args): void { // Build config from CLI options $config = $this->parse_options($assoc_args); - // Create and execute orchestrator - $orchestrator = new MicroChaos_LoadTest_Orchestrator($config); - $result = $orchestrator->execute(); + if ($config['concurrency'] > 1) { + $runner = new MicroChaos_Concurrency_Runner(); + $result = $runner->run($assoc_args, $config); + } else { + $orchestrator = new MicroChaos_LoadTest_Orchestrator($config); + $result = $orchestrator->execute(); + } // Final success message if ($result['run_by_duration']) { @@ -3984,6 +4515,9 @@ private function parse_options(array $assoc_args): array { 'save_baseline' => $save_baseline, 'compare_baseline' => $compare_baseline, 'log_path' => $assoc_args['log-to'] ?? null, + 'concurrency' => intval($assoc_args['concurrency'] ?? MicroChaos_Constants::DEFAULT_CONCURRENCY), + 'results_json' => $assoc_args['results-json'] ?? null, + 'worker_id' => isset($assoc_args['worker-id']) ? intval($assoc_args['worker-id']) : null, ]; } } diff --git a/tools/microchaos-cli/microchaos-cli.php b/tools/microchaos-cli/microchaos-cli.php index a839749..d0e8663 100644 --- a/tools/microchaos-cli/microchaos-cli.php +++ b/tools/microchaos-cli/microchaos-cli.php @@ -3,7 +3,7 @@ * Plugin Name: MicroChaos CLI Load Tester * Description: Internal WP-CLI based WordPress load tester for staging environments where * external load testing is restricted (like Pressable). - * Version: 4.1.0 + * Version: 4.2.0 * Author: Phill * License: GPL-3.0-or-later * diff --git a/tools/microchaos-cli/microchaos/bootstrap.php b/tools/microchaos-cli/microchaos/bootstrap.php index e58debc..eed123c 100644 --- a/tools/microchaos-cli/microchaos/bootstrap.php +++ b/tools/microchaos-cli/microchaos/bootstrap.php @@ -97,6 +97,7 @@ private static function load_core_components() { 'cache-analyzer.php', 'resource-monitor.php', 'reporting-engine.php', + 'concurrency-runner.php', 'commands.php', // Must be last - depends on orchestrators ]; diff --git a/tools/microchaos-cli/microchaos/core/commands.php b/tools/microchaos-cli/microchaos/core/commands.php index b90d6cf..a758c46 100644 --- a/tools/microchaos-cli/microchaos/core/commands.php +++ b/tools/microchaos-cli/microchaos/core/commands.php @@ -69,7 +69,8 @@ public static function register(): void { * When specified, this takes precedence over --count option. * * [--burst=] - * : Number of concurrent requests to fire per burst. Default: 10 + * : Number of sequential requests to fire back-to-back before the delay. + * This is pacing, not concurrency. Default: 10 * * [--delay=] * : Delay between bursts in seconds. Default: 2 @@ -114,8 +115,19 @@ public static function register(): void { * : Shorthand for GraphQL testing. Sets method=POST and endpoint=/graphql if not specified. * Use with --body to provide your GraphQL query. * + * [--concurrency=] + * : Launch this many sequential loadtest processes at the same instant so + * requests actually overlap. Default: 1 (no fan-out). Capped at 8. + * --count and --duration are per process: `--count=1 --concurrency=4` + * is a four-request stampede. Combined Throughput is not a Phase 4 RPS; + * size on a sequential `--cache-bust` run. + * + * [--results-json=] + * : Write this process's raw results as JSON and skip the printed summary. + * Used by `--concurrency` workers; also useful on its own for scripts. + * * [--rampup] - * : Gradually increase the number of concurrent requests from 1 up to the burst limit. + * : Gradually increase the number of back-to-back requests from 1 up to the burst limit. * * [--timeout=] * : Seconds to wait for a response before abandoning the request. Default: 30. @@ -221,6 +233,15 @@ public static function register(): void { * # subtracted from the per-site CPU reading before sizing workers from it. * wp microchaos loadtest --endpoint=home --duration=10 --resource-logging * + * # Overlap: four one-shot processes against a never-seen URL (stampede). + * # All regenerating means no cache lock. Size workers from a sequential + * # --cache-bust run, not from this Throughput figure. + * wp microchaos loadtest --endpoint=custom:/fresh-page/ --count=1 --cache-bust --concurrency=4 + * + * # Control: same N against a URL you already warmed. A HIT/regen split + * # is a race in the cache-validity check, not launch noise. + * wp microchaos loadtest --endpoint=home --count=1 --warm-cache --cache-headers --concurrency=4 + * * # Give a slow endpoint room to answer, so the tail is measured rather * # than cut off and recounted as an error. * wp microchaos loadtest --endpoint=custom:/checkout/ --duration=5 --timeout=60 @@ -250,9 +271,13 @@ public function loadtest(array $args, array $assoc_args): void { // Build config from CLI options $config = $this->parse_options($assoc_args); - // Create and execute orchestrator - $orchestrator = new MicroChaos_LoadTest_Orchestrator($config); - $result = $orchestrator->execute(); + if ($config['concurrency'] > 1) { + $runner = new MicroChaos_Concurrency_Runner(); + $result = $runner->run($assoc_args, $config); + } else { + $orchestrator = new MicroChaos_LoadTest_Orchestrator($config); + $result = $orchestrator->execute(); + } // Final success message if ($result['run_by_duration']) { @@ -314,6 +339,9 @@ private function parse_options(array $assoc_args): array { 'save_baseline' => $save_baseline, 'compare_baseline' => $compare_baseline, 'log_path' => $assoc_args['log-to'] ?? null, + 'concurrency' => intval($assoc_args['concurrency'] ?? MicroChaos_Constants::DEFAULT_CONCURRENCY), + 'results_json' => $assoc_args['results-json'] ?? null, + 'worker_id' => isset($assoc_args['worker-id']) ? intval($assoc_args['worker-id']) : null, ]; } } diff --git a/tools/microchaos-cli/microchaos/core/concurrency-runner.php b/tools/microchaos-cli/microchaos/core/concurrency-runner.php new file mode 100644 index 0000000..3f5d12d --- /dev/null +++ b/tools/microchaos-cli/microchaos/core/concurrency-runner.php @@ -0,0 +1,469 @@ + MicroChaos_Constants::MAX_CONCURRENCY) { + return MicroChaos_Constants::MAX_CONCURRENCY; + } + + return $requested; + } + + /** + * Build the assoc args one child process should receive. + * + * Forces concurrency=1 so a child cannot fan out again. Points the child + * at a results JSON file so the parent can merge without reading interleaved + * stdout. Drops warm-cache: the parent warms once, then the children hit. + * + * @param array $assoc_args Parent CLI args + * @param int $worker_id 1-based worker id + * @param string $results_json Absolute path the child should write + * @return array Child assoc args + */ + public static function child_assoc_args(array $assoc_args, int $worker_id, string $results_json): array { + $child = $assoc_args; + $child['concurrency'] = 1; + $child['results-json'] = $results_json; + $child['worker-id'] = $worker_id; + unset($child['warm-cache']); + + return $child; + } + + /** + * Format assoc args as a shell-safe WP-CLI flag string. + * + * @param array $assoc_args Flags to format + * @return string Leading-space flag string, or empty + */ + public static function format_assoc_args(array $assoc_args): string { + $parts = []; + + foreach ($assoc_args as $key => $value) { + if ($value === false || $value === null) { + continue; + } + if ($value === true) { + $parts[] = '--' . $key; + continue; + } + $parts[] = '--' . $key . '=' . escapeshellarg((string) $value); + } + + return $parts === [] ? '' : ' ' . implode(' ', $parts); + } + + /** + * Prefix used to re-invoke WP-CLI as a child. + * + * Prefers the current argv[0] so a `wp` wrapper stays a `wp` wrapper. + * Falls back to `wp` when argv is missing (tests). + * + * @return string Shell-escaped command prefix, no trailing space + */ + public static function command_prefix(): string { + $invoker = $GLOBALS['argv'][0] ?? 'wp'; + + if (self::invoker_is_php_script($invoker)) { + return escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg($invoker); + } + + return escapeshellarg($invoker); + } + + /** + * Whether the invoker is a PHP script that must be launched via PHP_BINARY. + * + * @param string $invoker Path from argv[0] + * @return bool + */ + public static function invoker_is_php_script(string $invoker): bool { + $base = strtolower(basename($invoker)); + + return str_ends_with($base, '.php') || str_ends_with($base, '.phar'); + } + + /** + * Fan out, wait, merge, and report. Returns the same shape as execute(). + * + * @param array $assoc_args Parent CLI args + * @param array $config Parsed config + * @return array{completed: int, count: int, run_by_duration: bool, actual_minutes: float} + */ + public function run(array $assoc_args, array $config): array { + $n = self::clamp((int) $config['concurrency']); + + if ($n !== (int) $config['concurrency']) { + MicroChaos_Log::warning( + "⚠️ --concurrency={$config['concurrency']} exceeds the cap of " + . MicroChaos_Constants::MAX_CONCURRENCY + . "; running {$n} processes." + ); + } + + MicroChaos_Log::log("🔀 Overlap run: {$n} sequential processes launched together."); + MicroChaos_Log::log(" Combined Throughput is not a Phase 4 RPS — size on a sequential --cache-bust run."); + + if (!empty($config['warm_cache'])) { + $this->warm_once($config); + } + + $tmp = $this->make_temp_dir(); + $handles = $this->launch($assoc_args, $n, $tmp); + $this->wait($handles); + $payloads = $this->read_payloads($handles); + + $this->report_merged($payloads, $n); + + $completed = 0; + $run_by_duration = false; + foreach ($payloads as $payload) { + $completed += (int) ($payload['completed'] ?? 0); + $run_by_duration = $run_by_duration || !empty($payload['run_by_duration']); + } + + return [ + 'completed' => $completed, + 'count' => $config['count'] * $n, + 'run_by_duration' => $run_by_duration, + 'actual_minutes' => $this->merged_minutes($payloads), + ]; + } + + /** + * Fire one warm request per endpoint in the parent, before children launch. + * + * @param array $config Parsed config + */ + private function warm_once(array $config): void { + MicroChaos_Log::log("🧤 Warming once in the parent, then launching children without --warm-cache."); + + $generator = new MicroChaos_Request_Generator([ + 'collect_cache_headers' => !empty($config['collect_cache_headers']), + 'timeout' => $config['timeout'] ?? MicroChaos_Constants::DEFAULT_REQUEST_TIMEOUT, + ]); + $endpoints = $this->endpoints_for_warm($generator, $config); + foreach ($endpoints as $endpoint_item) { + $generator->fire_request($endpoint_item['url'], null, null, $config['method'] ?? 'GET', null); + MicroChaos_Log::log(" Warmed {$endpoint_item['slug']}"); + } + } + + /** + * Resolve endpoints for the parent warm pass. + * + * @param MicroChaos_Request_Generator $generator Request generator + * @param array $config Parsed config + * @return array + */ + private function endpoints_for_warm(MicroChaos_Request_Generator $generator, array $config): array { + $orchestrator = new MicroChaos_LoadTest_Orchestrator($config); + $method = new \ReflectionMethod($orchestrator, 'resolve_endpoints'); + + return $method->invoke($orchestrator, $generator, $config); + } + + /** + * @return string Absolute temp directory + */ + private function make_temp_dir(): string { + $tmp = rtrim(sys_get_temp_dir(), '/') . '/microchaos-' . uniqid('', true); + if (!@mkdir($tmp, 0700) && !is_dir($tmp)) { + MicroChaos_Log::error("Could not create temp dir for overlap workers: {$tmp}"); + } + + return $tmp; + } + + /** + * Launch N children. Returns handles the waiter understands. + * + * @param array $assoc_args Parent CLI args + * @param int $n Process count + * @param string $tmp Temp directory + * @return array + */ + private function launch(array $assoc_args, int $n, string $tmp): array { + if (!function_exists('proc_open')) { + MicroChaos_Log::error( + "proc_open is disabled, so --concurrency cannot launch workers. " + . "Run the sequential test, or background N `wp microchaos loadtest --count=1` processes yourself." + ); + } + + $globals = $this->runtime_globals(); + $prefix = self::command_prefix(); + $handles = []; + + for ($i = 1; $i <= $n; $i++) { + $json = $tmp . '/worker-' . $i . '.json'; + $child = array_merge($globals, self::child_assoc_args($assoc_args, $i, $json)); + $cmd = $prefix . ' microchaos loadtest' . self::format_assoc_args($child); + $out = $tmp . '/worker-' . $i . '.out'; + $err = $tmp . '/worker-' . $i . '.err'; + + $descriptors = [ + 0 => ['file', '/dev/null', 'r'], + 1 => ['file', $out, 'w'], + 2 => ['file', $err, 'w'], + ]; + + $proc = proc_open($cmd, $descriptors, $pipes, null, null); + $handles[] = [ + 'proc' => $proc, + 'json' => $json, + 'id' => $i, + 'cmd' => $cmd, + 'err' => $err, + ]; + } + + return $handles; + } + + /** + * WP-CLI global flags the children must inherit or they hit the wrong site. + * + * @return array + */ + private function runtime_globals(): array { + $globals = []; + + if (!class_exists('WP_CLI')) { + return $globals; + } + + $config = \WP_CLI::get_runner()->config; + foreach (['path', 'url', 'user'] as $key) { + if (!empty($config[$key])) { + $globals[$key] = $config[$key]; + } + } + if (!empty($config['allow-root'])) { + $globals['allow-root'] = true; + } + + return $globals; + } + + /** + * Wait for every child, then terminate stragglers. + * + * @param array $handles + */ + private function wait(array $handles): void { + $deadline = time() + MicroChaos_Constants::DEFAULT_PARALLEL_TIMEOUT; + + while (time() < $deadline) { + $alive = 0; + foreach ($handles as $handle) { + if ($handle['proc'] === false) { + continue; + } + $status = proc_get_status($handle['proc']); + if (!empty($status['running'])) { + $alive++; + } + } + if ($alive === 0) { + break; + } + usleep(100000); + } + + foreach ($handles as $handle) { + if ($handle['proc'] === false) { + continue; + } + $status = proc_get_status($handle['proc']); + if (!empty($status['running'])) { + proc_terminate($handle['proc']); + MicroChaos_Log::warning("⚠️ Worker {$handle['id']} exceeded the overlap timeout and was terminated."); + } + proc_close($handle['proc']); + } + } + + /** + * @param array $handles + * @return array> + */ + private function read_payloads(array $handles): array { + $payloads = []; + + foreach ($handles as $handle) { + if (!is_readable($handle['json'])) { + $tail = ''; + if (!empty($handle['err']) && is_readable($handle['err'])) { + $tail = trim((string) file_get_contents($handle['err'])); + if (strlen($tail) > 300) { + $tail = substr($tail, -300); + } + } + MicroChaos_Log::warning( + "⚠️ Worker {$handle['id']} produced no results file." + . ($tail !== '' ? " stderr: {$tail}" : '') + ); + continue; + } + + $decoded = json_decode((string) file_get_contents($handle['json']), true); + if (!is_array($decoded) || !isset($decoded['results']) || !is_array($decoded['results'])) { + MicroChaos_Log::warning("⚠️ Worker {$handle['id']} wrote unreadable results JSON."); + continue; + } + + $payloads[] = $decoded; + } + + if ($payloads === []) { + MicroChaos_Log::error("No overlap workers returned results. The sequential path is unchanged; re-run without --concurrency."); + } + + return $payloads; + } + + /** + * Merge worker results and print one summary. No serial-ceiling projection: + * those numbers describe one-at-a-time cost and are a lie under overlap. + * + * @param array> $payloads + * @param int $n Process count + */ + private function report_merged(array $payloads, int $n): void { + $engine = new MicroChaos_Reporting_Engine(); + $starts = []; + $ends = []; + $completed = 0; + + foreach ($payloads as $payload) { + $engine->add_results($payload['results']); + $completed += (int) ($payload['completed'] ?? count($payload['results'])); + if (isset($payload['test_start_timestamp'])) { + $starts[] = (float) $payload['test_start_timestamp']; + } + if (isset($payload['test_end_timestamp'])) { + $ends[] = (float) $payload['test_end_timestamp']; + } + } + + $start = $starts === [] ? microtime(true) : min($starts); + $end = $ends === [] ? $start : max($ends); + $duration = max(0.0, $end - $start); + $rps = $duration > 0 ? round($completed / $duration, 2) : 0.0; + + $metrics = [ + 'started_at' => date('Y-m-d H:i:s', (int) $start), + 'started_at_iso' => date('c', (int) $start), + 'ended_at' => date('Y-m-d H:i:s', (int) $end), + 'ended_at_iso' => date('c', (int) $end), + 'duration_seconds' => round($duration, 2), + 'duration_formatted' => $this->format_duration($duration), + 'total_requests' => $completed, + 'throughput_rps' => $rps, + ]; + + MicroChaos_Log::log("📊 Overlap summary ({$n} processes)"); + MicroChaos_Log::log(" Do not size workers from this Throughput figure."); + $engine->report_summary(null, null, null, $metrics); + + $this->report_merged_cache($payloads, $completed); + } + + /** + * Rebuild the cache-header tally from workers so a stampede still shows + * HIT vs regen instead of losing it behind --results-json. + * + * @param array> $payloads + * @param int $completed Combined request count + */ + private function report_merged_cache(array $payloads, int $completed): void { + $analyzer = new MicroChaos_Cache_Analyzer(); + $has_cache = false; + + foreach ($payloads as $payload) { + if (empty($payload['cache_headers']) || !is_array($payload['cache_headers'])) { + continue; + } + foreach ($payload['cache_headers'] as $header => $values) { + if (!is_array($values)) { + continue; + } + foreach ($values as $value => $header_count) { + $has_cache = true; + $analyzer->collect_headers([$header => (string) $value], (int) $header_count); + } + } + } + + if ($has_cache) { + $analyzer->report_summary($completed); + } + } + + /** + * @param array> $payloads + */ + private function merged_minutes(array $payloads): float { + $starts = []; + $ends = []; + foreach ($payloads as $payload) { + if (isset($payload['test_start_timestamp'])) { + $starts[] = (float) $payload['test_start_timestamp']; + } + if (isset($payload['test_end_timestamp'])) { + $ends[] = (float) $payload['test_end_timestamp']; + } + } + if ($starts === [] || $ends === []) { + return 0.0; + } + + return round((max($ends) - min($starts)) / MicroChaos_Constants::SECONDS_PER_MINUTE, 1); + } + + /** + * @param float $seconds Duration in seconds + */ + private function format_duration(float $seconds): string { + $minutes = floor($seconds / 60); + $secs = (int) round($seconds % 60); + + if ($minutes > 0) { + return "{$minutes}m {$secs}s"; + } + + return "{$secs}s"; + } +} diff --git a/tools/microchaos-cli/microchaos/core/constants.php b/tools/microchaos-cli/microchaos/core/constants.php index b7eebda..e37288c 100644 --- a/tools/microchaos-cli/microchaos/core/constants.php +++ b/tools/microchaos-cli/microchaos/core/constants.php @@ -39,6 +39,17 @@ class MicroChaos_Constants { */ const DEFAULT_WORKERS = 3; + /** + * Default --concurrency. 1 is the sequential path; nothing fans out. + */ + const DEFAULT_CONCURRENCY = 1; + + /** + * Hard cap on --concurrency. Staging boxes are 5 workers; past 8 the + * generators themselves start to be the thing under test. + */ + const MAX_CONCURRENCY = 8; + /** * Time conversion constants */ diff --git a/tools/microchaos-cli/microchaos/core/orchestrators/loadtest-orchestrator.php b/tools/microchaos-cli/microchaos/core/orchestrators/loadtest-orchestrator.php index d61fc0c..5f5d7ae 100644 --- a/tools/microchaos-cli/microchaos/core/orchestrators/loadtest-orchestrator.php +++ b/tools/microchaos-cli/microchaos/core/orchestrators/loadtest-orchestrator.php @@ -73,6 +73,9 @@ private function normalize_config(array $config): array { 'save_baseline' => null, 'compare_baseline' => null, 'log_path' => null, + 'concurrency' => MicroChaos_Constants::DEFAULT_CONCURRENCY, + 'results_json' => null, + 'worker_id' => null, ], $config); } @@ -227,12 +230,21 @@ public function execute(): array { ); } - // Display reports - $reporting_engine->report_summary($perf_baseline, null, $use_thresholds, $execution_metrics); + if (!empty($config['results_json'])) { + $this->write_results_json($config['results_json'], $reporting_engine, $loop_result, $cache_analyzer); + } - $this->report_timeout_censoring($reporting_engine, $request_generator->get_timeout()); + // A worker writing results-json is collected by the parent; printing + // a second summary here would interleave with its siblings. + if (empty($config['results_json'])) { + $reporting_engine->report_summary($perf_baseline, null, $use_thresholds, $execution_metrics); + } - if ($config['resource_logging']) { + if (empty($config['results_json'])) { + $this->report_timeout_censoring($reporting_engine, $request_generator->get_timeout()); + } + + if ($config['resource_logging'] && empty($config['results_json'])) { $resource_monitor->report_summary($resource_baseline, null, $use_thresholds); if ($config['resource_trends']) { @@ -241,7 +253,7 @@ public function execute(): array { } // Save baseline if specified - if ($config['save_baseline']) { + if ($config['save_baseline'] && empty($config['results_json'])) { $reporting_engine->save_baseline($config['save_baseline']); if ($config['resource_logging']) { $resource_monitor->save_baseline($config['save_baseline']); @@ -250,7 +262,7 @@ public function execute(): array { } // Report cache headers if enabled - if ($config['collect_cache_headers']) { + if ($config['collect_cache_headers'] && empty($config['results_json'])) { $cache_analyzer->report_summary($reporting_engine->get_request_count()); } @@ -276,6 +288,40 @@ public function execute(): array { ]; } + /** + * Write this process's raw results for a parent overlap run to merge. + * + * @param string $path Absolute path + * @param MicroChaos_Reporting_Engine $reporting_engine + * @param array $loop_result + * @param MicroChaos_Cache_Analyzer $cache_analyzer + */ + private function write_results_json(string $path, MicroChaos_Reporting_Engine $reporting_engine, array $loop_result, MicroChaos_Cache_Analyzer $cache_analyzer): void { + $payload = [ + 'worker_id' => $this->config['worker_id'], + 'results' => $reporting_engine->get_results(), + 'completed' => $loop_result['completed'], + 'run_by_duration' => $loop_result['run_by_duration'], + 'test_start_timestamp' => $loop_result['test_start_timestamp'], + 'test_end_timestamp' => $loop_result['test_end_timestamp'], + 'cache_headers' => $cache_analyzer->get_cache_headers(), + ]; + + $dir = dirname($path); + if (!is_dir($dir) && !@mkdir($dir, 0700, true) && !is_dir($dir)) { + MicroChaos_Log::error("Could not write results JSON directory: {$dir}"); + } + + $json = function_exists('wp_json_encode') + ? wp_json_encode($payload, JSON_UNESCAPED_SLASHES) + : json_encode($payload, JSON_UNESCAPED_SLASHES); + + $written = @file_put_contents($path, $json); + if ($written === false) { + MicroChaos_Log::error("Could not write results JSON: {$path}"); + } + } + /** * Warn that the timing distribution is right-censored, if any request timed out * diff --git a/tools/microchaos-cli/tests/Unit/ConcurrencyRunnerTest.php b/tools/microchaos-cli/tests/Unit/ConcurrencyRunnerTest.php new file mode 100644 index 0000000..d7d6020 --- /dev/null +++ b/tools/microchaos-cli/tests/Unit/ConcurrencyRunnerTest.php @@ -0,0 +1,87 @@ +assertSame(4, \MicroChaos_Concurrency_Runner::clamp(4)); + } + + #[Test] + public function clamp_floors_below_one_to_one(): void + { + $this->assertSame(1, \MicroChaos_Concurrency_Runner::clamp(0)); + $this->assertSame(1, \MicroChaos_Concurrency_Runner::clamp(-3)); + } + + #[Test] + public function clamp_caps_at_max_concurrency(): void + { + $this->assertSame( + \MicroChaos_Constants::MAX_CONCURRENCY, + \MicroChaos_Concurrency_Runner::clamp(\MicroChaos_Constants::MAX_CONCURRENCY + 20) + ); + } + + #[Test] + public function child_args_force_concurrency_one_and_drop_warm_cache(): void + { + $child = \MicroChaos_Concurrency_Runner::child_assoc_args( + [ + 'endpoint' => 'home', + 'count' => 1, + 'concurrency' => 4, + 'warm-cache' => true, + 'cache-bust' => true, + ], + 2, + '/tmp/worker-2.json' + ); + + $this->assertSame(1, $child['concurrency']); + $this->assertSame(2, $child['worker-id']); + $this->assertSame('/tmp/worker-2.json', $child['results-json']); + $this->assertArrayNotHasKey('warm-cache', $child); + $this->assertTrue($child['cache-bust']); + $this->assertSame(1, $child['count']); + } + + #[Test] + public function format_assoc_args_emits_flags_and_skips_false(): void + { + $formatted = \MicroChaos_Concurrency_Runner::format_assoc_args([ + 'allow-root' => true, + 'count' => 1, + 'warm-cache' => false, + 'endpoint' => 'home', + ]); + + $this->assertStringContainsString('--allow-root', $formatted); + $this->assertStringContainsString('--count=', $formatted); + $this->assertStringContainsString('--endpoint=', $formatted); + $this->assertStringNotContainsString('warm-cache', $formatted); + } + + #[Test] + public function php_scripts_and_phars_are_launched_via_php_binary(): void + { + $this->assertTrue(\MicroChaos_Concurrency_Runner::invoker_is_php_script('wp-cli.phar')); + $this->assertTrue(\MicroChaos_Concurrency_Runner::invoker_is_php_script('/usr/local/bin/wp-cli.php')); + $this->assertFalse(\MicroChaos_Concurrency_Runner::invoker_is_php_script('wp')); + $this->assertFalse(\MicroChaos_Concurrency_Runner::invoker_is_php_script('/usr/local/bin/wp')); + } +} diff --git a/tools/microchaos-cli/tests/Unit/ConstantsTest.php b/tools/microchaos-cli/tests/Unit/ConstantsTest.php index 06f3edf..f668315 100644 --- a/tools/microchaos-cli/tests/Unit/ConstantsTest.php +++ b/tools/microchaos-cli/tests/Unit/ConstantsTest.php @@ -84,5 +84,12 @@ public function parallel_execution_defaults_are_reasonable(): void // 3 workers default $this->assertEquals(3, \MicroChaos_Constants::DEFAULT_WORKERS); $this->assertGreaterThan(0, \MicroChaos_Constants::DEFAULT_WORKERS); + + $this->assertSame(1, \MicroChaos_Constants::DEFAULT_CONCURRENCY); + $this->assertSame(8, \MicroChaos_Constants::MAX_CONCURRENCY); + $this->assertGreaterThan( + \MicroChaos_Constants::DEFAULT_CONCURRENCY, + \MicroChaos_Constants::MAX_CONCURRENCY + ); } } diff --git a/tools/microchaos-cli/tests/Unit/LoadTestOrchestratorTest.php b/tools/microchaos-cli/tests/Unit/LoadTestOrchestratorTest.php index 13701d6..58ac62b 100644 --- a/tools/microchaos-cli/tests/Unit/LoadTestOrchestratorTest.php +++ b/tools/microchaos-cli/tests/Unit/LoadTestOrchestratorTest.php @@ -38,6 +38,15 @@ private function bust(\MicroChaos_LoadTest_Orchestrator $orchestrator, string $u // Config normalization // ========================================================================= + #[Test] + public function concurrency_defaults_to_one(): void + { + $config = $this->config($this->orchestrator()); + + $this->assertSame(\MicroChaos_Constants::DEFAULT_CONCURRENCY, $config['concurrency']); + $this->assertNull($config['results_json']); + } + #[Test] public function cache_bust_defaults_to_off(): void { diff --git a/tools/microchaos-cli/tests/bootstrap.php b/tools/microchaos-cli/tests/bootstrap.php index 126c96d..f8ead9a 100644 --- a/tools/microchaos-cli/tests/bootstrap.php +++ b/tools/microchaos-cli/tests/bootstrap.php @@ -122,6 +122,9 @@ function wp_generate_uuid4(): string { // Orchestrator: throughput arithmetic is pure PHP and reachable without a run. require_once MICROCHAOS_CORE_PATH . '/orchestrators/loadtest-orchestrator.php'; +// Concurrency runner: clamp / child-arg / flag formatting are pure PHP. +require_once MICROCHAOS_CORE_PATH . '/concurrency-runner.php'; + /** * WordPress Stubs - Classes * From 162e641d5a50a1f13c4b9bac42e09b2336294a33 Mon Sep 17 00:00:00 2001 From: Dave Ventesca Date: Fri, 28 Aug 2026 12:33:20 -0400 Subject: [PATCH 2/3] microchaos: label every run with the test mode it measured Origin cost, cache effectiveness and overlap answer different questions, but all three printed near-identical summaries, so a report pasted into an audit relied on the operator remembering which flags produced it. Each run now resolves a mode descriptor, announces it before the run and repeats it in the summary block, and carries it in --results-json. A sequential run with neither cache flag is labelled as uncontrolled and warned about rather than passing as a sizing run. Co-authored-by: Cursor --- tools/microchaos-cli/README.md | 30 +++++ tools/microchaos-cli/build.js | 2 + tools/microchaos-cli/dist/microchaos-cli.php | 119 ++++++++++++++++-- tools/microchaos-cli/microchaos/bootstrap.php | 3 + .../microchaos/core/concurrency-runner.php | 8 +- .../orchestrators/loadtest-orchestrator.php | 13 +- .../microchaos/core/reporting-engine.php | 9 ++ .../microchaos/core/test-mode.php | 114 +++++++++++++++++ .../tests/Unit/TestModeTest.php | 98 +++++++++++++++ tools/microchaos-cli/tests/bootstrap.php | 3 + 10 files changed, 383 insertions(+), 16 deletions(-) create mode 100644 tools/microchaos-cli/microchaos/core/test-mode.php create mode 100644 tools/microchaos-cli/tests/Unit/TestModeTest.php diff --git a/tools/microchaos-cli/README.md b/tools/microchaos-cli/README.md index 80b74ed..a36b23c 100644 --- a/tools/microchaos-cli/README.md +++ b/tools/microchaos-cli/README.md @@ -42,6 +42,36 @@ stampede races, HIT/regen splits on an already-warm URL). - Combined Throughput from an overlap run is **not** a Phase 4 RPS. Size on a sequential `--cache-bust` run. The summary says so. - Sequential default is unchanged. Existing 4.1.0 commands keep their numbers. +- Every run now names its **test mode**, on screen as it starts and again in the + summary, so a pasted report says what it measured. + +### Test modes + +Three runs answer three different questions, and their numbers are not +interchangeable. The mode is printed before the run and repeated in the summary +block, so an operator reading a report later does not have to reconstruct which +flags produced it. + +| Mode | Flags | Measures | Sizing input | +|------|-------|----------|--------------| +| Origin cost | `--cache-bust` | What one uncached request costs the origin | Yes, this is the Phase 4 input | +| Cache effectiveness | `--warm-cache` | What the cache serves once the URL is warm | No, compare it against the origin-cost run | +| Overlap | `--concurrency=N` | How the site behaves when requests share workers | No, combined Throughput mixes queueing and N generators | + +A sequential run with neither cache flag is labelled `Sequential, cache state not +controlled` and flagged as a warning. It is a mix of hits and misses in unknown +proportion, so it answers neither question. + +```bash +# Run 1 — origin cost, the number Phase 4 sizes on +wp microchaos loadtest --endpoint=home --duration=2 --cache-bust --cache-headers + +# Run 2 — what the cache is buying +wp microchaos loadtest --endpoint=home --duration=2 --warm-cache --cache-headers + +# Run 3 — overlap, four processes at the same instant +wp microchaos loadtest --endpoint=home --count=5 --concurrency=4 --cache-headers +``` ## 🆕 What's New in v4.1.0 "Ground Truth" diff --git a/tools/microchaos-cli/build.js b/tools/microchaos-cli/build.js index c55931e..cab5657 100755 --- a/tools/microchaos-cli/build.js +++ b/tools/microchaos-cli/build.js @@ -19,6 +19,8 @@ const sources = { components: [ // Load constants first path.join(__dirname, "microchaos/core/constants.php"), + // Load the test-mode descriptor before anything that reports a run + path.join(__dirname, "microchaos/core/test-mode.php"), // Load interfaces (logger first - used by all components) path.join(__dirname, "microchaos/core/interfaces/logger.php"), path.join(__dirname, "microchaos/core/interfaces/baseline-storage.php"), diff --git a/tools/microchaos-cli/dist/microchaos-cli.php b/tools/microchaos-cli/dist/microchaos-cli.php index 14118ef..365678c 100644 --- a/tools/microchaos-cli/dist/microchaos-cli.php +++ b/tools/microchaos-cli/dist/microchaos-cli.php @@ -79,6 +79,95 @@ class MicroChaos_Constants { const HTTP_SERVER_ERROR = 500; } +class MicroChaos_Test_Mode { + + /** + * Resolve the mode descriptor for a run. + * + * @param array $config Parsed config + * @return array{id: string, label: string, measures: string, sizing: string, sizing_ok: bool} + */ + public static function resolve(array $config): array { + $concurrency = (int) ($config['concurrency'] ?? 1); + + if ($concurrency > 1) { + return [ + 'id' => 'overlap', + 'label' => "Overlap ({$concurrency} processes launched together)", + 'measures' => 'How the site behaves when requests actually share workers.', + 'sizing' => 'Not a sizing input. Combined Throughput mixes queueing and ' + . "{$concurrency} load generators into one figure.", + 'sizing_ok' => false, + ]; + } + + $cache_bust = !empty($config['cache_bust']); + $warm_cache = !empty($config['warm_cache']); + + // --cache-bust wins the label when both are set. The orchestrator already + // warns that warming is pointless alongside it, and the run does measure + // origin cost, so calling it anything else would be the bigger lie. + if ($cache_bust) { + return [ + 'id' => 'origin-cost', + 'label' => 'Origin cost (sequential, cache-busted)', + 'measures' => 'What one uncached request costs the origin, one request at a time.', + 'sizing' => 'Throughput from this run is the Phase 4 sizing input.', + 'sizing_ok' => true, + ]; + } + + if ($warm_cache) { + return [ + 'id' => 'cache-effectiveness', + 'label' => 'Cache effectiveness (sequential, warmed)', + 'measures' => 'What the cache serves once the URL is warm.', + 'sizing' => 'Not a sizing input. Compare it against a --cache-bust run to see ' + . 'what the cache is buying the customer.', + 'sizing_ok' => false, + ]; + } + + return [ + 'id' => 'uncontrolled', + 'label' => 'Sequential, cache state not controlled', + 'measures' => 'A mix. Some requests may be served from cache and some not, ' + . 'and the split is not recorded.', + 'sizing' => 'Neither origin cost nor cache effectiveness. Add --cache-bust for ' + . 'sizing, or --warm-cache to measure the cache.', + 'sizing_ok' => false, + ]; + } + + /** + * Announce the mode before the run starts, so the operator sees what is + * being measured while it happens rather than only in the summary. + * + * @param array{label: string, measures: string, sizing: string, sizing_ok: bool} $mode + */ + public static function announce(array $mode): void { + MicroChaos_Log::log("🧪 Test mode: {$mode['label']}"); + MicroChaos_Log::log(" Measures: {$mode['measures']}"); + + if ($mode['sizing_ok']) { + MicroChaos_Log::log(" Sizing: {$mode['sizing']}"); + return; + } + + MicroChaos_Log::warning(" Sizing: {$mode['sizing']}"); + } + + /** + * The one-line form carried in the summary block. + * + * @param array{label: string, sizing: string} $mode + * @return string + */ + public static function summary_line(array $mode): string { + return "{$mode['label']} — {$mode['sizing']}"; + } +} + interface MicroChaos_Logger_Interface { /** @@ -2777,6 +2866,15 @@ public function report_summary(?array $baseline = null, ?array $provided_summary MicroChaos_Log::log("📊 Load Test Summary"); MicroChaos_Log::log(" ═══════════════════════════════════════════════════"); + // The summary is what gets pasted into an audit, so it names the mode + // it measured. Without it, three runs produce three near-identical + // blocks and only the operator's memory says which is the sizing one. + if (!empty($execution_metrics['mode'])) { + MicroChaos_Log::log(" Mode: {$execution_metrics['mode']['label']}"); + MicroChaos_Log::log(" {$execution_metrics['mode']['sizing']}"); + MicroChaos_Log::log(" ───────────────────────────────────────────────────"); + } + // Test Execution Metrics section if ($execution_metrics) { MicroChaos_Log::log(" Test Execution:"); @@ -3101,10 +3199,13 @@ public function execute(): array { // Log test start $this->log_test_start($config, $endpoint_list, $integration_logger); - // Announce the measurement mode: the two modes answer different questions - // and the numbers are not comparable, so the log has to say which ran. - if ($config['cache_bust']) { - MicroChaos_Log::log("🚫 Cache busting enabled — every request gets a unique URL, so this measures origin cost, not cache performance."); + // Announce the measurement mode: the modes answer different questions and + // their numbers are not interchangeable, so the log has to say which ran. + // Fan-out children are skipped, because the parent labels the ensemble and + // a child announcing itself as a sizing run would contradict it. + $mode = MicroChaos_Test_Mode::resolve($config); + if (empty($config['worker_id'])) { + MicroChaos_Test_Mode::announce($mode); } // Warm cache if specified @@ -3145,6 +3246,7 @@ public function execute(): array { $loop_result['completed'], $reporting_engine->get_results() ); + $execution_metrics['mode'] = $mode; // Handle baseline comparison $perf_baseline = $config['compare_baseline'] @@ -3239,6 +3341,7 @@ public function execute(): array { private function write_results_json(string $path, MicroChaos_Reporting_Engine $reporting_engine, array $loop_result, MicroChaos_Cache_Analyzer $cache_analyzer): void { $payload = [ 'worker_id' => $this->config['worker_id'], + 'mode' => MicroChaos_Test_Mode::resolve($this->config), 'results' => $reporting_engine->get_results(), 'completed' => $loop_result['completed'], 'run_by_duration' => $loop_result['run_by_duration'], @@ -3854,8 +3957,9 @@ public function run(array $assoc_args, array $config): array { ); } - MicroChaos_Log::log("🔀 Overlap run: {$n} sequential processes launched together."); - MicroChaos_Log::log(" Combined Throughput is not a Phase 4 RPS — size on a sequential --cache-bust run."); + $mode = MicroChaos_Test_Mode::resolve(['concurrency' => $n]); + MicroChaos_Test_Mode::announce($mode); + MicroChaos_Log::log(" Each process runs the sequential test; the overlap is the ensemble."); if (!empty($config['warm_cache'])) { $this->warm_once($config); @@ -4114,10 +4218,9 @@ private function report_merged(array $payloads, int $n): void { 'duration_formatted' => $this->format_duration($duration), 'total_requests' => $completed, 'throughput_rps' => $rps, + 'mode' => MicroChaos_Test_Mode::resolve(['concurrency' => $n]), ]; - MicroChaos_Log::log("📊 Overlap summary ({$n} processes)"); - MicroChaos_Log::log(" Do not size workers from this Throughput figure."); $engine->report_summary(null, null, null, $metrics); $this->report_merged_cache($payloads, $completed); diff --git a/tools/microchaos-cli/microchaos/bootstrap.php b/tools/microchaos-cli/microchaos/bootstrap.php index eed123c..db1cef9 100644 --- a/tools/microchaos-cli/microchaos/bootstrap.php +++ b/tools/microchaos-cli/microchaos/bootstrap.php @@ -71,6 +71,9 @@ private static function load_core_components() { // Load constants first require_once MICROCHAOS_CORE_PATH . '/constants.php'; + // Load the test-mode descriptor before anything that reports a run + require_once MICROCHAOS_CORE_PATH . '/test-mode.php'; + // Load interfaces (order matters - logger before components that use it) require_once MICROCHAOS_CORE_PATH . '/interfaces/logger.php'; require_once MICROCHAOS_CORE_PATH . '/interfaces/baseline-storage.php'; diff --git a/tools/microchaos-cli/microchaos/core/concurrency-runner.php b/tools/microchaos-cli/microchaos/core/concurrency-runner.php index 3f5d12d..1943bbb 100644 --- a/tools/microchaos-cli/microchaos/core/concurrency-runner.php +++ b/tools/microchaos-cli/microchaos/core/concurrency-runner.php @@ -132,8 +132,9 @@ public function run(array $assoc_args, array $config): array { ); } - MicroChaos_Log::log("🔀 Overlap run: {$n} sequential processes launched together."); - MicroChaos_Log::log(" Combined Throughput is not a Phase 4 RPS — size on a sequential --cache-bust run."); + $mode = MicroChaos_Test_Mode::resolve(['concurrency' => $n]); + MicroChaos_Test_Mode::announce($mode); + MicroChaos_Log::log(" Each process runs the sequential test; the overlap is the ensemble."); if (!empty($config['warm_cache'])) { $this->warm_once($config); @@ -392,10 +393,9 @@ private function report_merged(array $payloads, int $n): void { 'duration_formatted' => $this->format_duration($duration), 'total_requests' => $completed, 'throughput_rps' => $rps, + 'mode' => MicroChaos_Test_Mode::resolve(['concurrency' => $n]), ]; - MicroChaos_Log::log("📊 Overlap summary ({$n} processes)"); - MicroChaos_Log::log(" Do not size workers from this Throughput figure."); $engine->report_summary(null, null, null, $metrics); $this->report_merged_cache($payloads, $completed); diff --git a/tools/microchaos-cli/microchaos/core/orchestrators/loadtest-orchestrator.php b/tools/microchaos-cli/microchaos/core/orchestrators/loadtest-orchestrator.php index 5f5d7ae..15ab61f 100644 --- a/tools/microchaos-cli/microchaos/core/orchestrators/loadtest-orchestrator.php +++ b/tools/microchaos-cli/microchaos/core/orchestrators/loadtest-orchestrator.php @@ -161,10 +161,13 @@ public function execute(): array { // Log test start $this->log_test_start($config, $endpoint_list, $integration_logger); - // Announce the measurement mode: the two modes answer different questions - // and the numbers are not comparable, so the log has to say which ran. - if ($config['cache_bust']) { - MicroChaos_Log::log("🚫 Cache busting enabled — every request gets a unique URL, so this measures origin cost, not cache performance."); + // Announce the measurement mode: the modes answer different questions and + // their numbers are not interchangeable, so the log has to say which ran. + // Fan-out children are skipped, because the parent labels the ensemble and + // a child announcing itself as a sizing run would contradict it. + $mode = MicroChaos_Test_Mode::resolve($config); + if (empty($config['worker_id'])) { + MicroChaos_Test_Mode::announce($mode); } // Warm cache if specified @@ -205,6 +208,7 @@ public function execute(): array { $loop_result['completed'], $reporting_engine->get_results() ); + $execution_metrics['mode'] = $mode; // Handle baseline comparison $perf_baseline = $config['compare_baseline'] @@ -299,6 +303,7 @@ public function execute(): array { private function write_results_json(string $path, MicroChaos_Reporting_Engine $reporting_engine, array $loop_result, MicroChaos_Cache_Analyzer $cache_analyzer): void { $payload = [ 'worker_id' => $this->config['worker_id'], + 'mode' => MicroChaos_Test_Mode::resolve($this->config), 'results' => $reporting_engine->get_results(), 'completed' => $loop_result['completed'], 'run_by_duration' => $loop_result['run_by_duration'], diff --git a/tools/microchaos-cli/microchaos/core/reporting-engine.php b/tools/microchaos-cli/microchaos/core/reporting-engine.php index e06bae0..8f7a8ec 100644 --- a/tools/microchaos-cli/microchaos/core/reporting-engine.php +++ b/tools/microchaos-cli/microchaos/core/reporting-engine.php @@ -228,6 +228,15 @@ public function report_summary(?array $baseline = null, ?array $provided_summary MicroChaos_Log::log("📊 Load Test Summary"); MicroChaos_Log::log(" ═══════════════════════════════════════════════════"); + // The summary is what gets pasted into an audit, so it names the mode + // it measured. Without it, three runs produce three near-identical + // blocks and only the operator's memory says which is the sizing one. + if (!empty($execution_metrics['mode'])) { + MicroChaos_Log::log(" Mode: {$execution_metrics['mode']['label']}"); + MicroChaos_Log::log(" {$execution_metrics['mode']['sizing']}"); + MicroChaos_Log::log(" ───────────────────────────────────────────────────"); + } + // Test Execution Metrics section if ($execution_metrics) { MicroChaos_Log::log(" Test Execution:"); diff --git a/tools/microchaos-cli/microchaos/core/test-mode.php b/tools/microchaos-cli/microchaos/core/test-mode.php new file mode 100644 index 0000000..8cc73c5 --- /dev/null +++ b/tools/microchaos-cli/microchaos/core/test-mode.php @@ -0,0 +1,114 @@ + $config Parsed config + * @return array{id: string, label: string, measures: string, sizing: string, sizing_ok: bool} + */ + public static function resolve(array $config): array { + $concurrency = (int) ($config['concurrency'] ?? 1); + + if ($concurrency > 1) { + return [ + 'id' => 'overlap', + 'label' => "Overlap ({$concurrency} processes launched together)", + 'measures' => 'How the site behaves when requests actually share workers.', + 'sizing' => 'Not a sizing input. Combined Throughput mixes queueing and ' + . "{$concurrency} load generators into one figure.", + 'sizing_ok' => false, + ]; + } + + $cache_bust = !empty($config['cache_bust']); + $warm_cache = !empty($config['warm_cache']); + + // --cache-bust wins the label when both are set. The orchestrator already + // warns that warming is pointless alongside it, and the run does measure + // origin cost, so calling it anything else would be the bigger lie. + if ($cache_bust) { + return [ + 'id' => 'origin-cost', + 'label' => 'Origin cost (sequential, cache-busted)', + 'measures' => 'What one uncached request costs the origin, one request at a time.', + 'sizing' => 'Throughput from this run is the Phase 4 sizing input.', + 'sizing_ok' => true, + ]; + } + + if ($warm_cache) { + return [ + 'id' => 'cache-effectiveness', + 'label' => 'Cache effectiveness (sequential, warmed)', + 'measures' => 'What the cache serves once the URL is warm.', + 'sizing' => 'Not a sizing input. Compare it against a --cache-bust run to see ' + . 'what the cache is buying the customer.', + 'sizing_ok' => false, + ]; + } + + return [ + 'id' => 'uncontrolled', + 'label' => 'Sequential, cache state not controlled', + 'measures' => 'A mix. Some requests may be served from cache and some not, ' + . 'and the split is not recorded.', + 'sizing' => 'Neither origin cost nor cache effectiveness. Add --cache-bust for ' + . 'sizing, or --warm-cache to measure the cache.', + 'sizing_ok' => false, + ]; + } + + /** + * Announce the mode before the run starts, so the operator sees what is + * being measured while it happens rather than only in the summary. + * + * @param array{label: string, measures: string, sizing: string, sizing_ok: bool} $mode + */ + public static function announce(array $mode): void { + MicroChaos_Log::log("🧪 Test mode: {$mode['label']}"); + MicroChaos_Log::log(" Measures: {$mode['measures']}"); + + if ($mode['sizing_ok']) { + MicroChaos_Log::log(" Sizing: {$mode['sizing']}"); + return; + } + + MicroChaos_Log::warning(" Sizing: {$mode['sizing']}"); + } + + /** + * The one-line form carried in the summary block. + * + * @param array{label: string, sizing: string} $mode + * @return string + */ + public static function summary_line(array $mode): string { + return "{$mode['label']} — {$mode['sizing']}"; + } +} diff --git a/tools/microchaos-cli/tests/Unit/TestModeTest.php b/tools/microchaos-cli/tests/Unit/TestModeTest.php new file mode 100644 index 0000000..d938022 --- /dev/null +++ b/tools/microchaos-cli/tests/Unit/TestModeTest.php @@ -0,0 +1,98 @@ + true]); + + $this->assertSame('origin-cost', $mode['id']); + $this->assertTrue($mode['sizing_ok']); + } + + #[Test] + public function warm_cache_measures_the_cache_and_is_not_a_sizing_input(): void + { + $mode = \MicroChaos_Test_Mode::resolve(['warm_cache' => true]); + + $this->assertSame('cache-effectiveness', $mode['id']); + $this->assertFalse($mode['sizing_ok']); + } + + #[Test] + public function concurrency_above_one_is_the_overlap_run(): void + { + $mode = \MicroChaos_Test_Mode::resolve(['concurrency' => 4, 'cache_bust' => true]); + + $this->assertSame('overlap', $mode['id']); + $this->assertFalse($mode['sizing_ok']); + } + + #[Test] + public function overlap_names_the_process_count_so_the_summary_is_self_describing(): void + { + $mode = \MicroChaos_Test_Mode::resolve(['concurrency' => 4]); + + $this->assertStringContainsString('4 processes', $mode['label']); + } + + #[Test] + public function concurrency_of_one_is_not_an_overlap_run(): void + { + $mode = \MicroChaos_Test_Mode::resolve(['concurrency' => 1, 'cache_bust' => true]); + + $this->assertSame('origin-cost', $mode['id']); + } + + #[Test] + public function neither_cache_flag_is_reported_as_uncontrolled(): void + { + $mode = \MicroChaos_Test_Mode::resolve([]); + + $this->assertSame('uncontrolled', $mode['id']); + $this->assertFalse($mode['sizing_ok']); + } + + #[Test] + public function cache_bust_wins_when_both_cache_flags_are_set(): void + { + $mode = \MicroChaos_Test_Mode::resolve(['cache_bust' => true, 'warm_cache' => true]); + + $this->assertSame('origin-cost', $mode['id']); + } + + #[Test] + public function every_mode_carries_a_label_and_a_sizing_statement(): void + { + $configs = [ + ['cache_bust' => true], + ['warm_cache' => true], + ['concurrency' => 3], + [], + ]; + + foreach ($configs as $config) { + $mode = \MicroChaos_Test_Mode::resolve($config); + + $this->assertNotSame('', $mode['label']); + $this->assertNotSame('', $mode['measures']); + $this->assertNotSame('', $mode['sizing']); + $this->assertNotSame('', \MicroChaos_Test_Mode::summary_line($mode)); + } + } +} diff --git a/tools/microchaos-cli/tests/bootstrap.php b/tools/microchaos-cli/tests/bootstrap.php index f8ead9a..9fca189 100644 --- a/tools/microchaos-cli/tests/bootstrap.php +++ b/tools/microchaos-cli/tests/bootstrap.php @@ -85,6 +85,9 @@ function wp_generate_uuid4(): string { // Core constants (other components may reference these) require_once MICROCHAOS_CORE_PATH . '/constants.php'; +// Test-mode descriptor: pure config-to-label mapping, no runtime needed. +require_once MICROCHAOS_CORE_PATH . '/test-mode.php'; + // Interfaces (load before implementations) require_once MICROCHAOS_CORE_PATH . '/interfaces/logger.php'; require_once MICROCHAOS_CORE_PATH . '/interfaces/baseline-storage.php'; From 7a748d843cd87d9e4e41762230bcb13598aa6bd6 Mon Sep 17 00:00:00 2001 From: Dave Ventesca Date: Fri, 28 Aug 2026 12:37:09 -0400 Subject: [PATCH 3/3] microchaos: delete the orphaned curl_multi request path fire_requests_async() had been unreferenced since v3.0.0 removed the one-process --concurrency flag, and its continued presence was read by both the defect report and the audit doctrine as concurrency the tool had but declined to use. Fan-out is the mechanism that actually overlaps on Pressable, so the dead path goes and the runner's docblock records why it must not come back. Takes with it the two private helpers only that path used, classify_curl_failure() and process_curl_headers(), and their tests. Every request now goes through wp_remote_request(). Co-authored-by: Cursor --- tools/microchaos-cli/README.md | 4 + tools/microchaos-cli/dist/microchaos-cli.php | 158 ------------------ .../microchaos/core/concurrency-runner.php | 14 +- .../microchaos/core/request-generator.php | 158 ------------------ .../Unit/RequestGeneratorTimeoutTest.php | 24 +-- 5 files changed, 18 insertions(+), 340 deletions(-) diff --git a/tools/microchaos-cli/README.md b/tools/microchaos-cli/README.md index a36b23c..43195ac 100644 --- a/tools/microchaos-cli/README.md +++ b/tools/microchaos-cli/README.md @@ -44,6 +44,10 @@ stampede races, HIT/regen splits on an already-warm URL). - Sequential default is unchanged. Existing 4.1.0 commands keep their numbers. - Every run now names its **test mode**, on screen as it starts and again in the summary, so a pasted report says what it measured. +- The orphaned `fire_requests_async()` curl_multi implementation is **deleted**. + It had been unreferenced since v3.0.0 and kept implying the tool had + concurrency it was choosing not to use. Fan-out replaces it; do not + reintroduce the one-process curl_multi path. ### Test modes diff --git a/tools/microchaos-cli/dist/microchaos-cli.php b/tools/microchaos-cli/dist/microchaos-cli.php index 365678c..6ee76a0 100644 --- a/tools/microchaos-cli/dist/microchaos-cli.php +++ b/tools/microchaos-cli/dist/microchaos-cli.php @@ -1433,129 +1433,6 @@ public function set_user_agent(string $user_agent): void { $this->custom_user_agent = $user_agent; } - /** - * Fire an asynchronous batch of requests - * - * @param string $url Target URL - * @param string|null $log_path Optional path for logging - * @param array|null $cookies Optional cookies for authentication - * @param int $current_burst Number of concurrent requests to fire - * @param string $method HTTP method - * @param string|null $body Request body for POST/PUT - * @return array Results of the requests - */ - public function fire_requests_async(string $url, ?string $log_path, ?array $cookies, int $current_burst, string $method = 'GET', ?string $body = null): array { - $results = []; - $multi_handle = curl_multi_init(); - $curl_handles = []; - - for ($i = 0; $i < $current_burst; $i++) { - $curl = curl_init($url); - curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); - curl_setopt($curl, CURLOPT_TIMEOUT, $this->timeout); - curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $method); - - // Prepare headers array - $headers = [ - 'User-Agent: ' . $this->get_user_agent(), - ]; - - // Add custom headers if any - if (!empty($this->custom_headers)) { - foreach ($this->custom_headers as $name => $value) { - $headers[] = "$name: $value"; - } - } - - curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); - - // For cache header collection - if ($this->collect_cache_headers) { - curl_setopt($curl, CURLOPT_HEADER, true); - } - - // Handle body data - if ($body) { - if ($this->is_json($body)) { - // Add content-type header to existing headers - $headers[] = 'Content-Type: application/json'; - curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); - curl_setopt($curl, CURLOPT_POSTFIELDS, $body); - } else { - curl_setopt($curl, CURLOPT_POSTFIELDS, $body); - } - } - - if ($cookies) { - $selected = MicroChaos_Authentication_Manager::is_multi_auth($cookies) - ? MicroChaos_Authentication_Manager::select_random_session($cookies) - : $cookies; - curl_setopt($curl, CURLOPT_COOKIE, MicroChaos_Authentication_Manager::format_for_curl($selected)); - } - curl_setopt($curl, CURLOPT_URL, $url); - $start = microtime(true); // record start time for this request - curl_multi_add_handle($multi_handle, $curl); - $curl_handles[] = ['handle' => $curl, 'url' => $url, 'start' => $start]; - } - - do { - curl_multi_exec($multi_handle, $active); - curl_multi_select($multi_handle); - } while ($active); - - foreach ($curl_handles as $entry) { - $curl = $entry['handle']; - $url = $entry['url']; - $start = $entry['start']; - $response = curl_multi_getcontent($curl); - $end = microtime(true); - $duration = round($end - $start, 4); - $info = curl_getinfo($curl); - $code = $info['http_code'] ?: self::classify_curl_failure(curl_errno($curl)); - - // Extract body for GraphQL error detection - // Note: CURLOPT_HEADER is only set when collect_cache_headers is enabled - $response_body = $response; - if ($this->collect_cache_headers && $response) { - $header_size = $info['header_size']; - $header = substr($response, 0, $header_size); - $response_body = substr($response, $header_size); - $this->process_curl_headers($header); - } - - // Detect GraphQL errors in response body - $graphql_errors = $this->detect_graphql_errors($response_body); - - $message = "⏱ MicroChaos Request | Time: {$duration}s | Code: {$code} | URL: $url | Method: $method"; - error_log($message); - - if ($log_path) { - $this->log_to_file($message, $log_path); - } - - if (class_exists('WP_CLI')) { - $cache_display = ''; - if ($this->collect_cache_headers && !empty($this->last_request_cache_headers)) { - $cache_display = ' ' . $this->format_cache_headers_for_display($this->last_request_cache_headers); - } - $gql_display = $graphql_errors > 0 ? " [GQL errors: {$graphql_errors}]" : ''; - MicroChaos_Log::log("-> {$code} in {$duration}s{$cache_display}{$gql_display}"); - } - - $results[] = [ - 'time' => $duration, - 'code' => $code, - 'graphql_errors' => $graphql_errors, - ]; - - curl_multi_remove_handle($multi_handle, $curl); - curl_close($curl); - } - - curl_multi_close($multi_handle); - return $results; - } - /** * Fire a single request * @@ -1646,22 +1523,6 @@ public function fire_request(string $url, ?string $log_path = null, ?array $cook ]; } - /** - * Classify a cURL failure - * - * A timeout and a refused connection are different findings — one says the - * site is slow, the other says it is unreachable — so they get separate - * sentinels rather than sharing 'ERROR'. - * - * @param int $errno cURL error number from curl_errno() - * @return string Status sentinel for the result row - */ - private static function classify_curl_failure(int $errno): string { - return CURLE_OPERATION_TIMEOUTED === $errno - ? self::STATUS_TIMEOUT - : self::STATUS_ERROR; - } - /** * Classify a WP_Error returned by wp_remote_request() * @@ -1699,25 +1560,6 @@ public function resolve_endpoint(string $slug): string|false { } } - /** - * Process headers from cURL response for cache analysis - * - * @param string $header_text Raw header text from cURL response - */ - private function process_curl_headers(string $header_text): void { - $headers = []; - foreach(explode("\r\n", $header_text) as $line) { - if (strpos($line, ':') !== false) { - list($key, $value) = explode(':', $line, 2); - $key = strtolower(trim($key)); - $value = trim($value); - $headers[$key] = $value; - } - } - - $this->collect_cache_header_data($headers); - } - /** * Collect and catalog cache headers from the response * diff --git a/tools/microchaos-cli/microchaos/core/concurrency-runner.php b/tools/microchaos-cli/microchaos/core/concurrency-runner.php index 1943bbb..3b0c534 100644 --- a/tools/microchaos-cli/microchaos/core/concurrency-runner.php +++ b/tools/microchaos-cli/microchaos/core/concurrency-runner.php @@ -3,9 +3,17 @@ * Concurrency Runner * * Overlap is N independent WP-CLI processes launched at the same instant, - * each running the existing sequential loadtest. One process cannot overlap - * (`--burst` is pacing; fire_requests_async() is unused). This is the method - * that produced real overlap on Pressable — not curl_multi in one process. + * each running the existing sequential loadtest. One process cannot overlap: + * `--burst` is pacing, not concurrency. + * + * curl_multi inside one process was tried and rejected. Pressable's loopback + * serializes it, so the requests do not actually overlap and the run looks + * like an ordinary sequential test with worse bookkeeping. v3.0.0 shipped that + * mechanism as `--concurrency` and removed it for the same reason. The dead + * implementation (`fire_requests_async()`) survived until 4.2.0 and was + * deleted with this feature, because leaving it there kept suggesting the tool + * had concurrency it was declining to use. Process fan-out is the method that + * produced real overlap on Pressable; do not reintroduce the curl_multi path. * * --count / --duration are per process. --count=1 --concurrency=4 is a * four-request stampede. Combined wall-clock throughput is not a Phase 4 RPS. diff --git a/tools/microchaos-cli/microchaos/core/request-generator.php b/tools/microchaos-cli/microchaos/core/request-generator.php index 5fd4b89..78f2992 100644 --- a/tools/microchaos-cli/microchaos/core/request-generator.php +++ b/tools/microchaos-cli/microchaos/core/request-generator.php @@ -107,129 +107,6 @@ public function set_user_agent(string $user_agent): void { $this->custom_user_agent = $user_agent; } - /** - * Fire an asynchronous batch of requests - * - * @param string $url Target URL - * @param string|null $log_path Optional path for logging - * @param array|null $cookies Optional cookies for authentication - * @param int $current_burst Number of concurrent requests to fire - * @param string $method HTTP method - * @param string|null $body Request body for POST/PUT - * @return array Results of the requests - */ - public function fire_requests_async(string $url, ?string $log_path, ?array $cookies, int $current_burst, string $method = 'GET', ?string $body = null): array { - $results = []; - $multi_handle = curl_multi_init(); - $curl_handles = []; - - for ($i = 0; $i < $current_burst; $i++) { - $curl = curl_init($url); - curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); - curl_setopt($curl, CURLOPT_TIMEOUT, $this->timeout); - curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $method); - - // Prepare headers array - $headers = [ - 'User-Agent: ' . $this->get_user_agent(), - ]; - - // Add custom headers if any - if (!empty($this->custom_headers)) { - foreach ($this->custom_headers as $name => $value) { - $headers[] = "$name: $value"; - } - } - - curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); - - // For cache header collection - if ($this->collect_cache_headers) { - curl_setopt($curl, CURLOPT_HEADER, true); - } - - // Handle body data - if ($body) { - if ($this->is_json($body)) { - // Add content-type header to existing headers - $headers[] = 'Content-Type: application/json'; - curl_setopt($curl, CURLOPT_HTTPHEADER, $headers); - curl_setopt($curl, CURLOPT_POSTFIELDS, $body); - } else { - curl_setopt($curl, CURLOPT_POSTFIELDS, $body); - } - } - - if ($cookies) { - $selected = MicroChaos_Authentication_Manager::is_multi_auth($cookies) - ? MicroChaos_Authentication_Manager::select_random_session($cookies) - : $cookies; - curl_setopt($curl, CURLOPT_COOKIE, MicroChaos_Authentication_Manager::format_for_curl($selected)); - } - curl_setopt($curl, CURLOPT_URL, $url); - $start = microtime(true); // record start time for this request - curl_multi_add_handle($multi_handle, $curl); - $curl_handles[] = ['handle' => $curl, 'url' => $url, 'start' => $start]; - } - - do { - curl_multi_exec($multi_handle, $active); - curl_multi_select($multi_handle); - } while ($active); - - foreach ($curl_handles as $entry) { - $curl = $entry['handle']; - $url = $entry['url']; - $start = $entry['start']; - $response = curl_multi_getcontent($curl); - $end = microtime(true); - $duration = round($end - $start, 4); - $info = curl_getinfo($curl); - $code = $info['http_code'] ?: self::classify_curl_failure(curl_errno($curl)); - - // Extract body for GraphQL error detection - // Note: CURLOPT_HEADER is only set when collect_cache_headers is enabled - $response_body = $response; - if ($this->collect_cache_headers && $response) { - $header_size = $info['header_size']; - $header = substr($response, 0, $header_size); - $response_body = substr($response, $header_size); - $this->process_curl_headers($header); - } - - // Detect GraphQL errors in response body - $graphql_errors = $this->detect_graphql_errors($response_body); - - $message = "⏱ MicroChaos Request | Time: {$duration}s | Code: {$code} | URL: $url | Method: $method"; - error_log($message); - - if ($log_path) { - $this->log_to_file($message, $log_path); - } - - if (class_exists('WP_CLI')) { - $cache_display = ''; - if ($this->collect_cache_headers && !empty($this->last_request_cache_headers)) { - $cache_display = ' ' . $this->format_cache_headers_for_display($this->last_request_cache_headers); - } - $gql_display = $graphql_errors > 0 ? " [GQL errors: {$graphql_errors}]" : ''; - MicroChaos_Log::log("-> {$code} in {$duration}s{$cache_display}{$gql_display}"); - } - - $results[] = [ - 'time' => $duration, - 'code' => $code, - 'graphql_errors' => $graphql_errors, - ]; - - curl_multi_remove_handle($multi_handle, $curl); - curl_close($curl); - } - - curl_multi_close($multi_handle); - return $results; - } - /** * Fire a single request * @@ -320,22 +197,6 @@ public function fire_request(string $url, ?string $log_path = null, ?array $cook ]; } - /** - * Classify a cURL failure - * - * A timeout and a refused connection are different findings — one says the - * site is slow, the other says it is unreachable — so they get separate - * sentinels rather than sharing 'ERROR'. - * - * @param int $errno cURL error number from curl_errno() - * @return string Status sentinel for the result row - */ - private static function classify_curl_failure(int $errno): string { - return CURLE_OPERATION_TIMEOUTED === $errno - ? self::STATUS_TIMEOUT - : self::STATUS_ERROR; - } - /** * Classify a WP_Error returned by wp_remote_request() * @@ -373,25 +234,6 @@ public function resolve_endpoint(string $slug): string|false { } } - /** - * Process headers from cURL response for cache analysis - * - * @param string $header_text Raw header text from cURL response - */ - private function process_curl_headers(string $header_text): void { - $headers = []; - foreach(explode("\r\n", $header_text) as $line) { - if (strpos($line, ':') !== false) { - list($key, $value) = explode(':', $line, 2); - $key = strtolower(trim($key)); - $value = trim($value); - $headers[$key] = $value; - } - } - - $this->collect_cache_header_data($headers); - } - /** * Collect and catalog cache headers from the response * diff --git a/tools/microchaos-cli/tests/Unit/RequestGeneratorTimeoutTest.php b/tools/microchaos-cli/tests/Unit/RequestGeneratorTimeoutTest.php index aa4563a..a78a70a 100644 --- a/tools/microchaos-cli/tests/Unit/RequestGeneratorTimeoutTest.php +++ b/tools/microchaos-cli/tests/Unit/RequestGeneratorTimeoutTest.php @@ -95,27 +95,9 @@ public function cache_header_collection_still_works_alongside_a_timeout(): void // Failure classification // ========================================================================= - #[Test] - public function a_curl_timeout_is_labelled_as_a_timeout(): void - { - $this->assertSame( - \MicroChaos_Request_Generator::STATUS_TIMEOUT, - $this->classify('classify_curl_failure', CURLE_OPERATION_TIMEOUTED) - ); - } - - #[Test] - public function other_curl_failures_stay_generic_errors(): void - { - // A refused connection or an unresolvable host is a different finding - // from a slow response and must not be reported as one. - foreach ([CURLE_COULDNT_CONNECT, CURLE_COULDNT_RESOLVE_HOST] as $errno) { - $this->assertSame( - \MicroChaos_Request_Generator::STATUS_ERROR, - $this->classify('classify_curl_failure', $errno) - ); - } - } + // The cURL classifier was removed with fire_requests_async() in 4.2.0. Every + // request now goes through wp_remote_request(), so the WP_Error path below is + // the only classification the tool performs. #[Test] public function a_wp_error_mentioning_a_timeout_is_labelled_as_a_timeout(): void