diff --git a/.spelling b/.spelling index b0d79ac3a..34d3e98df 100644 --- a/.spelling +++ b/.spelling @@ -931,6 +931,8 @@ symlinked uninherited closers unmanaged +nonblocking SHA upserts EWMA +Untimed diff --git a/Cargo.lock b/Cargo.lock index a7b65564e..3a86dfb43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -540,11 +540,13 @@ dependencies = [ "assert_cmd", "cargo_metadata", "clap", + "command-group", "mutants", "ohno", "predicates", "serde_json", "tempfile", + "toml", ] [[package]] @@ -890,6 +892,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "command-group" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68fa787550392a9d58f44c21a3022cfb3ea3e2458b7f85d3b399d0ceeccf409" +dependencies = [ + "nix 0.27.1", + "winapi", +] + [[package]] name = "compact_str" version = "0.10.0" @@ -3088,7 +3100,7 @@ dependencies = [ "combine", "libc", "mach2", - "nix", + "nix 0.30.1", "sysctl", "thiserror 2.0.20", "widestring", @@ -3110,6 +3122,17 @@ version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "add0ac067452ff1aca8c5002111bd6b1c895baee6e45fcbc44e0193aea17be56" +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "libc", +] + [[package]] name = "nix" version = "0.30.1" diff --git a/Cargo.toml b/Cargo.toml index 9fe00d78c..35df70590 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ cel = { version = "0.14.4", default-features = false } chrono = { version = "0.4.45", default-features = false } clap = { version = "4.6.6", default-features = false } clap_complete = { version = "4.6.9", default-features = false } +command-group = { version = "5.0.1", default-features = false } compact_str = { version = "0.10.0", default-features = false } csv = { version = "1.4.0", default-features = false } duct = { version = "1.1.1", default-features = false } diff --git a/crates/cargo-each/Cargo.toml b/crates/cargo-each/Cargo.toml index 586c77e2c..58cb0df67 100644 --- a/crates/cargo-each/Cargo.toml +++ b/crates/cargo-each/Cargo.toml @@ -19,14 +19,16 @@ repository = "https://github.com/microsoft/ox-tools/tree/main/crates/cargo-each" [dependencies] cargo_metadata = { workspace = true } clap = { workspace = true, features = ["derive", "std", "help", "usage", "error-context"] } +command-group = { workspace = true } mutants = { workspace = true } ohno = { workspace = true, features = ["app-err"] } serde_json = { workspace = true, features = ["std"] } +tempfile = { workspace = true } +toml = { workspace = true, features = ["parse", "serde"] } [dev-dependencies] assert_cmd = { workspace = true } predicates = { workspace = true } -tempfile = { workspace = true } # >>> anvil-managed: anvil-lints [lints] diff --git a/crates/cargo-each/README.md b/crates/cargo-each/README.md index c320f4e3b..743cf2852 100644 --- a/crates/cargo-each/README.md +++ b/crates/cargo-each/README.md @@ -44,16 +44,19 @@ directly (argv, not a shell string) after substituting placeholders. * `-p` / `--package ` — select a member. Repeatable. `SPEC` is a package name, a `name@version` spec, or a Unix glob (`tokio-*`). +* `--package-file ` — read package specs from a UTF-8 file, one per + nonempty line. Repeatable; specs are unioned with `--package`. An empty + file explicitly selects no members. One leading UTF-8 byte-order mark is + ignored. * `--workspace` / `--all` — select every workspace member. * `--exclude ` — drop a member (with `--workspace`). Repeatable. * `--none` — explicitly select zero members (a no-op that exits 0). When nothing is named the default is cargo `default-members`, exactly like `cargo build`; pass `--workspace` for every member. A selector that -matches no member is an error, so typos fail loudly. A computed selection -(for example a CI affected-packages set) is fed in as ordinary flags via -shell expansion — `cargo-each` has no file or environment-variable source -of its own. +matches no member is an error, so typos fail loudly. Package files contain +package specs only: comments, command-line tokens, malformed input, and +missing, unreadable, or non-UTF-8 files are errors. ### Filters @@ -82,9 +85,16 @@ can be double-quoted. Expression atoms: `--target-required-feature` further narrows targets. `--keep-going` runs every invocation and exits non-zero if any failed -(default is fail-fast); `--chdir` runs each per-package or per-target -command from that member crate root; `--dry-run` prints commands without -running them. +(default is fail-fast). `--jobs ` bounds concurrent per-package or +per-target work. Omitting it runs exactly one invocation at a time; `auto` +resolves once to the machine’s available parallelism. Detection failure is +reported explicitly without falling back. `--timeout ` terminates +each invocation’s Windows job object or Unix process group independently +(`250ms`, `30s`, or `2m`). Unix descendants can escape a process group by +starting a new session, so timeout cleanup is best-effort for those escaped +descendants. +`--chdir` runs each per-package or per-target command from that member crate +root; `--dry-run` prints commands without running them. ### Placeholders @@ -99,6 +109,9 @@ Substituted inside each command argument: * `{packages}` — the cargo selection flags for the resolved set (`--workspace` for the whole workspace, else `--package name@version …`); valid only in `--once` mode and only as a standalone argument. +* `{workspace-rust-version}` — the root `[workspace.package].rust-version`, + or root `[package].rust-version` in a single-package repository; valid in + every mode. Using a placeholder in the wrong mode is a usage error. Only the tokens above are interpreted; any other `{…}` sequence (a typo, or a literal brace @@ -110,12 +123,57 @@ no brace-escape, so this passthrough is part of the contract. An empty resolved selection (via `--none`, or a filter that removes every member) is a **successful no-op**: `cargo-each` prints a one-line note and exits 0. This is what lets callers drop bespoke nothing-to-do guards. +Workspace Rust-version validation is lazy: it runs only when the command +uses `{workspace-rust-version}` and the resolved plan has work, then requires +every member’s resolved minimum to be present and no newer than the root +floor. Placeholder mode validation still runs before an empty-plan no-op. + +The effective worker count is the requested `--jobs` value capped by plan +size. An effective count of one uses sequential +execution with inherited standard input, output, and error even when the +requested value was larger. A genuinely parallel count disconnects child input and +buffers stdout and stderr; complete blocks are emitted in deterministic +plan order. Fail-fast stops launching after the first +observed failure, waits for running work, and chooses the final failure by +plan order. `--keep-going` runs the complete plan. Worker panics and +unexpected worker-channel disconnections become infrastructure-failure +outcomes instead of blocking the scheduler. Worker launch failures retain +output already collected at earlier plan indices. Parallel work runs in +plan-contiguous waves capped by the effective worker count; each completed +wave is emitted and dropped before the next wave starts, bounding retained +temporary-file storage. Untimed effective-one execution uses an ordinary +child, preserving terminal foreground behavior and Ctrl-C delivery; a +post-spawn wait failure gets bounded child cleanup and reaper ownership. +Timed and genuinely parallel commands use a job or process group. Without +`--timeout`, cargo-each observes only the leader and does not kill background +descendants. Every genuinely parallel invocation redirects stdout and +stderr directly to separate unique temporary files. +Child writers and parent readers are separately reopened so parent seeks +cannot move descendant write positions. Cargo-each records each file’s +current length when the leader completes (or after timeout cleanup), then +reads exactly that finite snapshot in plan order without loading unbounded +output into memory. Later writes by background or escaped descendants are +outside the snapshot. RAII removes cargo-each’s directory entry, but a +preserved descendant can keep the backing storage allocated and growing +until its inherited writer closes. Capture create, reopen, length, seek, and +read failures are infrastructure failures. + +Timed-out group termination gets a bounded 250 ms reap grace. If the group +still has not completed, its handle moves to a cargo-each-local polling +reaper started before any command. The reaper checks every retained group +without blocking on one child, remains the wait owner after the caller +returns, and exits after all senders disconnect and retained groups are +collected. Interrupted observations are retried; terminal observation +errors are reported and removed. Reaper startup and handoff failures are +explicit infrastructure failures; a failed handoff retains the group handle +in a persistent fallback queue and starts an emergency polling reaper. Child commands inherit `PATH` explicitly. On Windows this makes relative program lookup honor the inherited `PATH` order instead of preferring an unrelated executable beside `cargo-each`. -Otherwise the exit code is the first failing command code (fail-fast), -`1` under `--keep-going` if any command failed, or `2` for a `cargo-each` -usage error (unknown selector, bad filter expression, misused placeholder). +Exit `0` means all work succeeded or there was no work. In fail-fast mode a +command failure returns its code, a timeout returns `1`, and usage, +configuration, spawn, or post-spawn infrastructure failures return `2`. +Under `--keep-going`, any failure maps the aggregate result to `1`. ## Examples diff --git a/crates/cargo-each/docs/design/README.md b/crates/cargo-each/docs/design/README.md index a5fbe1147..47092ad9f 100644 --- a/crates/cargo-each/docs/design/README.md +++ b/crates/cargo-each/docs/design/README.md @@ -48,18 +48,19 @@ target (with placeholder substitution), or exactly once for the whole set. flag surface is already familiar and the impact step's `--package name@version` output can be consumed verbatim. 2. **Absorb the CI skip/default dance.** A resolved-empty selection is a no-op - that exits 0 — no `--skip` sentinel in callers. cargo-each is entirely - flag-driven: a computed selection (an impact tier) is fed in as ordinary - `-p` / `--workspace` / `--none` flags via shell expansion, so cargo-each - stays agnostic about where the selectors came from and callers never write - a skip/default conditional. + that exits 0 — no `--skip` sentinel in callers. A computed selection can be + supplied as ordinary `-p` flags or as one Cargo package spec per line in a + `--package-file`, so callers do not need shell array expansion. cargo-each + stays agnostic about who produced the file and what the selection means. 3. **Three execution modes.** *per-package* (run the command once per member, substituting `{name}`/`{spec}`/`{version}`/`{manifest}`) covers per-manifest tools; *once* (run the command a single time when the set is non-empty) covers workspace-wide tools and single-invocation cargo commands, with a `{packages}` placeholder that expands to the cargo selection flags; and *per-target* runs once for each Cargo target of requested kinds, preserving - the package placeholders and adding `{target}`. + the package placeholders and adding `{target}`. The workspace-scoped + `{workspace-rust-version}` placeholder exposes the root compatibility floor + to commands that provision or validate a shared toolchain. 4. **A small, general filter language** (`--filter` and `--exclude-filter`) with `not`, `and`, `or`, and parentheses over cargo metadata — target kinds, publication state, declared features and dependencies, and @@ -67,7 +68,10 @@ target (with placeholder substitution), or exactly once for the whole set. recipes collapses to flags. 5. **Bare names for free.** `{name}` yields the un-qualified package name, so `@version` stripping disappears from callers even though the input carries it. -6. **Works identically locally and in CI**, on any platform, with no shell +6. **Bounded execution.** Per-package and per-target commands may run with a + caller-selected concurrency limit and timeout. Defaults remain sequential and + unbounded for backward compatibility. +7. **Works identically locally and in CI**, on any platform, with no shell dialect assumptions. **Open source**: ships from `ox-tools` to crates.io. ## 3. Non-Goals @@ -78,10 +82,12 @@ target (with placeholder substitution), or exactly once for the whole set. aggregation, llvm-cov's dual-config instrumentation, and the per-crate readme `doc2readme` reconciliation stay in their recipes. `cargo-each` owns only the selection → filter → iterate spine those recipes wrap. -- **Parallel scheduling / job pools.** Commands run sequentially. Parallelism, if - ever wanted, is a later, additive concern. - **A general templating engine.** Placeholder substitution is a fixed, small set of `{token}` replacements, not an expression language. +- **Tool installation or workflow orchestration.** cargo-each can execute + `rustup` or another installer when the caller asks it to, but it does not + decide which tools a repository needs, select binary versus source + installation, inspect Git, collect coverage, or understand Anvil tiers. - **A public library API.** `cargo-each` ships as an executable only. Its modules are crate-internal (`pub(crate)`), so there is no semver-committed library surface, no `check-external-types` obligation, and nothing to consume @@ -168,24 +174,31 @@ beyond placeholder substitution. | Flag | Meaning | |------|---------| | `-p`, `--package ` | Select a member. Repeatable. `SPEC` is a package name, a `name@version` spec, or a Unix glob (`tokio-*`), matching `cargo-coverage-gate`'s existing `-p` idiom. | +| `--package-file ` | Read package specs from a UTF-8 file, one spec per nonempty line. A single leading UTF-8 byte-order mark is ignored. Repeatable; specs are unioned with `--package`. A present empty file is an explicit empty selection. | | `--workspace`, `--all` | Select every workspace member. | | `--exclude ` | Remove a member from the selection (requires `--workspace`). Repeatable. | -| `--none` | Explicitly select zero members. Resolves to an empty set (a no-op, exit 0). Emitted by the impact hand-off when a tier is empty; replaces the `--skip` sentinel. | +| `--none` | Explicitly select zero members. Resolves to an empty set (a no-op, exit 0). | -A computed selection (e.g. an impact tier) is fed in as ordinary flags via -shell expansion — cargo-each has no `--from-file` / `--from-env` source, so it -stays agnostic about origin. See section 6 for the anvil hand-off. +A package file contains only package specs, not command-line tokens, comments, +an impact-tier name, or policy. `foo@1.2.3` has the same meaning whether it came +from `--package foo@1.2.3` or a file. A missing, unreadable, non-UTF-8, or +malformed file is an error. This keeps cargo-each independent of cargo-delta +while allowing cargo-delta output to be consumed without command substitution. **Resolution order.** The literal flags resolve to: 1. If `--none` appears anywhere → empty set. 2. Else if `--workspace`/`--all` appears → all members, minus `--exclude`. -3. Else if any `-p` matched → the matched members. -4. Else → `default-members` (exactly like `cargo build`; pass `--workspace` +3. Else if any direct or file-supplied spec exists → the matching members. +4. Else if at least one `--package-file` was supplied → empty set. +5. Else → `default-members` (exactly like `cargo build`; pass `--workspace` for the whole workspace). -A `-p` selector that matches no member is an error (same policy as +A selector that matches no member is an error (same policy as `cargo-coverage-gate`), so typos fail loudly rather than silently skipping. +An empty package file is different: it is the producer's explicit statement +that the computed set is empty and therefore exits successfully without +running the command. ### 4.2 Filters @@ -237,6 +250,8 @@ filtered set is empty, `cargo-each` exits 0, exactly like an empty selection. | `--each-target ` | **per-target**: run once for each selected member target of `KIND`. Repeatable; kinds are OR-combined and each target runs at most once. Mutually exclusive with `--once`. | | `--target-required-feature ` | In per-target mode, retain targets whose `required-features` contains `FEATURE`. Repeatable; values are AND-combined. Requires `--each-target`. | | `--keep-going` | Don't stop at the first failing command; run them all and exit non-zero if any failed. Default is fail-fast (exit with the first failure's code). | +| `--jobs ` | Run at most the positive integer `N` per-package or per-target commands concurrently. When omitted, the default is exactly `1`. `auto` resolves once during CLI parsing via `std::thread::available_parallelism()`; detection failure is an explicit usage error with no fallback. The effective worker count remains capped by the plan size. With `--once`, resolved values other than `1` are a usage error. | +| `--timeout ` | Terminate an invocation's Windows job object or Unix process group when it exceeds the positive duration, such as `30s` or `2m`. Applies independently to every invocation, including `--once`. Unix descendants can escape by starting a new session, so termination is best-effort for those escaped descendants. No timeout by default. | | `--chdir` | Run each per-package or per-target command from that member's crate root (the directory containing its `Cargo.toml`) instead of the caller's CWD. Combined with `--once` it is a usage error (exit 2). Placeholders stay absolute, so only *relative* args in the command shift to the member dir. | | `--manifest-path ` | Workspace root `Cargo.toml`. Defaults to auto-detection from CWD. | | `--dry-run` | Print the fully-substituted commands that *would* run, one per line, without executing. | @@ -253,11 +268,29 @@ Substituted inside each `ARG` of the command template: | `{manifest}` | absolute path to the member's `Cargo.toml` | per-package | | `{target}` | Cargo target name | per-target | | `{packages}` | the cargo selection flags for the resolved set: `--workspace` when the whole workspace was selected via `--workspace`/`--all` with no excludes **and no package filters applied**, else `--package name@version …` (one pair per member). Only valid as a standalone `ARG`; it expands to multiple tokens. | once | +| `{workspace-rust-version}` | Root `[workspace.package].rust-version`, or root `[package].rust-version` in a single-package repository. | all | Per-target mode accepts all per-package placeholders plus `{target}`. Using a per-package or per-target token in `--once` mode, `{target}` in per-package mode, or `{packages}` outside `--once` is a usage error. +Substitution scans each template argument once. Text inserted for one +placeholder is never scanned as another placeholder, so literal token-shaped +path components in manifest paths and other replacement values are preserved. + +`{workspace-rust-version}` is workspace-scoped rather than tied to one selected +member. Resolving it requires a root declaration. cargo-each also requires every +workspace member to expose a resolved `rust_version` no newer than the root +floor. Missing values, a member requiring a newer compiler, or a non-Rust +semantic version is a configuration error. Lower member minima are valid. This +matches the meaning of one compiler selected for a complete workspace; it is +not a per-package toolchain matrix. The validation is lazy: commands that do +not contain the placeholder do not require a workspace Rust version, and a +resolved plan with no invocations does not resolve or validate the value even +when the template contains the placeholder. Placeholder mode validation still +runs before that no-op decision, so misuse remains an exit-2 usage error on an +empty set. + Targets run in package-name order and then target-name order. A target matching more than one requested kind runs once. No matching targets is a successful no-op. @@ -265,18 +298,91 @@ no-op. ## 5. Semantics - **Exit codes.** `0` when every executed command succeeded *or* the set was - empty; the failing command's code (fail-fast) or `1` (`--keep-going` with any - failure — including a command that could not be spawned) otherwise; `2` for a - `cargo-each` usage/configuration error (unknown selector, bad filter expression, - unknown target kind, invalid mode combination, misused placeholder, - `--chdir` with `--once`, or — in fail-fast mode — a command that could not - be spawned at all). + empty. In fail-fast mode, a command failure returns that command's code, a + timeout returns `1`, and a post-spawn infrastructure failure (including + output capture, worker, wait, reaper, or cleanup failure) returns `2`. + Pre-execution usage/configuration and spawn failures also return `2`. Under + `--keep-going`, any command, timeout, spawn, or infrastructure failure maps + the aggregate result to `1`. - **Empty set is success.** Both an empty selection (`--none`, or an impact variable that resolved to nothing) and an empty *filtered* set exit 0 after a one-line note to stderr. This is what lets callers drop their `--skip` guards. - **No shell.** The command is spawned directly (argv, not a shell string), so there is no quoting/dialect surface. Placeholder expansion is textual and happens before spawn. +- **Bounded concurrency.** Omitting `--jobs` requests exactly one concurrent + invocation. A positive integer requests that fixed limit; `auto` resolves + exactly once during CLI parsing to the machine's available parallelism and + fails explicitly if detection is unavailable. The scheduler caps every + request by the plan size. With an effective job + count above one, output from each invocation is buffered and emitted as one + block in deterministic plan order. Fail-fast stops launching new work after + the first observed failure and waits for already-running children; + `--keep-going` launches the complete plan. The final failure is chosen by + plan order, not scheduler timing. Requested parallelism does not by itself + select this captured mode: when plan-size capping leaves + an effective worker count of one, cargo-each uses the sequential path and the + child inherits standard input, output, and error. With a genuinely parallel + effective worker count, child standard input is disconnected (`null`) so + workers cannot race to consume the caller's input; output and error are + captured for deterministic emission. A worker panic is converted into an + infrastructure-failure outcome; each worker has a dedicated completion + channel, so an unexpected exit is observable as disconnection rather than + leaving the scheduler blocked forever. A worker-thread launch failure is + represented as an infrastructure outcome at that invocation's plan index, + so output already collected from earlier invocations is still emitted. + Parallel work runs in plan-contiguous waves capped by the effective worker + count. A wave is fully observed, emitted, and dropped before the next wave + starts, so the number of retained invocation captures is also capped by the + effective worker count. + Untimed effective-one execution uses an ordinary child so inherited terminal + streams, foreground-group behavior, and Ctrl-C delivery match direct command + execution. It spawns and waits separately; a post-spawn observation failure + receives bounded direct-child termination and transfers an unreaped handle + to the local polling reaper. Timed and genuinely parallel commands use a + Windows job or Unix process group. Without `--timeout`, cargo-each observes + only the launched leader and does not kill ordinary background descendants. +- **Parallel capture uses finite temporary-file snapshots.** Every genuinely + parallel invocation redirects stdout and stderr directly to separate unique + temporary files before group spawn; no pipe-reader threads are created. The + child writer and parent reader are separately reopened so parent seeks cannot + move a descendant's write position. The parent records each file's current + length when the leader completes, or after timeout cleanup completes. + Plan-order emission seeks to the beginning and streams exactly that many + bytes, so memory does not scale with command output and output is not + intentionally truncated. + The plan-contiguous wave bound also caps the number of retained capture files. + A background or escaped descendant can continue and can append through an + inherited handle; bytes written after finalization are outside the finite + snapshot. RAII removes cargo-each's directory entry after emission, but an + untimed descendant that preserves the inherited writer can keep the backing + storage allocated and continue growing it until that handle closes. The wave + bound therefore limits cargo-each-owned files, not storage retained by + preserved descendants. There is no portable way to revoke an inherited file + handle without terminating that descendant. Capture create, handle-reopen, + length, seek, or read failures are infrastructure failures. +- **Timeouts terminate jobs or process groups.** A timed-out command is a + failure. `command-group` creates a job object on Windows and a process group + on Unix. Both timed streamed and captured execution observe the launched + leader directly, preserving its exit status even while an ordinary + background group member remains. The group handle remains available solely + for deadline termination. At the deadline cargo-each kills that boundary, + polls the direct leader with bounded sleeps, and allows 250 ms for it to + finish. If it still has not completed, the `GroupChild` moves to one + cargo-each-local polling reaper started before any child process. The reaper + polls every retained group rather than blocking forever on one, owns groups + after the caller returns, and shuts down only after all senders disconnect + and its retained set is empty. Startup failure therefore aborts before + command launch. A disconnected handoff reports an infrastructure failure and + places the recovered handle in a persistent fallback queue before starting an + emergency polling reaper. If that thread cannot start, the queue retains + ownership and a later failed handoff retries startup. A failed kill, + observation, bounded reap, reaper startup, or handoff is an infrastructure + failure. Reaper polling retries interrupted observations; a terminal + observation error is reported asynchronously and the unobservable handle is + no longer retained forever. Unix process groups are not sealed containment: + a descendant can escape by creating a new session, so timeout cleanup remains + best-effort for escaped descendants. - **Child executable resolution follows `PATH`.** `cargo-each` explicitly copies an inherited `PATH` onto every child command. This is equivalent to ordinary inheritance on other platforms and makes Windows resolve a relative @@ -285,17 +391,16 @@ no-op. ## 6. How it simplifies cargo-anvil -The recipes stop parsing the impact selection and metadata by hand. anvil's -`_anvil-impact-include ` helper reads -`target/anvil/impact/include_.txt`, applies the `ANVIL_IMPACT=off` -override, and emits a **concrete selector for every tier** — `--workspace` -(unscoped / local / off), `--package name@version …` (scoped), or `--none` -(empty tier). A `cargo-each` check just splats that output straight in as -flags. Because the helper always emits a concrete selector, cargo-each never -falls back to `default-members`, so no per-call default flag is needed. -Illustrative before/after (the recipe keeps its own comments, setup deps, -`: anvil-impact` dependency, and any domain glue; only the selection spine -changes): +A planned cargo-anvil adoption can stop parsing impact selections and metadata +by hand, but the examples below are not usable with the current producer yet. +Today it writes `include_.txt` values containing `--package` tokens or the +`--workspace` / `--skip` sentinels, all of which package-file validation +intentionally rejects. The producer must first change to write one +`name@version` package spec per line under `target/anvil/impact/`, with an empty +file for an empty tier. After that producer change, a cargo-each check can +supply the appropriate file directly and get a successful no-op for an empty +tier. Illustrative planned before/after (the recipe keeps its own setup and +`anvil-impact` dependencies; only the selection spine changes): **clippy** (affected tier, single invocation): @@ -305,9 +410,9 @@ if (-not $env:ANVIL_INCLUDE_AFFECTED) { $env:ANVIL_INCLUDE_AFFECTED = (& just _a if ($env:ANVIL_INCLUDE_AFFECTED -eq '--skip') { exit 0 } & cargo clippy @(if ($env:ANVIL_INCLUDE_AFFECTED) { -split $env:ANVIL_INCLUDE_AFFECTED } else { '--workspace' }) --all-targets --all-features --locked -- -D warnings ``` -```powershell +```just # after -cargo each @(& {{ just_executable() }} _anvil-impact-include affected) --once -- \ +cargo each --package-file target/anvil/impact/affected.packages --once -- \ cargo clippy {packages} --all-targets --all-features --locked -- -D warnings ``` @@ -315,52 +420,38 @@ cargo each @(& {{ just_executable() }} _anvil-impact-include affected) --once -- name-to-manifest map, `--workspace` branch, `@version` strip, and iteration loop collapse to: -```powershell -cargo each @(& {{ just_executable() }} _anvil-impact-include affected) --filter lib -- \ +```just +cargo each --package-file target/anvil/impact/affected.packages --filter lib -- \ cargo +{{ rust_nightly_external_types }} check-external-types --manifest-path {manifest} ``` **loom** (affected packages that depend on loom): -```powershell -cargo each @(& {{ just_executable() }} _anvil-impact-include affected) --filter dep:loom -- \ +```just +cargo each --package-file target/anvil/impact/affected.packages --filter dep:loom -- \ cargo +{{ rust_nightly }} test --package {name} ... ``` -**llvm-cov opt-out drop** (exclude coverage-opted-out members): +**per-target examples** (run each selected example with a timeout): -```powershell -cargo each @(& {{ just_executable() }} _anvil-impact-include affected) \ - --exclude-filter metadata:coverage-gate.min-lines-percent=0 --once -- +```just +cargo each --package-file target/anvil/impact/affected.packages \ + --each-target example --timeout 30s -- \ + cargo run --package {spec} --example {target} ``` -Recipes whose only per-tier logic is the skip/splat preamble (bench, clippy, -doc-build, examples, miri*, doc-test, cargo-hack, udeps, careful) become a -single `cargo each … --once` line. Modified-tier workspace-wide tools (fmt, -cargo-sort, license-headers, spellcheck, ensure-no-*) become -`cargo each @(& just _anvil-impact-include modified) --once -- ` — the -`--once` skip-when-empty behavior replaces the `--skip` guard while the tool -still runs workspace-wide. - -Three small `anvil-impact` adjustments complete the picture (all part of the -adoption change, not this crate): - -- **Emit `--none`, not `--skip`, for an empty tier**, and drop the modified - tier's empty default: `_anvil-impact-include` emits `--workspace` / - `--package …` / `--none` **uniformly across all three tiers**. cargo-delta - makes no fundamental distinction between the tiers — they are just three - package sets — so neither should the helper. `--none` is `cargo-each`'s - native "select zero members" token, so the include file needs no - anvil-specific sentinel and `cargo each` skips the tier with no caller guard. -- **Print one token per line** from `_anvil-impact-include`, so the recipe's - `@(& …)` capture is a ready-to-splat array — no `-split`, no `if/else`. -- **Stop version-qualifying.** `_anvil-impact-format` can emit bare package - names; `cargo-each` derives `{spec}`/`{packages}` (the `name@version` form a - child cargo command needs) from live metadata itself. - -With the helper's output splatted straight into `cargo each`, the per-check -`_anvil-impact-include` *self-populate* line and the `ANVIL_INCLUDE_` -environment variable are no longer needed by scoped checks. +Recipes whose only per-tier logic is the skip/splat preamble become one +`cargo each --package-file …` command. Unscoped runs pass `--workspace` +instead; choosing scoped versus unscoped input remains caller policy and is not +encoded into cargo-each. + +The setup graph can use `{workspace-rust-version}` to install the single root +MSRV fallback without parsing Cargo TOML in a shell: + +```just +cargo each --workspace --once -- \ + rustup toolchain install {workspace-rust-version} --profile minimal +``` ## 7. Rejected alternatives @@ -373,11 +464,9 @@ environment variable are no longer needed by scoped checks. - **A generic expression language for filters.** Over-built for the handful of predicates the recipes actually need; the fixed predicate set covers every current `cargo metadata` filter and stays trivially auditable. -- **A `--from-file` / `--from-env` selection source.** Rejected: it would pull - the impact artifact layout (and the `ANVIL_IMPACT=off` widening + tier-default - policy) into cargo-each, duplicating logic that already lives in anvil's - `_anvil-impact-include` helper. Keeping cargo-each flag-only and letting the - caller splat that helper's output in is smaller and keeps the impact policy in - one place. +- **An Anvil-aware selection source.** Rejected: cargo-each does not accept a + tier name, inspect `ANVIL_IMPACT`, or assume a `target/anvil` layout. + `--package-file` is deliberately generic: one Cargo package spec per line, + with an empty file meaning an explicitly empty set. - **Reuse `cargo xtask`/a justfile function.** Neither is cargo-native selection; both re-introduce a shell dialect. A small binary is portable and testable. diff --git a/crates/cargo-each/src/cli.rs b/crates/cargo-each/src/cli.rs index 9b338e4d9..06ab56026 100644 --- a/crates/cargo-each/src/cli.rs +++ b/crates/cargo-each/src/cli.rs @@ -3,7 +3,9 @@ //! Command-line interface definitions for `cargo-each`. +use std::num::NonZeroUsize; use std::path::PathBuf; +use std::time::Duration; use clap::{Args, Parser}; @@ -36,6 +38,11 @@ pub(crate) struct EachArgs { #[arg(short = 'p', long = "package", value_name = "SPEC")] pub(crate) packages: Vec, + /// Read package specs from a UTF-8 file, one per nonempty line. + /// Repeatable; specs are unioned with --package. + #[arg(long = "package-file", value_name = "PATH")] + pub(crate) package_files: Vec, + /// Select every workspace member. #[arg(long, visible_alias = "all")] pub(crate) workspace: bool, @@ -87,6 +94,19 @@ pub(crate) struct EachArgs { #[arg(long)] pub(crate) keep_going: bool, + /// Run at most N per-package or per-target commands concurrently. Use + /// `auto` to detect available parallelism once. Defaults to 1. Buffered + /// output is redirected to unique temporary files and emitted in plan order. + /// Only an effective count above 1 disconnects child standard input for capture. + #[arg(long, default_value_t = NonZeroUsize::MIN, value_name = "N|auto", value_parser = parse_jobs)] + pub(crate) jobs: NonZeroUsize, + + /// Terminate each invocation's Windows job or Unix process group after this + /// duration. Unix descendants can escape by starting a new session. Accepts + /// a positive integer followed by `ms`, `s`, or `m`. + #[arg(long, value_name = "DURATION", value_parser = parse_duration)] + pub(crate) timeout: Option, + /// Print the fully-substituted commands without executing them. #[arg(long)] pub(crate) dry_run: bool, @@ -101,10 +121,60 @@ pub(crate) struct EachArgs { pub(crate) command: Vec, } +fn parse_jobs(value: &str) -> Result { + parse_jobs_with(value, std::thread::available_parallelism) +} + +fn parse_jobs_with(value: &str, available_parallelism: impl FnOnce() -> std::io::Result) -> Result { + if value == "auto" { + available_parallelism().map_err(|error| format!("failed to detect available parallelism for `--jobs auto`: {error}")) + } else { + value + .parse() + .map_err(|error| format!("expected a positive integer or `auto`: {error}")) + } +} + +fn parse_duration(value: &str) -> Result { + enum Unit { + Milliseconds, + Seconds, + Minutes, + } + + let (digits, unit) = if let Some(digits) = value.strip_suffix("ms") { + (digits, Unit::Milliseconds) + } else if let Some(digits) = value.strip_suffix('s') { + (digits, Unit::Seconds) + } else if let Some(digits) = value.strip_suffix('m') { + (digits, Unit::Minutes) + } else { + return Err("expected a positive integer followed by `ms`, `s`, or `m`".to_owned()); + }; + if digits.is_empty() { + return Err("expected a positive integer followed by `ms`, `s`, or `m`".to_owned()); + } + if !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return Err("expected a positive integer followed by `ms`, `s`, or `m`".to_owned()); + } + let amount = digits.parse::().map_err(|error| format!("duration is too large: {error}"))?; + if amount == 0 { + return Err("duration must be greater than zero".to_owned()); + } + match unit { + Unit::Milliseconds => Ok(Duration::from_millis(amount)), + Unit::Seconds => Ok(Duration::from_secs(amount)), + Unit::Minutes => amount + .checked_mul(60) + .map(Duration::from_secs) + .ok_or_else(|| "duration is too large".to_owned()), + } +} + #[cfg(test)] #[cfg_attr(coverage_nightly, coverage(off))] mod tests { - use clap::CommandFactory; + use clap::{CommandFactory, Parser as _}; use super::*; @@ -112,4 +182,65 @@ mod tests { fn cli_definition_is_well_formed() { CargoCli::command().debug_assert(); } + + #[test] + fn jobs_default_is_one() { + let CargoCli::Each(args) = + CargoCli::try_parse_from(["cargo", "each", "--", "echo"]).expect("documented minimal invocation must parse"); + assert_eq!(args.jobs, NonZeroUsize::MIN); + } + + #[test] + fn parses_positive_and_auto_jobs() { + assert_eq!( + parse_jobs_with("7", || panic!("numeric jobs must not detect parallelism")), + Ok(NonZeroUsize::new(7).expect("literal seven is nonzero")) + ); + + let mut detections = 0; + let jobs = parse_jobs_with("auto", || { + detections += 1; + Ok(NonZeroUsize::new(8).expect("literal eight is nonzero")) + }); + assert_eq!(jobs, Ok(NonZeroUsize::new(8).expect("literal eight is nonzero"))); + assert_eq!(detections, 1); + } + + #[test] + fn rejects_invalid_jobs() { + for value in ["0", "-1", "bogus", "AUTO"] { + assert!(parse_jobs(value).is_err(), "{value}"); + } + } + + #[test] + fn reports_auto_jobs_detection_failure() { + let error = parse_jobs_with("auto", || Err(std::io::Error::other("parallelism unavailable"))) + .expect_err("detection failure must not fall back"); + assert_eq!( + error, + "failed to detect available parallelism for `--jobs auto`: parallelism unavailable" + ); + } + + #[test] + fn parses_documented_durations() { + assert_eq!(parse_duration("250ms"), Ok(Duration::from_millis(250))); + assert_eq!(parse_duration("30s"), Ok(Duration::from_secs(30))); + assert_eq!(parse_duration("2m"), Ok(Duration::from_mins(2))); + } + + #[test] + fn rejects_zero_malformed_and_overflowing_durations() { + for value in ["0s", "1", "1h", "-1s", "1.5s", "ms", "18446744073709551615m"] { + assert!(parse_duration(value).is_err(), "{value}"); + } + } + + #[test] + fn malformed_duration_uses_the_grammar_diagnostic() { + let expected = Err("expected a positive integer followed by `ms`, `s`, or `m`".to_owned()); + assert_eq!(parse_duration("ms"), expected); + assert_eq!(parse_duration("1.5s"), expected); + } } diff --git a/crates/cargo-each/src/error.rs b/crates/cargo-each/src/error.rs index b36708495..43b62a984 100644 --- a/crates/cargo-each/src/error.rs +++ b/crates/cargo-each/src/error.rs @@ -32,7 +32,14 @@ InvalidFilterExpressionError, InvalidTargetKindError, PlaceholderMisuseError, - ChdirConflictsWithOnceError + ChdirConflictsWithOnceError, + JobsConflictWithOnceError, + PackageFileReadError, + PackageFileUtf8Error, + InvalidPackageFileLineError, + WorkspaceManifestReadError, + WorkspaceManifestParseError, + WorkspaceRustVersionError )] pub(crate) struct EachError; @@ -82,6 +89,62 @@ pub(crate) struct PlaceholderMisuseError { #[display("`--chdir` cannot be combined with `--once`")] pub(crate) struct ChdirConflictsWithOnceError; +/// `--jobs` greater than one was combined with `--once`. +#[ohno::error] +#[display("`--jobs` must be 1 when combined with `--once`")] +pub(crate) struct JobsConflictWithOnceError; + +/// A `--package-file` could not be read. +#[ohno::error] +#[display("could not read package file `{path}`")] +#[from(std::io::Error)] +pub(crate) struct PackageFileReadError { + pub(crate) path: String, +} + +/// A `--package-file` was not valid UTF-8. +#[ohno::error] +#[display("package file `{path}` is not valid UTF-8")] +#[from(std::string::FromUtf8Error)] +pub(crate) struct PackageFileUtf8Error { + pub(crate) path: String, +} + +/// A nonempty line in a `--package-file` was not a package spec. +#[ohno::error] +#[display("invalid package spec in `{path}` at line {line}: `{spec}` ({reason})")] +pub(crate) struct InvalidPackageFileLineError { + pub(crate) path: String, + pub(crate) line: usize, + pub(crate) spec: String, + pub(crate) reason: String, +} + +/// The root manifest could not be read while resolving +/// `{workspace-rust-version}`. +#[ohno::error] +#[display("could not read workspace manifest `{path}`")] +#[from(std::io::Error)] +pub(crate) struct WorkspaceManifestReadError { + pub(crate) path: String, +} + +/// The root manifest could not be parsed while resolving +/// `{workspace-rust-version}`. +#[ohno::error] +#[display("could not parse workspace manifest `{path}`")] +#[from(toml::de::Error)] +pub(crate) struct WorkspaceManifestParseError { + pub(crate) path: String, +} + +/// The workspace Rust-version contract is incomplete or inconsistent. +#[ohno::error] +#[display("cannot resolve `{{workspace-rust-version}}`: {reason}")] +pub(crate) struct WorkspaceRustVersionError { + pub(crate) reason: String, +} + #[cfg(test)] #[cfg_attr(coverage_nightly, coverage(off))] mod tests { @@ -135,4 +198,26 @@ mod tests { assert!(rendered.contains("--chdir")); assert!(rendered.contains("--once")); } + + #[test] + fn package_file_line_error_names_source() { + let err = InvalidPackageFileLineError::new( + "affected.packages".to_owned(), + 3_usize, + "--workspace".to_owned(), + "command-line tokens are not package specs".to_owned(), + ); + let rendered = err.to_string(); + assert!(rendered.contains("affected.packages")); + assert!(rendered.contains("line 3")); + assert!(rendered.contains("--workspace")); + } + + #[test] + fn workspace_rust_version_error_renders_reason() { + let err = WorkspaceRustVersionError::new("member `alpha` does not declare `rust-version`".to_owned()); + let rendered = err.to_string(); + assert!(rendered.contains("{workspace-rust-version}")); + assert!(rendered.contains("alpha")); + } } diff --git a/crates/cargo-each/src/filter.rs b/crates/cargo-each/src/filter.rs index 3fc17fb1e..184861fe9 100644 --- a/crates/cargo-each/src/filter.rs +++ b/crates/cargo-each/src/filter.rs @@ -420,6 +420,7 @@ mod tests { Member { name: "m".to_owned(), version: "0.1.0".to_owned(), + rust_version: Some("1.70.0".parse().expect("valid Rust version")), manifest_path: PathBuf::from("/ws/m/Cargo.toml"), publishable: true, features: BTreeSet::new(), diff --git a/crates/cargo-each/src/main.rs b/crates/cargo-each/src/main.rs index c06c501be..6babcb115 100644 --- a/crates/cargo-each/src/main.rs +++ b/crates/cargo-each/src/main.rs @@ -34,16 +34,19 @@ //! //! - `-p` / `--package ` — select a member. Repeatable. `SPEC` is a //! package name, a `name@version` spec, or a Unix glob (`tokio-*`). +//! - `--package-file ` — read package specs from a UTF-8 file, one per +//! nonempty line. Repeatable; specs are unioned with `--package`. An empty +//! file explicitly selects no members. One leading UTF-8 byte-order mark is +//! ignored. //! - `--workspace` / `--all` — select every workspace member. //! - `--exclude ` — drop a member (with `--workspace`). Repeatable. //! - `--none` — explicitly select zero members (a no-op that exits 0). //! //! When nothing is named the default is cargo `default-members`, exactly //! like `cargo build`; pass `--workspace` for every member. A selector that -//! matches no member is an error, so typos fail loudly. A computed selection -//! (for example a CI affected-packages set) is fed in as ordinary flags via -//! shell expansion — `cargo-each` has no file or environment-variable source -//! of its own. +//! matches no member is an error, so typos fail loudly. Package files contain +//! package specs only: comments, command-line tokens, malformed input, and +//! missing, unreadable, or non-UTF-8 files are errors. //! //! ## Filters //! @@ -72,9 +75,16 @@ //! `--target-required-feature` further narrows targets. //! //! `--keep-going` runs every invocation and exits non-zero if any failed -//! (default is fail-fast); `--chdir` runs each per-package or per-target -//! command from that member crate root; `--dry-run` prints commands without -//! running them. +//! (default is fail-fast). `--jobs ` bounds concurrent per-package or +//! per-target work. Omitting it runs exactly one invocation at a time; `auto` +//! resolves once to the machine's available parallelism. Detection failure is +//! reported explicitly without falling back. `--timeout ` terminates +//! each invocation's Windows job object or Unix process group independently +//! (`250ms`, `30s`, or `2m`). Unix descendants can escape a process group by +//! starting a new session, so timeout cleanup is best-effort for those escaped +//! descendants. +//! `--chdir` runs each per-package or per-target command from that member crate +//! root; `--dry-run` prints commands without running them. //! //! ## Placeholders //! @@ -89,6 +99,9 @@ //! - `{packages}` — the cargo selection flags for the resolved set //! (`--workspace` for the whole workspace, else `--package name@version …`); //! valid only in `--once` mode and only as a standalone argument. +//! - `{workspace-rust-version}` — the root `[workspace.package].rust-version`, +//! or root `[package].rust-version` in a single-package repository; valid in +//! every mode. //! //! Using a placeholder in the wrong mode is a usage error. Only the tokens //! above are interpreted; any other `{…}` sequence (a typo, or a literal brace @@ -100,12 +113,57 @@ //! An empty resolved selection (via `--none`, or a filter that removes every //! member) is a **successful no-op**: `cargo-each` prints a one-line note and //! exits 0. This is what lets callers drop bespoke nothing-to-do guards. +//! Workspace Rust-version validation is lazy: it runs only when the command +//! uses `{workspace-rust-version}` and the resolved plan has work, then requires +//! every member's resolved minimum to be present and no newer than the root +//! floor. Placeholder mode validation still runs before an empty-plan no-op. +//! +//! The effective worker count is the requested `--jobs` value capped by plan +//! size. An effective count of one uses sequential +//! execution with inherited standard input, output, and error even when the +//! requested value was larger. A genuinely parallel count disconnects child input and +//! buffers stdout and stderr; complete blocks are emitted in deterministic +//! plan order. Fail-fast stops launching after the first +//! observed failure, waits for running work, and chooses the final failure by +//! plan order. `--keep-going` runs the complete plan. Worker panics and +//! unexpected worker-channel disconnections become infrastructure-failure +//! outcomes instead of blocking the scheduler. Worker launch failures retain +//! output already collected at earlier plan indices. Parallel work runs in +//! plan-contiguous waves capped by the effective worker count; each completed +//! wave is emitted and dropped before the next wave starts, bounding retained +//! temporary-file storage. Untimed effective-one execution uses an ordinary +//! child, preserving terminal foreground behavior and Ctrl-C delivery; a +//! post-spawn wait failure gets bounded child cleanup and reaper ownership. +//! Timed and genuinely parallel commands use a job or process group. Without +//! `--timeout`, cargo-each observes only the leader and does not kill background +//! descendants. Every genuinely parallel invocation redirects stdout and +//! stderr directly to separate unique temporary files. +//! Child writers and parent readers are separately reopened so parent seeks +//! cannot move descendant write positions. Cargo-each records each file's +//! current length when the leader completes (or after timeout cleanup), then +//! reads exactly that finite snapshot in plan order without loading unbounded +//! output into memory. Later writes by background or escaped descendants are +//! outside the snapshot. RAII removes cargo-each's directory entry, but a +//! preserved descendant can keep the backing storage allocated and growing +//! until its inherited writer closes. Capture create, reopen, length, seek, and +//! read failures are infrastructure failures. +//! +//! Timed-out group termination gets a bounded 250 ms reap grace. If the group +//! still has not completed, its handle moves to a cargo-each-local polling +//! reaper started before any command. The reaper checks every retained group +//! without blocking on one child, remains the wait owner after the caller +//! returns, and exits after all senders disconnect and retained groups are +//! collected. Interrupted observations are retried; terminal observation +//! errors are reported and removed. Reaper startup and handoff failures are +//! explicit infrastructure failures; a failed handoff retains the group handle +//! in a persistent fallback queue and starts an emergency polling reaper. //! Child commands inherit `PATH` explicitly. On Windows this makes relative //! program lookup honor the inherited `PATH` order instead of preferring an //! unrelated executable beside `cargo-each`. -//! Otherwise the exit code is the first failing command code (fail-fast), -//! `1` under `--keep-going` if any command failed, or `2` for a `cargo-each` -//! usage error (unknown selector, bad filter expression, misused placeholder). +//! Exit `0` means all work succeeded or there was no work. In fail-fast mode a +//! command failure returns its code, a timeout returns `1`, and usage, +//! configuration, spawn, or post-spawn infrastructure failures return `2`. +//! Under `--keep-going`, any failure maps the aggregate result to `1`. //! //! # Examples //! diff --git a/crates/cargo-each/src/plan.rs b/crates/cargo-each/src/plan.rs index cd6003150..db145d9b6 100644 --- a/crates/cargo-each/src/plan.rs +++ b/crates/cargo-each/src/plan.rs @@ -63,7 +63,40 @@ pub(crate) struct Plan { pub(crate) invocations: Vec, } +/// Inputs that control how a selected member set becomes invocations. +#[derive(Debug, Clone, Copy)] +pub(crate) struct BuildOptions<'a> { + pub(crate) mode: Mode, + pub(crate) chdir: bool, + pub(crate) packages: PackagesExpansion, + pub(crate) target_kinds: &'a BTreeSet, + pub(crate) target_required_features: &'a BTreeSet, + pub(crate) workspace_rust_version: Option<&'a str>, +} + impl Plan { + /// Validate plan-wide configuration and report whether it produces no + /// invocations without expanding placeholders. + /// + /// This lets callers preserve usage validation while deferring lazy + /// workspace-scoped values until the resolved selection is known to + /// produce work. + /// + /// # Errors + /// + /// Returns [`EachError`] under the same invalid `chdir` and placeholder + /// combinations as [`Self::build`]. + pub(crate) fn is_empty(members: &[&Member], command: &[String], options: BuildOptions<'_>) -> Result { + validate_build(command, options)?; + Ok(match options.mode { + Mode::PerPackage | Mode::Once => members.is_empty(), + Mode::PerTarget => !members + .iter() + .flat_map(|member| &member.targets) + .any(|target| target_matches(target, options.target_kinds, options.target_required_features)), + }) + } + /// Build the plan. /// /// `chdir` runs each per-package or per-target invocation from the member's @@ -81,27 +114,20 @@ impl Plan { /// /// Returns [`EachError`] if `chdir` is combined with [`Mode::Once`], or if /// a placeholder in `command` is used in the wrong mode. - pub(crate) fn build( - members: &[&Member], - mode: Mode, - chdir: bool, - packages: PackagesExpansion, - target_kinds: &BTreeSet, - target_required_features: &BTreeSet, - command: &[String], - ) -> Result { - if chdir && mode == Mode::Once { - return Err(ChdirConflictsWithOnceError::new().into()); - } - // Validate placeholder/mode consistency up front — before the - // empty-set short-circuit — so a misused template (e.g. `{name}` under - // `--once`) is a usage error even when the selection resolves to no - // members, rather than silently passing until some tier is non-empty. - validate_placeholders(command, mode)?; - if members.is_empty() { + pub(crate) fn build(members: &[&Member], command: &[String], options: BuildOptions<'_>) -> Result { + if Self::is_empty(members, command, options)? { return Ok(Self { invocations: Vec::new() }); } + let BuildOptions { + mode, + chdir, + packages, + target_kinds, + target_required_features, + workspace_rust_version, + } = options; + let invocations = match mode { Mode::PerPackage => members .iter() @@ -111,6 +137,7 @@ impl Plan { spec: m.spec(), version: m.version.clone(), manifest: m.manifest_path.display().to_string(), + workspace_rust_version: workspace_rust_version.map(str::to_owned), }; Ok(Invocation { label: Some(m.name.clone()), @@ -125,12 +152,7 @@ impl Plan { member .targets .iter() - .filter(|target| { - target.kinds.iter().any(|kind| target_kinds.contains(kind)) - && target_required_features - .iter() - .all(|feature| target.required_features.contains(feature)) - }) + .filter(|target| target_matches(target, target_kinds, target_required_features)) .map(|target| { let placeholders = Placeholders::Target { name: member.name.clone(), @@ -138,6 +160,7 @@ impl Plan { version: member.version.clone(), manifest: member.manifest_path.display().to_string(), target: target.name.clone(), + workspace_rust_version: workspace_rust_version.map(str::to_owned), }; Ok(Invocation { label: Some(format!("{}::{}", member.name, target.name)), @@ -149,7 +172,10 @@ impl Plan { .collect::, EachError>>()?, Mode::Once => { let packages = packages_flags(members, packages); - let placeholders = Placeholders::Once { packages }; + let placeholders = Placeholders::Once { + packages, + workspace_rust_version: workspace_rust_version.map(str::to_owned), + }; vec![Invocation { label: None, argv: substitute(command, &placeholders)?, @@ -162,6 +188,26 @@ impl Plan { } } +fn validate_build(command: &[String], options: BuildOptions<'_>) -> Result<(), EachError> { + if options.chdir && options.mode == Mode::Once { + return Err(ChdirConflictsWithOnceError::new().into()); + } + // Validate placeholder/mode consistency before any empty-set short-circuit + // so an invalid template never depends on whether a computed tier has work. + validate_placeholders(command, options.mode) +} + +fn target_matches( + target: &crate::workspace::MemberTarget, + target_kinds: &BTreeSet, + target_required_features: &BTreeSet, +) -> bool { + target.kinds.iter().any(|kind| target_kinds.contains(kind)) + && target_required_features + .iter() + .all(|feature| target.required_features.contains(feature)) +} + /// The `{packages}` expansion: `--workspace` for the whole workspace, else an /// explicit `--package name@version` per member. fn packages_flags(members: &[&Member], packages: PackagesExpansion) -> Vec { @@ -188,6 +234,7 @@ mod tests { Member { name: name.to_owned(), version: "1.2.3".to_owned(), + rust_version: Some("1.70.0".parse().expect("valid Rust version")), manifest_path: PathBuf::from(format!("/ws/{name}/Cargo.toml")), publishable: true, features: BTreeSet::new(), @@ -202,7 +249,18 @@ mod tests { } fn build(members: &[&Member], mode: Mode, chdir: bool, packages: PackagesExpansion, command: &[String]) -> Result { - Plan::build(members, mode, chdir, packages, &BTreeSet::new(), &BTreeSet::new(), command) + Plan::build( + members, + command, + BuildOptions { + mode, + chdir, + packages, + target_kinds: &BTreeSet::new(), + target_required_features: &BTreeSet::new(), + workspace_rust_version: None, + }, + ) } #[test] @@ -333,12 +391,15 @@ mod tests { let required = std::iter::once("loom".to_owned()).collect(); let plan = Plan::build( &[&a], - Mode::PerTarget, - false, - PackagesExpansion::Explicit, - &kinds, - &required, &cmd(&["cargo", "test", "-p", "{name}", "--test", "{target}"]), + BuildOptions { + mode: Mode::PerTarget, + chdir: false, + packages: PackagesExpansion::Explicit, + target_kinds: &kinds, + target_required_features: &required, + workspace_rust_version: None, + }, ) .expect("build target plan"); assert_eq!(plan.invocations.len(), 1); @@ -357,14 +418,36 @@ mod tests { let kinds = std::iter::once(TargetKind::Example).collect(); let plan = Plan::build( &[&a], - Mode::PerTarget, - true, - PackagesExpansion::Explicit, - &kinds, - &BTreeSet::new(), &cmd(&["echo", "{target}"]), + BuildOptions { + mode: Mode::PerTarget, + chdir: true, + packages: PackagesExpansion::Explicit, + target_kinds: &kinds, + target_required_features: &BTreeSet::new(), + workspace_rust_version: None, + }, ) .expect("build target plan"); assert_eq!(plan.invocations[0].work_dir.as_deref(), Some(PathBuf::from("/ws/alpha").as_path())); } + + #[test] + fn workspace_rust_version_is_available_in_once_mode() { + let a = member("alpha"); + let plan = Plan::build( + &[&a], + &cmd(&["rustup", "toolchain", "install", "{workspace-rust-version}"]), + BuildOptions { + mode: Mode::Once, + chdir: false, + packages: PackagesExpansion::Workspace, + target_kinds: &BTreeSet::new(), + target_required_features: &BTreeSet::new(), + workspace_rust_version: Some("1.80"), + }, + ) + .expect("build"); + assert_eq!(plan.invocations[0].argv, ["rustup", "toolchain", "install", "1.80"]); + } } diff --git a/crates/cargo-each/src/run.rs b/crates/cargo-each/src/run.rs index 4e1d6e29b..c53ea84ce 100644 --- a/crates/cargo-each/src/run.rs +++ b/crates/cargo-each/src/run.rs @@ -5,20 +5,38 @@ //! apply filters, build the plan, and run it. use std::collections::BTreeSet; -use std::process::{Command, ExitCode}; +use std::io::{self, Read as _, Seek as _, SeekFrom, Write as _}; +use std::num::NonZeroUsize; +use std::panic::{self, UnwindSafe}; +use std::process::{Child, Command, ExitCode, ExitStatus, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, OnceLock, mpsc}; +use std::time::{Duration, Instant}; +use std::{fmt, thread}; use cargo_metadata::TargetKind; +use command_group::{CommandGroup as _, GroupChild}; use ohno::{AppError, IntoAppError}; use crate::cli::EachArgs; -use crate::error::InvalidTargetKindError; +use crate::error::{InvalidTargetKindError, JobsConflictWithOnceError}; use crate::filter::Predicate; -use crate::plan::{Mode, PackagesExpansion, Plan}; +use crate::plan::{BuildOptions, Invocation, Mode, PackagesExpansion, Plan}; use crate::select::Selection; +use crate::substitute::uses_workspace_rust_version; use crate::workspace::{Member, Workspace}; +#[cfg(test)] +const WORKER_PANIC_TEST_PROGRAM: &str = "__cargo_each_injected_worker_panic"; +#[cfg(test)] +const WORKER_SPAWN_ERROR_TEST_PROGRAM: &str = "__cargo_each_injected_worker_spawn_error"; +const TERMINATION_GRACE: Duration = Duration::from_millis(250); +const REAPER_POLL_INTERVAL: Duration = Duration::from_millis(10); + +type ReaperJob = Box; + pub(crate) fn run(args: &EachArgs) -> Result { - let selection = build_selection(args); + let selection = build_selection(args).into_app_err("failed to read package selection")?; let workspace = Workspace::load(args.manifest_path.as_deref()).into_app_err("failed to load workspace")?; let mut members = selection.resolve(&workspace).into_app_err("failed to resolve package selection")?; @@ -41,22 +59,36 @@ pub(crate) fn run(args: &EachArgs) -> Result { } else { Mode::PerTarget }; - let plan = Plan::build( - &members, + if mode == Mode::Once && args.jobs.get() != 1 { + return Err(JobsConflictWithOnceError::new()).into_app_err("invalid execution configuration"); + } + + let mut build_options = BuildOptions { mode, - args.chdir, + chdir: args.chdir, packages, - &target_kinds, - &target_required_features, - &args.command, - ) - .into_app_err("failed to build command plan")?; - - if plan.invocations.is_empty() { + target_kinds: &target_kinds, + target_required_features: &target_required_features, + workspace_rust_version: None, + }; + if Plan::is_empty(&members, &args.command, build_options).into_app_err("failed to build command plan")? { eprintln!("cargo each: selection resolved to no work; nothing to do"); return Ok(ExitCode::SUCCESS); } + let workspace_rust_version = if uses_workspace_rust_version(&args.command) { + Some( + workspace + .workspace_rust_version() + .into_app_err("failed to resolve workspace Rust version")?, + ) + } else { + None + }; + build_options.workspace_rust_version = workspace_rust_version.as_deref(); + + let plan = Plan::build(&members, &args.command, build_options).into_app_err("failed to build command plan")?; + if args.dry_run { for inv in &plan.invocations { match &inv.work_dir { @@ -64,25 +96,16 @@ pub(crate) fn run(args: &EachArgs) -> Result { None => println!("{}", shell_join(&inv.argv)), } } + return Ok(ExitCode::SUCCESS); } - execute(&plan, args.keep_going) + execute(&plan, args.keep_going, args.jobs, args.timeout) } -/// Assemble a [`Selection`] from the parsed flags. -/// -/// The selection is entirely flag-driven: a computed selection (e.g. an -/// impact tier) is fed in by the caller via ordinary shell expansion — anvil -/// splats `_anvil-impact-include ` into the `cargo each` invocation — -/// so cargo-each stays agnostic about where the selectors came from. -fn build_selection(args: &EachArgs) -> Selection { - Selection { - packages: args.packages.clone(), - all: args.workspace, - exclude: args.exclude.clone(), - none: args.none, - } +/// Assemble a [`Selection`] from direct and file-backed package specs. +fn build_selection(args: &EachArgs) -> Result { + Selection::from_sources(&args.packages, &args.package_files, args.workspace, &args.exclude, args.none) } /// Narrow `members` by package keep and drop expressions. Repeated `--filter` @@ -116,116 +139,2161 @@ fn parse_target_kinds(kinds: &[String]) -> Result, AppError .into_app_err("invalid per-target configuration") } -/// Run each invocation, honoring the fail-fast / `--keep-going` policy. -/// -/// A spawn failure (the child could not be launched at all) is treated the -/// same as a non-zero child exit: under `--keep-going` it is logged, counted -/// as a failure, and the run continues (final exit `1`); under fail-fast it -/// aborts. This keeps the documented "run them all" contract intact even when -/// one invocation cannot start. -fn execute(plan: &Plan, keep_going: bool) -> Result { +fn execute(plan: &Plan, keep_going: bool, jobs: NonZeroUsize, timeout: Option) -> Result { + let reaper = ProcessReaper::start().into_app_err("failed to start cargo-each process reaper")?; + let worker_count = effective_worker_count(jobs, plan.invocations.len()); + if worker_count.get() == 1 { + Ok(execute_sequential(plan, keep_going, timeout, &reaper)) + } else { + execute_parallel(plan, keep_going, worker_count, timeout, &reaper) + } +} + +fn effective_worker_count(requested: NonZeroUsize, plan_size: usize) -> NonZeroUsize { + NonZeroUsize::new(requested.get().min(plan_size)).expect("Plan::is_empty is checked before execute, so the execution plan is nonempty") +} + +fn execute_sequential(plan: &Plan, keep_going: bool, timeout: Option, reaper: &ProcessReaper) -> ExitCode { + execute_sequential_with(plan, keep_going, timeout, |invocation, timeout| { + if let Some(timeout) = timeout { + run_streamed_with_timeout(invocation, timeout, reaper) + } else { + run_streamed(invocation, reaper) + } + }) +} + +fn execute_sequential_with( + plan: &Plan, + keep_going: bool, + timeout: Option, + mut run_invocation: impl FnMut(&Invocation, Option) -> InvocationResult, +) -> ExitCode { let mut any_failed = false; - for inv in &plan.invocations { - if let Some(label) = &inv.label { - eprintln!("cargo each: {label}"); - } - let (program, rest) = inv.argv.split_first().expect("Plan::build never emits an empty argv"); - let mut command = Command::new(program); - if let Some(path) = std::env::var_os("PATH") { - command.env("PATH", path); - } - command.args(rest); - if let Some(dir) = &inv.work_dir { - command.current_dir(dir); - } - let status = match command.status() { - Ok(status) => status, - // A spawn failure under --keep-going is a failed invocation, not an - // abort: log it, mark the run failed, and move on so the remaining - // members still run (contract: exit 1 when any invocation failed). - Err(err) if keep_going => { - eprintln!("cargo each: failed to spawn `{program}`: {err}"); + for invocation in &plan.invocations { + emit_label(invocation); + let result = run_invocation(invocation, timeout); + match result { + InvocationResult::Exited(status) if status.success() => {} + InvocationResult::Exited(status) => { + if !keep_going { + return ExitCode::from(exit_byte(status.code())); + } any_failed = true; - continue; } - // Fail-fast: a spawn failure is a hard error (exit 2 via main.rs). - other => other.into_app_err(format!("failed to spawn `{program}`"))?, - }; - if !status.success() { - if !keep_going { - // Fail-fast: propagate the failing child's own exit code, - // reduced to the u8 `ExitCode` can carry (see `exit_byte`). - return Ok(ExitCode::from(exit_byte(status.code()))); + InvocationResult::TimedOut(duration) => { + eprintln!("cargo each: invocation timed out after {}", display_duration(duration)); + if !keep_going { + return ExitCode::from(1); + } + any_failed = true; + } + InvocationResult::Infrastructure(message) => { + eprintln!("cargo each: {message}"); + if !keep_going { + return ExitCode::from(2); + } + any_failed = true; } - any_failed = true; } } - // Under --keep-going the individual child codes may differ, so we cannot - // pick a single meaningful one; the documented contract is a flat `1` when - // any invocation failed. - Ok(if any_failed { ExitCode::from(1) } else { ExitCode::SUCCESS }) + if any_failed { ExitCode::from(1) } else { ExitCode::SUCCESS } } -/// Render an argv for display (`--dry-run`). Best-effort quoting for -/// readability only — nothing consumes this as input. -fn shell_join(argv: &[String]) -> String { - argv.iter() - .map(|a| { - if a.contains(char::is_whitespace) { - format!("\"{a}\"") - } else { - a.clone() +fn execute_parallel( + plan: &Plan, + keep_going: bool, + worker_count: NonZeroUsize, + timeout: Option, + reaper: &ProcessReaper, +) -> Result { + let invocations = plan.invocations.clone(); + let mut workers = Vec::with_capacity(worker_count.get()); + let mut outcomes = Vec::with_capacity(worker_count.get()); + let mut stop_launching = false; + let mut any_failed = false; + let mut first_failure = None; + let mut next_index = 0; + + for wave in invocations.chunks(worker_count.get()) { + for invocation in wave.iter().cloned() { + if stop_launching { + break; } - }) + let index = next_index; + next_index += 1; + match spawn_worker(index, invocation, timeout, reaper.clone()) { + Ok(worker) => workers.push(worker), + Err(error) => { + outcomes.push(IndexedOutcome { + index, + outcome: BufferedOutcome::infrastructure(format!("failed to create cargo-each worker thread: {error}")), + }); + any_failed = true; + if failure_stops_launching(keep_going, true) { + stop_launching = true; + } + } + } + } + + while let Some(outcome) = wait_for_worker(&mut workers) { + if outcome.outcome.result.failed() { + any_failed = true; + if failure_stops_launching(keep_going, true) { + stop_launching = true; + } + } + outcomes.push(outcome); + } + + outcomes.sort_by_key(|outcome| outcome.index); + for indexed in &mut outcomes { + emit_buffered(&invocations[indexed.index], &mut indexed.outcome).into_app_err("failed to emit buffered command output")?; + record_emitted_failure( + &indexed.outcome.result, + keep_going, + &mut any_failed, + &mut stop_launching, + &mut first_failure, + ); + } + outcomes.clear(); + if stop_launching { + break; + } + } + + if any_failed { + if keep_going { + Ok(ExitCode::from(1)) + } else { + Ok(first_failure.expect("a fail-fast failure is recorded while its completed wave is emitted")) + } + } else { + Ok(ExitCode::SUCCESS) + } +} + +fn record_emitted_failure( + result: &InvocationResult, + keep_going: bool, + any_failed: &mut bool, + stop_launching: &mut bool, + first_failure: &mut Option, +) { + if !result.failed() { + return; + } + *any_failed = true; + if failure_stops_launching(keep_going, true) { + *stop_launching = true; + } + if first_failure.is_none() { + *first_failure = Some(parallel_failure_exit_code(result)); + } +} + +fn parallel_failure_exit_code(result: &InvocationResult) -> ExitCode { + match result { + InvocationResult::Exited(status) => ExitCode::from(exit_byte(status.code())), + InvocationResult::TimedOut(_) => ExitCode::from(1), + InvocationResult::Infrastructure(_) => ExitCode::from(2), + } +} + +fn failure_stops_launching(keep_going: bool, failed: bool) -> bool { + matches!((keep_going, failed), (false, true)) +} + +fn spawn_worker(index: usize, invocation: Invocation, timeout: Option, reaper: ProcessReaper) -> io::Result { + #[cfg(test)] + if invocation + .argv + .first() + .is_some_and(|program| program == WORKER_SPAWN_ERROR_TEST_PROGRAM) + { + return Err(io::Error::other("injected worker spawn failure")); + } + + let (sender, receiver) = mpsc::channel(); + let thread = thread::Builder::new().name(format!("cargo-each-worker-{index}")).spawn(move || { + complete_worker(&sender, move || run_captured(&invocation, timeout, &reaper)); + })?; + Ok(RunningWorker { index, receiver, thread }) +} + +fn complete_worker(sender: &mpsc::Sender, work: impl FnOnce() -> BufferedOutcome + UnwindSafe) { + let outcome = match panic::catch_unwind(work) { + Ok(outcome) => outcome, + Err(payload) => BufferedOutcome::infrastructure(format!( + "worker panicked while running invocation: {}", + panic_description(payload.as_ref()) + )), + }; + let _receiver_gone = sender.send(outcome); +} + +fn wait_for_worker(workers: &mut Vec) -> Option { + if workers.is_empty() { + return None; + } + loop { + let ready = workers + .iter() + .enumerate() + .find_map(|(position, worker)| match worker.receiver.try_recv() { + Ok(outcome) => Some((position, Some(outcome))), + Err(mpsc::TryRecvError::Disconnected) => Some((position, None)), + Err(mpsc::TryRecvError::Empty) => None, + }); + let Some((position, reported)) = ready else { + thread::sleep(Duration::from_millis(1)); + continue; + }; + + let RunningWorker { index, thread, .. } = workers.swap_remove(position); + let outcome = match (reported, thread.join()) { + (Some(outcome), Ok(())) => outcome, + (Some(_) | None, Err(payload)) => BufferedOutcome::infrastructure(format!( + "worker panicked while running invocation: {}", + panic_description(payload.as_ref()) + )), + (None, Ok(())) => BufferedOutcome::infrastructure("worker exited without reporting an invocation outcome".to_owned()), + }; + return Some(IndexedOutcome { index, outcome }); + } +} + +fn panic_description(payload: &(dyn std::any::Any + Send)) -> &str { + if let Some(message) = payload.downcast_ref::<&str>() { + message + } else if let Some(message) = payload.downcast_ref::() { + message + } else { + "non-string panic payload" + } +} + +fn run_streamed(invocation: &Invocation, reaper: &ProcessReaper) -> InvocationResult { + run_streamed_with(invocation, reaper, spawn_child) +} + +fn run_streamed_with( + invocation: &Invocation, + reaper: &ProcessReaper, + spawn: impl FnOnce(Command) -> Result, +) -> InvocationResult { + let (program, command) = match command_for(invocation) { + Ok(command) => command, + Err(message) => return InvocationResult::Infrastructure(message), + }; + let child = match spawn(command) { + Ok(child) => child, + Err(error) => { + return InvocationResult::Infrastructure(format!("failed to spawn `{program}`: {error}")); + } + }; + let control = StreamedChild { child, reaper }; + wait_for_process( + control, + None, + observe_streamed_child, + terminate_streamed_child, + "observe child process", + ) + .result +} + +struct StreamedChild<'a> { + child: Child, + reaper: &'a ProcessReaper, +} + +fn observe_streamed_child(control: &mut StreamedChild<'_>) -> io::Result> { + control.child.try_wait() +} + +#[cfg_attr(coverage_nightly, coverage(off))] +#[mutants::skip] // Thin ownership adapter for an OS wait-error path; terminate_child_bounded has a real-process regression. +fn terminate_streamed_child(control: StreamedChild<'_>) -> io::Result { + terminate_child_bounded(control.child, control.reaper) +} + +fn run_streamed_with_timeout(invocation: &Invocation, timeout: Duration, reaper: &ProcessReaper) -> InvocationResult { + run_streamed_with_timeout_with(invocation, timeout, reaper, spawn_group) +} + +fn run_streamed_with_timeout_with( + invocation: &Invocation, + timeout: Duration, + reaper: &ProcessReaper, + spawn: impl FnOnce(Command) -> Result, +) -> InvocationResult { + run_streamed_group_with(invocation, Some(timeout), reaper, spawn) +} + +fn run_streamed_group_with( + invocation: &Invocation, + timeout: Option, + reaper: &ProcessReaper, + spawn: impl FnOnce(Command) -> Result, +) -> InvocationResult { + let (program, command) = match command_for(invocation) { + Ok(command) => command, + Err(message) => return InvocationResult::Infrastructure(message), + }; + let tree = match spawn(command) { + Ok(tree) => tree, + Err(error) => { + return InvocationResult::Infrastructure(format!("failed to spawn `{program}`: {error}")); + } + }; + wait_for_process( + tree, + timeout, + |process| process.inner().try_wait(), + |process| terminate_group_bounded(process, reaper), + "observe child process leader", + ) + .result +} + +fn run_captured(invocation: &Invocation, timeout: Option, reaper: &ProcessReaper) -> BufferedOutcome { + #[cfg(test)] + assert!( + invocation.argv.first().is_none_or(|program| program != WORKER_PANIC_TEST_PROGRAM), + "injected worker panic" + ); + + run_captured_with(invocation, timeout, reaper, create_output_capture, spawn_group) +} + +fn run_captured_with( + invocation: &Invocation, + timeout: Option, + reaper: &ProcessReaper, + mut capture: impl FnMut(&'static str) -> io::Result<(Box, Stdio)>, + spawner: impl FnOnce(Command) -> Result, +) -> BufferedOutcome { + let (program, mut command) = match command_for(invocation) { + Ok(command) => command, + Err(message) => return BufferedOutcome::infrastructure(message), + }; + let (stdout, stdout_stdio) = match capture("stdout") { + Ok(capture) => capture, + Err(error) => { + return BufferedOutcome::infrastructure(format!("failed to prepare child stdout capture: {error}")); + } + }; + let (stderr, stderr_stdio) = match capture("stderr") { + Ok(capture) => capture, + Err(error) => { + return BufferedOutcome::infrastructure(format!("failed to prepare child stderr capture: {error}")); + } + }; + let _ = command.stdin(Stdio::null()).stdout(stdout_stdio).stderr(stderr_stdio); + let process = match spawner(command) { + Ok(process) => process, + Err(error) => { + return BufferedOutcome::infrastructure(format!("failed to spawn `{program}`: {error}")); + } + }; + + let process_outcome = wait_for_process( + process, + timeout, + |process| process.inner().try_wait(), + |process| terminate_group_bounded(process, reaper), + "observe child process leader", + ); + combine_captured_output( + finish_capture(stdout, "stdout"), + finish_capture(stderr, "stderr"), + process_outcome.result, + ) +} + +fn combine_captured_output(stdout: CapturedStream, stderr: CapturedStream, result: InvocationResult) -> BufferedOutcome { + let failure = [stdout.failure.as_deref(), stderr.failure.as_deref()] + .into_iter() + .flatten() .collect::>() - .join(" ") + .join("; "); + let result = add_infrastructure_failure(result, failure); + BufferedOutcome { + stdout: stdout.output, + stderr: stderr.output, + result, + } } -/// Reduce a raw process exit code to the `u8` that [`ExitCode`] can carry. -/// -/// `ExitCode` is a `u8`, but process exit codes are wider: `None` means the -/// child was terminated by a signal (Unix) and non-`None` codes are a full -/// `i32` on Windows. We reduce a code to its low byte, which is a closer -/// approximation of the child's code than collapsing everything to `1`. Two -/// cases still map to `1`: a signal-terminated child (no numeric code), and a -/// non-zero code whose low byte is `0` (e.g. `256`) — which would otherwise be -/// indistinguishable from success. This function is only called on the -/// fail-fast path, where the child has already failed, so `1` is always a -/// correct non-zero fallback. -fn exit_byte(raw: Option) -> u8 { - let Some(raw) = raw else { return 1 }; - let byte = u8::try_from(raw & 0xFF).expect("`raw & 0xFF` masks to the low byte, always within 0..=255"); - if byte == 0 { 1 } else { byte } +fn add_infrastructure_failure(result: InvocationResult, failure: String) -> InvocationResult { + if failure.is_empty() { + return result; + } + InvocationResult::Infrastructure(match result { + InvocationResult::Infrastructure(primary) => format!("{primary}; {failure}"), + InvocationResult::TimedOut(duration) => { + format!("invocation timed out after {}; {failure}", display_duration(duration)) + } + InvocationResult::Exited(_) => failure, + }) +} + +fn command_for(invocation: &Invocation) -> Result<(&str, Command), String> { + let Some((program, arguments)) = invocation.argv.split_first() else { + return Err("internal command-plan error: invocation has an empty argument vector".to_owned()); + }; + let mut command = Command::new(program); + if let Some(path) = std::env::var_os("PATH") { + command.env("PATH", path); + } + command.args(arguments); + if let Some(directory) = &invocation.work_dir { + command.current_dir(directory); + } + Ok((program, command)) +} + +fn spawn_group(mut command: Command) -> Result { + command.group_spawn().map_err(|error| error.to_string()) +} + +fn spawn_child(mut command: Command) -> Result { + command.spawn().map_err(|error| error.to_string()) +} + +fn create_output_capture(_stream: &'static str) -> io::Result<(Box, Stdio)> { + let temporary = tempfile::NamedTempFile::new()?; + let reader = temporary.reopen()?; + let child = temporary.reopen()?; + let path = temporary.into_temp_path(); + Ok((Box::new(TemporarySnapshot { _path: path, reader }), Stdio::from(child))) +} + +fn finish_capture(mut source: Box, stream: &str) -> CapturedStream { + let result = source + .snapshot_len() + .and_then(|length| source.seek(SeekFrom::Start(0)).map(|_| length)); + match result { + Ok(length) => CapturedStream { + output: CapturedOutput::Snapshot { source, length }, + failure: None, + }, + Err(error) => CapturedStream { + output: CapturedOutput::empty(), + failure: Some(format!("failed to finalize child {stream} capture: {error}")), + }, + } +} + +fn wait_for_process( + mut control: T, + timeout: Option, + mut observe: impl FnMut(&mut T) -> io::Result>, + terminate: impl FnOnce(T) -> io::Result, + operation: &str, +) -> TreeOutcome { + let started = Instant::now(); + let mut terminate = Some(terminate); + loop { + if let Some(timeout) = timeout + && timeout.checked_sub(started.elapsed()).is_none() + { + return match terminate.take().expect("termination is consumed only on a returning branch")(control) { + Ok(_) => TreeOutcome::new(InvocationResult::TimedOut(timeout)), + Err(error) => TreeOutcome::new(InvocationResult::Infrastructure(format!( + "invocation timed out after {}; process-group termination failed: {error}", + display_duration(timeout) + ))), + }; + } + + match observe(&mut control) { + Ok(Some(status)) => return TreeOutcome::new(InvocationResult::Exited(status)), + Ok(None) => {} + Err(error) => { + let cleanup = terminate.take().expect("termination is consumed only on a returning branch")(control); + return TreeOutcome::new(InvocationResult::Infrastructure(with_cleanup_failure( + format!("failed to {operation}: {error}"), + &cleanup, + ))); + } + } + + let pause = timeout + .and_then(|timeout| timeout.checked_sub(started.elapsed())) + .map_or(Duration::from_millis(10), |remaining| remaining.min(Duration::from_millis(10))); + thread::sleep(pause); + } } -#[cfg(test)] #[cfg_attr(coverage_nightly, coverage(off))] -mod tests { - use super::exit_byte; +fn terminate_group_bounded(child: GroupChild, reaper: &ProcessReaper) -> io::Result { + terminate_group_with( + child, + TERMINATION_GRACE, + GroupChild::kill, + |child| child.inner().try_wait(), + |child| reaper.handoff_group(child), + ) +} - #[test] - fn signal_terminated_child_maps_to_one() { - assert_eq!(exit_byte(None), 1); +#[cfg_attr(coverage_nightly, coverage(off))] +fn terminate_child_bounded(child: Child, reaper: &ProcessReaper) -> io::Result { + terminate_group_with(child, TERMINATION_GRACE, Child::kill, Child::try_wait, |child| { + reaper.handoff_child(child) + }) +} + +fn terminate_group_with( + mut child: T, + grace: Duration, + kill: impl FnOnce(&mut T) -> io::Result<()>, + mut try_wait: impl FnMut(&mut T) -> io::Result>, + detach: impl FnOnce(T) -> io::Result<()>, +) -> io::Result { + let kill_error = kill(&mut child).err(); + let observed = poll_process_exit(&mut child, grace, &mut try_wait); + match observed { + Ok(Some(status)) => match kill_error { + Some(error) => Err(error), + None => Ok(status), + }, + Ok(None) => { + let reaper = detach(child); + let message = kill_error.map_or_else( + || format!("process boundary did not exit within {} ms after termination", grace.as_millis()), + |error| { + format!( + "{error}; process boundary did not exit within {} ms after termination", + grace.as_millis() + ) + }, + ); + Err(io::Error::new(io::ErrorKind::WouldBlock, with_reaper_handoff(&message, &reaper))) + } + Err(error) => { + let reaper = detach(child); + Err(io::Error::new( + error.kind(), + with_reaper_handoff(&format!("failed to observe process boundary after termination: {error}"), &reaper), + )) + } } +} - #[test] - fn in_range_codes_pass_through() { - assert_eq!(exit_byte(Some(1)), 1); - assert_eq!(exit_byte(Some(2)), 2); - assert_eq!(exit_byte(Some(255)), 255); +#[derive(Debug)] +struct ProcessReaper { + group_sender: mpsc::Sender, + child_sender: mpsc::Sender, +} + +impl Clone for ProcessReaper { + fn clone(&self) -> Self { + Self { + group_sender: self.group_sender.clone(), + child_sender: self.child_sender.clone(), + } } +} - #[test] - fn wide_codes_reduce_to_low_byte() { - // 259 = 0x103 -> low byte 3 (a common Windows code). - assert_eq!(exit_byte(Some(259)), 3); - assert_eq!(exit_byte(Some(257)), 1); +impl ProcessReaper { + fn start() -> io::Result { + Self::start_with(|job| { + thread::Builder::new() + .name("cargo-each-process-reaper".to_owned()) + .spawn(job) + .map(drop) + }) } - #[test] - fn nonzero_code_with_zero_low_byte_maps_to_one() { - // 256 = 0x100 -> low byte 0, which would look like success; map to 1. - assert_eq!(exit_byte(Some(256)), 1); - assert_eq!(exit_byte(Some(512)), 1); + fn start_with(mut spawn: impl FnMut(ReaperJob) -> io::Result<()>) -> io::Result { + let (group_sender, group_receiver) = mpsc::channel(); + spawn(Box::new(move || { + poll_reaper(&group_receiver, GroupChild::try_wait, report_reaper_failure); + }))?; + let (child_sender, child_receiver) = mpsc::channel(); + spawn(Box::new(move || { + poll_reaper(&child_receiver, Child::try_wait, report_reaper_failure); + }))?; + Ok(Self { + group_sender, + child_sender, + }) + } + + fn handoff_group(&self, child: GroupChild) -> io::Result<()> { + let retained = failed_handoffs(); + handoff_group_with_fallback(&self.group_sender, retained, child, || start_failed_handoff_reaper(retained)) + } + + fn handoff_child(&self, child: Child) -> io::Result<()> { + let retained = failed_child_handoffs(); + handoff_group_with_fallback(&self.child_sender, retained, child, || start_failed_child_handoff_reaper(retained)) + } +} + +fn failed_handoffs() -> &'static Mutex> { + static FAILED_HANDOFFS: OnceLock>> = OnceLock::new(); + FAILED_HANDOFFS.get_or_init(|| Mutex::new(Vec::new())) +} + +fn start_failed_handoff_reaper(retained: &'static Mutex>) -> io::Result<()> { + static RUNNING: AtomicBool = AtomicBool::new(false); + start_failed_handoff_reaper_with(retained, &RUNNING, GroupChild::try_wait, report_reaper_failure, |job| { + thread::Builder::new() + .name("cargo-each-fallback-reaper".to_owned()) + .spawn(job) + .map(drop) + }) +} + +fn failed_child_handoffs() -> &'static Mutex> { + static FAILED_HANDOFFS: OnceLock>> = OnceLock::new(); + FAILED_HANDOFFS.get_or_init(|| Mutex::new(Vec::new())) +} + +fn start_failed_child_handoff_reaper(retained: &'static Mutex>) -> io::Result<()> { + static RUNNING: AtomicBool = AtomicBool::new(false); + start_failed_handoff_reaper_with(retained, &RUNNING, Child::try_wait, report_reaper_failure, |job| { + thread::Builder::new() + .name("cargo-each-fallback-child-reaper".to_owned()) + .spawn(job) + .map(drop) + }) +} + +fn start_failed_handoff_reaper_with( + retained: &'static Mutex>, + running: &'static AtomicBool, + try_wait: fn(&mut T) -> io::Result>, + report_failure: fn(&io::Error), + spawn: impl FnOnce(ReaperJob) -> io::Result<()>, +) -> io::Result<()> { + if running.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire).is_err() { + return Ok(()); + } + match spawn(Box::new(move || { + poll_failed_handoffs(retained, running, try_wait, report_failure); + })) { + Ok(()) => Ok(()), + Err(error) => { + running.store(false, Ordering::Release); + Err(error) + } + } +} + +fn poll_failed_handoffs( + retained: &Mutex>, + running: &AtomicBool, + mut try_wait: impl FnMut(&mut T) -> io::Result>, + mut report_failure: impl FnMut(&io::Error), +) { + loop { + let mut children = retained.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + children.retain_mut(|child| retain_after_reaper_observation(try_wait(child), &mut report_failure)); + if children.is_empty() { + running.store(false, Ordering::Release); + return; + } + drop(children); + thread::sleep(REAPER_POLL_INTERVAL); + } +} + +fn handoff_group(sender: &mpsc::Sender, retained: &Mutex>, child: T) -> io::Result<()> { + match sender.send(child) { + Ok(()) => Ok(()), + Err(error) => { + retained.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push(error.0); + Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "process reaper channel disconnected; a persistent fallback retained the wait handle", + )) + } + } +} + +fn handoff_group_with_fallback( + sender: &mpsc::Sender, + retained: &Mutex>, + child: T, + start_fallback: impl FnOnce() -> io::Result<()>, +) -> io::Result<()> { + let handoff = handoff_group(sender, retained, child); + if handoff.is_err() + && let Err(error) = start_fallback() + { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + format!("process reaper channel disconnected; the fallback retained the wait handle but failed to start: {error}"), + )); + } + handoff +} + +#[cfg_attr(coverage_nightly, coverage(off))] +#[mutants::skip] // Process-thread diagnostic for an OS observation failure; behavior is covered through the injected reporter seam. +fn report_reaper_failure(error: &io::Error) { + let message = error.to_string(); + let _ = thread::Builder::new() + .name("cargo-each-reaper-diagnostic".to_owned()) + .spawn(move || { + let _ = writeln!( + io::stderr().lock(), + "cargo each: process reaper failed to observe a retained wait handle: {message}" + ); + }); +} + +#[mutants::skip] // Deleting the disconnected-and-empty shutdown arm hangs by definition; deterministic tests cover polling and exit. +fn poll_reaper( + receiver: &mpsc::Receiver, + mut try_wait: impl FnMut(&mut T) -> io::Result>, + mut report_failure: impl FnMut(&io::Error), +) { + let mut retained: Vec = Vec::new(); + let mut connected = true; + loop { + if connected { + match receiver.recv_timeout(REAPER_POLL_INTERVAL) { + Ok(child) => retained.push(child), + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => connected = false, + } + } + + retained.retain_mut(|child| retain_after_reaper_observation(try_wait(child), &mut report_failure)); + + match (connected, retained.is_empty()) { + (false, true) => return, + _ => thread::sleep(REAPER_POLL_INTERVAL), + } + } +} + +fn retain_after_reaper_observation(observation: io::Result>, report_failure: &mut impl FnMut(&io::Error)) -> bool { + match observation { + Ok(Some(_)) => false, + Ok(None) => true, + Err(error) if error.kind() == io::ErrorKind::Interrupted => true, + Err(error) => { + report_failure(&error); + false + } + } +} + +fn with_reaper_handoff(message: &str, reaper: &io::Result<()>) -> String { + match reaper { + Ok(()) => format!("{message}; the process wait handle was moved to the local polling reaper"), + Err(error) => format!("{message}; failed to hand the process wait handle to the local reaper: {error}"), + } +} + +fn poll_process_exit( + control: &mut T, + grace: Duration, + mut try_wait: impl FnMut(&mut T) -> io::Result>, +) -> io::Result> { + let started = Instant::now(); + loop { + if let Some(status) = try_wait(control)? { + return Ok(Some(status)); + } + let Some(remaining) = grace.checked_sub(started.elapsed()) else { + return Ok(None); + }; + thread::sleep(remaining.min(Duration::from_millis(10))); + } +} + +fn with_cleanup_failure(message: String, cleanup: &io::Result) -> String { + match cleanup { + Ok(_) => message, + Err(error) => format!("{message}; process-group cleanup also failed: {error}"), + } +} + +fn emit_label(invocation: &Invocation) { + if let Some(label) = &invocation.label { + eprintln!("cargo each: {label}"); + } +} + +fn emit_buffered(invocation: &Invocation, outcome: &mut BufferedOutcome) -> io::Result<()> { + let mut stdout = io::stdout().lock(); + let mut stderr = io::stderr().lock(); + emit_buffered_to(invocation, outcome, &mut stdout, &mut stderr) +} + +fn emit_buffered_to( + invocation: &Invocation, + outcome: &mut BufferedOutcome, + stdout: &mut dyn io::Write, + stderr: &mut dyn io::Write, +) -> io::Result<()> { + emit_label(invocation); + let mut source_failures = Vec::new(); + match outcome.stdout.emit_to(stdout) { + Ok(()) => {} + Err(OutputEmitError::Source(error)) => { + source_failures.push(format!("failed to read captured child stdout: {error}")); + } + Err(OutputEmitError::Destination(error)) => return Err(error), + } + stdout.flush()?; + match outcome.stderr.emit_to(stderr) { + Ok(()) => {} + Err(OutputEmitError::Source(error)) => { + source_failures.push(format!("failed to read captured child stderr: {error}")); + } + Err(OutputEmitError::Destination(error)) => return Err(error), + } + if !source_failures.is_empty() { + let original = std::mem::replace( + &mut outcome.result, + InvocationResult::Infrastructure("output emission failed".to_owned()), + ); + outcome.result = add_infrastructure_failure(original, source_failures.join("; ")); + } + match &outcome.result { + InvocationResult::TimedOut(duration) => { + writeln!(stderr, "cargo each: invocation timed out after {}", display_duration(*duration))?; + } + InvocationResult::Infrastructure(message) => { + writeln!(stderr, "cargo each: {message}")?; + } + InvocationResult::Exited(_) => {} + } + stderr.flush() +} + +fn display_duration(duration: Duration) -> String { + if duration.subsec_nanos() == 0 && duration.as_secs().is_multiple_of(60) { + format!("{}m", duration.as_secs() / 60) + } else if duration.subsec_nanos() == 0 { + format!("{}s", duration.as_secs()) + } else { + format!("{}ms", duration.as_millis()) + } +} + +#[derive(Debug)] +struct IndexedOutcome { + index: usize, + outcome: BufferedOutcome, +} + +#[derive(Debug)] +struct RunningWorker { + index: usize, + receiver: mpsc::Receiver, + thread: thread::JoinHandle<()>, +} + +trait SnapshotSource: io::Read + io::Seek + Send + fmt::Debug { + fn snapshot_len(&self) -> io::Result; +} + +#[derive(Debug)] +struct TemporarySnapshot { + _path: tempfile::TempPath, + reader: std::fs::File, +} + +impl io::Read for TemporarySnapshot { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.reader.read(buf) + } +} + +impl io::Seek for TemporarySnapshot { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + self.reader.seek(pos) + } +} + +impl SnapshotSource for TemporarySnapshot { + fn snapshot_len(&self) -> io::Result { + self.reader.metadata().map(|metadata| metadata.len()) + } +} + +#[cfg(test)] +#[cfg_attr(coverage_nightly, coverage(off))] +impl SnapshotSource for io::Cursor> { + fn snapshot_len(&self) -> io::Result { + u64::try_from(self.get_ref().len()).map_err(io::Error::other) + } +} + +#[derive(Debug)] +enum CapturedOutput { + Empty, + Snapshot { source: Box, length: u64 }, +} + +impl CapturedOutput { + fn empty() -> Self { + Self::Empty + } + + fn emit_to(&mut self, destination: &mut dyn io::Write) -> Result<(), OutputEmitError> { + match self { + Self::Empty => Ok(()), + Self::Snapshot { source, length } => { + source.seek(SeekFrom::Start(0)).map_err(OutputEmitError::Source)?; + let mut buffer = [0_u8; 8192]; + let mut remaining = *length; + while remaining > 0 { + let limit = usize::try_from(remaining.min(buffer.len() as u64)) + .expect("the read size is capped by the 8192-byte buffer length"); + let read = source.read(&mut buffer[..limit]).map_err(OutputEmitError::Source)?; + let Some(read) = NonZeroUsize::new(read) else { + return Err(OutputEmitError::Source(io::Error::new( + io::ErrorKind::UnexpectedEof, + "captured output ended before its finalized snapshot length", + ))); + }; + destination.write_all(&buffer[..read.get()]).map_err(OutputEmitError::Destination)?; + remaining -= u64::try_from(read.get()).expect("a read byte count always fits in u64"); + } + Ok(()) + } + } + } +} + +#[derive(Debug)] +enum OutputEmitError { + Source(io::Error), + Destination(io::Error), +} + +#[derive(Debug)] +struct CapturedStream { + output: CapturedOutput, + failure: Option, +} + +#[derive(Debug)] +struct TreeOutcome { + result: InvocationResult, +} + +impl TreeOutcome { + fn new(result: InvocationResult) -> Self { + Self { result } + } +} + +#[derive(Debug)] +struct BufferedOutcome { + stdout: CapturedOutput, + stderr: CapturedOutput, + result: InvocationResult, +} + +impl BufferedOutcome { + fn infrastructure(message: String) -> Self { + Self { + stdout: CapturedOutput::empty(), + stderr: CapturedOutput::empty(), + result: InvocationResult::Infrastructure(message), + } + } +} + +#[derive(Debug)] +enum InvocationResult { + Exited(ExitStatus), + TimedOut(Duration), + Infrastructure(String), +} + +impl InvocationResult { + fn failed(&self) -> bool { + match self { + Self::Exited(status) => !status.success(), + Self::TimedOut(_) | Self::Infrastructure(_) => true, + } + } +} + +/// Render an argv for display (`--dry-run`). Best-effort quoting for +/// readability only — nothing consumes this as input. +fn shell_join(argv: &[String]) -> String { + argv.iter() + .map(|a| { + if a.contains(char::is_whitespace) { + format!("\"{a}\"") + } else { + a.clone() + } + }) + .collect::>() + .join(" ") +} + +/// Reduce a raw process exit code to the `u8` that [`ExitCode`] can carry. +fn exit_byte(raw: Option) -> u8 { + let Some(raw) = raw else { return 1 }; + let byte = u8::try_from(raw & 0xFF).expect("`raw & 0xFF` masks to the low byte, always within 0..=255"); + if byte == 0 { 1 } else { byte } +} + +#[cfg(test)] +#[cfg_attr(coverage_nightly, coverage(off))] +mod tests { + use std::collections::VecDeque; + use std::io::{Read as _, Seek as _}; + use std::num::NonZeroUsize; + #[cfg(unix)] + use std::os::unix::process::ExitStatusExt as _; + #[cfg(windows)] + use std::os::windows::process::ExitStatusExt as _; + use std::process::{Command, ExitCode, ExitStatus, Stdio}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex, OnceLock, mpsc}; + use std::time::{Duration, Instant}; + use std::{io, thread}; + + use super::{ + BufferedOutcome, CapturedOutput, CapturedStream, Invocation, InvocationResult, OutputEmitError, Plan, ProcessReaper, RunningWorker, + SnapshotSource, StreamedChild, TemporarySnapshot, TreeOutcome, WORKER_PANIC_TEST_PROGRAM, WORKER_SPAWN_ERROR_TEST_PROGRAM, + add_infrastructure_failure, combine_captured_output, display_duration, effective_worker_count, emit_buffered_to, execute_parallel, + exit_byte, failed_child_handoffs, failed_handoffs, failure_stops_launching, finish_capture, handoff_group, + handoff_group_with_fallback, observe_streamed_child, panic_description, parallel_failure_exit_code, poll_process_exit, poll_reaper, + record_emitted_failure, retain_after_reaper_observation, run_captured, run_captured_with, run_streamed, run_streamed_with_timeout, + run_streamed_with_timeout_with, spawn_group, spawn_worker, start_failed_handoff_reaper_with, terminate_child_bounded, + terminate_group_bounded, terminate_group_with, wait_for_process, wait_for_worker, with_cleanup_failure, with_reaper_handoff, + }; + + fn invocation(argv: &[&str]) -> Invocation { + Invocation { + label: None, + argv: argv.iter().map(|value| (*value).to_owned()).collect(), + work_dir: None, + } + } + + fn successful_status() -> ExitStatus { + ExitStatus::from_raw(0) + } + + #[cfg(unix)] + fn failed_status(code: i32) -> ExitStatus { + ExitStatus::from_raw(code << 8) + } + + #[cfg(windows)] + fn failed_status(code: i32) -> ExitStatus { + ExitStatus::from_raw(u32::try_from(code).expect("test exit code is nonnegative")) + } + + static FALLBACK_TEST_GROUPS: OnceLock>> = OnceLock::new(); + static FALLBACK_TEST_RUNNING: AtomicBool = AtomicBool::new(false); + static FALLBACK_TEST_COLLECTED: AtomicUsize = AtomicUsize::new(0); + static FALLBACK_TEST_REPORTED: AtomicUsize = AtomicUsize::new(0); + + fn observe_fallback_test_group(state: &mut usize) -> io::Result> { + match *state { + 3 => Err(io::Error::other("injected terminal fallback observation failure")), + 2 => { + *state = 1; + Err(io::Error::new( + io::ErrorKind::Interrupted, + "injected interrupted fallback observation", + )) + } + 1 => { + *state = 0; + Ok(None) + } + _ => { + FALLBACK_TEST_COLLECTED.fetch_add(1, Ordering::SeqCst); + Ok(Some(successful_status())) + } + } + } + + fn report_fallback_test_error(_error: &io::Error) { + FALLBACK_TEST_REPORTED.fetch_add(1, Ordering::SeqCst); + } + + fn result_infrastructure_message(result: InvocationResult) -> String { + let InvocationResult::Infrastructure(message) = result else { + panic!("the test expects an infrastructure outcome"); + }; + message + } + + fn infrastructure_message(outcome: BufferedOutcome) -> String { + result_infrastructure_message(outcome.result) + } + + fn output_bytes(output: &mut CapturedOutput) -> Vec { + let mut bytes = Vec::new(); + match output.emit_to(&mut bytes) { + Ok(()) => bytes, + Err(OutputEmitError::Source(error) | OutputEmitError::Destination(error)) => { + panic!("captured test output cannot be read: {error}") + } + } + } + + fn captured(bytes: &[u8], failure: Option<&str>) -> CapturedStream { + CapturedStream { + output: CapturedOutput::Snapshot { + source: Box::new(io::Cursor::new(bytes.to_vec())), + length: u64::try_from(bytes.len()).expect("test output length fits in u64"), + }, + failure: failure.map(str::to_owned), + } + } + + fn test_reaper() -> ProcessReaper { + ProcessReaper::start().expect("the test process can start its reaper") + } + + struct FakeProcess { + observations: VecDeque>>, + termination: Option>, + } + + impl FakeProcess { + fn observe(&mut self) -> io::Result> { + self.observations.pop_front().unwrap_or(Ok(None)) + } + + fn terminate(mut self) -> io::Result { + self.termination + .take() + .unwrap_or_else(|| Err(io::Error::other("unexpected termination"))) + } + } + + #[derive(Debug)] + struct FaultySnapshot { + cursor: io::Cursor>, + reported_length: u64, + fail_length: bool, + fail_seek: bool, + fail_read: bool, + } + + impl io::Read for FaultySnapshot { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.fail_read { + Err(io::Error::other("injected snapshot read failure")) + } else { + io::Read::read(&mut self.cursor, buf) + } + } + } + + impl io::Seek for FaultySnapshot { + fn seek(&mut self, pos: io::SeekFrom) -> io::Result { + if self.fail_seek { + Err(io::Error::other("injected snapshot seek failure")) + } else { + io::Seek::seek(&mut self.cursor, pos) + } + } + } + + impl SnapshotSource for FaultySnapshot { + fn snapshot_len(&self) -> io::Result { + if self.fail_length { + Err(io::Error::other("injected snapshot length failure")) + } else { + Ok(self.reported_length) + } + } + } + + struct FailingWriter; + + impl io::Write for FailingWriter { + fn write(&mut self, _buffer: &[u8]) -> io::Result { + Err(io::Error::other("injected destination write failure")) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + fn sleeping_test_command() -> Command { + let mut command = Command::new(std::env::current_exe().expect("the test binary knows its path")); + let _ = command + .args(["--exact", "run::tests::child_sleep_probe", "--nocapture"]) + .env("CARGO_EACH_CHILD_SLEEP_MS", "30000") + .stdout(Stdio::null()) + .stderr(Stdio::null()); + command + } + + #[test] + fn child_sleep_probe() { + if let Some(duration) = std::env::var_os("CARGO_EACH_CHILD_SLEEP_MS") { + let millis = duration.to_string_lossy().parse().expect("the parent passes milliseconds"); + thread::sleep(Duration::from_millis(millis)); + } + } + + #[test] + fn exit_codes_and_durations_follow_the_cli_contract() { + assert_eq!(exit_byte(None), 1); + assert_eq!(exit_byte(Some(7)), 7); + assert_eq!(exit_byte(Some(259)), 3); + assert_eq!(exit_byte(Some(256)), 1); + assert_eq!(display_duration(Duration::from_millis(250)), "250ms"); + assert_eq!(display_duration(Duration::from_secs(30)), "30s"); + assert_eq!(display_duration(Duration::from_mins(2)), "2m"); + } + + #[test] + fn worker_count_and_launch_policy_are_plan_bounded() { + let four = NonZeroUsize::new(4).expect("literal four is nonzero"); + assert_eq!(effective_worker_count(four, 1), NonZeroUsize::MIN); + assert_eq!(effective_worker_count(four, 8), four); + assert!(failure_stops_launching(false, true)); + assert!(!failure_stops_launching(false, false)); + assert!(!failure_stops_launching(true, true)); + } + + #[test] + fn emitted_capture_failures_update_parallel_scheduler_state() { + let result = InvocationResult::Infrastructure("capture read failed".to_owned()); + let mut any_failed = false; + let mut stop_launching = false; + let mut first_failure = None; + record_emitted_failure( + &InvocationResult::Exited(successful_status()), + false, + &mut any_failed, + &mut stop_launching, + &mut first_failure, + ); + assert!(!any_failed); + assert!(!stop_launching); + assert!(first_failure.is_none()); + + record_emitted_failure(&result, false, &mut any_failed, &mut stop_launching, &mut first_failure); + assert!(any_failed); + assert!(stop_launching); + assert_eq!(first_failure, Some(ExitCode::from(2))); + + let mut keep_going_failed = false; + let mut keep_going_stop = false; + let mut keep_going_first = None; + record_emitted_failure(&result, true, &mut keep_going_failed, &mut keep_going_stop, &mut keep_going_first); + assert!(keep_going_failed); + assert!(!keep_going_stop); + assert_eq!(keep_going_first, Some(ExitCode::from(2))); + + record_emitted_failure( + &InvocationResult::Exited(failed_status(7)), + true, + &mut keep_going_failed, + &mut keep_going_stop, + &mut keep_going_first, + ); + assert_eq!(keep_going_first, Some(ExitCode::from(2)), "the first plan-order failure wins"); + } + + #[test] + fn sequential_and_parallel_failures_preserve_their_taxonomy() { + let plan = Plan { + invocations: vec![invocation(&["first"]), invocation(&["second"])], + }; + let mut results = VecDeque::from([ + InvocationResult::TimedOut(Duration::from_millis(50)), + InvocationResult::Exited(successful_status()), + ]); + let mut calls = 0; + let fail_fast = super::execute_sequential_with(&plan, false, Some(Duration::from_millis(50)), |_, timeout| { + assert_eq!(timeout, Some(Duration::from_millis(50))); + calls += 1; + results.pop_front().expect("one result per launched invocation") + }); + assert_eq!(fail_fast, ExitCode::from(1)); + assert_eq!(calls, 1); + assert_eq!( + parallel_failure_exit_code(&InvocationResult::Exited(failed_status(7))), + ExitCode::from(7) + ); + assert_eq!( + parallel_failure_exit_code(&InvocationResult::Infrastructure("capture failed".to_owned())), + ExitCode::from(2) + ); + assert_eq!( + parallel_failure_exit_code(&InvocationResult::TimedOut(Duration::from_secs(1))), + ExitCode::from(1) + ); + } + + #[test] + #[cfg_attr(miri, ignore = "spawns rustc subprocesses")] + fn scheduler_surfaces_worker_launch_failures_in_both_policies() { + let reaper = test_reaper(); + let fail_fast = Plan { + invocations: vec![invocation(&[WORKER_SPAWN_ERROR_TEST_PROGRAM]), invocation(&["rustc", "--version"])], + }; + let code = execute_parallel( + &fail_fast, + false, + NonZeroUsize::new(2).expect("literal two is nonzero"), + None, + &reaper, + ) + .expect("worker launch failure is an invocation outcome"); + assert_eq!(code, ExitCode::from(2)); + + let keep_going = Plan { + invocations: vec![invocation(&[WORKER_SPAWN_ERROR_TEST_PROGRAM]), invocation(&["rustc", "--version"])], + }; + let code = execute_parallel( + &keep_going, + true, + NonZeroUsize::new(2).expect("literal two is nonzero"), + None, + &reaper, + ) + .expect("keep-going retains worker launch failures"); + assert_eq!(code, ExitCode::from(1)); + } + + #[test] + fn process_waiting_handles_completion_timeout_and_cleanup_failures() { + let completed = FakeProcess { + observations: VecDeque::from([Ok(None), Ok(Some(successful_status()))]), + termination: None, + }; + let outcome = wait_for_process( + completed, + None, + FakeProcess::observe, + FakeProcess::terminate, + "observe fake process", + ); + assert!(matches!(outcome.result, InvocationResult::Exited(status) if status.success())); + + let timed_out = FakeProcess { + observations: VecDeque::from([Ok(None)]), + termination: Some(Ok(successful_status())), + }; + let outcome = wait_for_process( + timed_out, + Some(Duration::ZERO), + FakeProcess::observe, + FakeProcess::terminate, + "observe fake process", + ); + assert!(matches!(outcome.result, InvocationResult::TimedOut(duration) if duration.is_zero())); + + let late_success = FakeProcess { + observations: VecDeque::from([Ok(None), Ok(Some(successful_status()))]), + termination: Some(Ok(successful_status())), + }; + let outcome = wait_for_process( + late_success, + Some(Duration::from_millis(1)), + FakeProcess::observe, + FakeProcess::terminate, + "observe fake process", + ); + assert!( + matches!(outcome.result, InvocationResult::TimedOut(duration) if duration == Duration::from_millis(1)), + "completion observed after the deadline must not win the race" + ); + + let failed = FakeProcess { + observations: VecDeque::from([Err(io::Error::other("observation failed"))]), + termination: Some(Err(io::Error::other("cleanup failed"))), + }; + let outcome = wait_for_process(failed, None, FakeProcess::observe, FakeProcess::terminate, "observe fake process"); + let message = result_infrastructure_message(outcome.result); + assert!(message.contains("observation failed")); + assert!(message.contains("cleanup failed")); + + let failed_timeout_cleanup = FakeProcess { + observations: VecDeque::from([Ok(None)]), + termination: Some(Err(io::Error::other("timeout cleanup failed"))), + }; + let outcome = wait_for_process( + failed_timeout_cleanup, + Some(Duration::ZERO), + FakeProcess::observe, + FakeProcess::terminate, + "observe fake process", + ); + assert!(result_infrastructure_message(outcome.result).contains("timeout cleanup failed")); + } + + #[test] + fn bounded_polling_stops_on_exit_error_or_deadline() { + let mut exited = VecDeque::from([Ok(None), Ok(Some(successful_status()))]); + let status = poll_process_exit(&mut exited, Duration::from_secs(1), |observations| { + observations.pop_front().expect("the fake has enough observations") + }) + .expect("polling succeeds") + .expect("the fake exits"); + assert!(status.success()); + + let mut failed = VecDeque::from([Err(io::Error::other("poll failed"))]); + let error = poll_process_exit(&mut failed, Duration::from_secs(1), |observations| { + observations.pop_front().expect("the fake has one observation") + }) + .expect_err("polling failure propagates"); + assert!(error.to_string().contains("poll failed")); + + let mut running = (); + assert!( + poll_process_exit(&mut running, Duration::ZERO, |()| Ok(None)) + .expect("deadline is not an I/O failure") + .is_none() + ); + } + + #[test] + fn bounded_group_termination_reports_every_local_failure_shape() { + let status = successful_status(); + let kill_error = terminate_group_with( + (), + Duration::from_secs(1), + |()| Err(io::Error::new(io::ErrorKind::PermissionDenied, "kill failed")), + move |()| Ok(Some(status)), + |()| Ok(()), + ) + .expect_err("a kill error is not hidden by later completion"); + assert_eq!(kill_error.kind(), io::ErrorKind::PermissionDenied); + + let deadline = terminate_group_with((), Duration::ZERO, |()| Ok(()), |()| Ok(None), |()| Ok(())) + .expect_err("an unreaped group reaches the deadline"); + assert_eq!(deadline.kind(), io::ErrorKind::WouldBlock); + assert!(deadline.to_string().contains("local polling reaper")); + + let failed_handoff = terminate_group_with( + (), + Duration::ZERO, + |()| Err(io::Error::other("kill failed")), + |()| Ok(None), + |()| Err(io::Error::other("reaper failed")), + ) + .expect_err("kill and handoff failures are both reported"); + assert!(failed_handoff.to_string().contains("kill failed")); + assert!(failed_handoff.to_string().contains("reaper failed")); + + let failed_observation = terminate_group_with( + (), + Duration::from_secs(1), + |()| Ok(()), + |()| Err(io::Error::other("observation failed")), + |()| Ok(()), + ) + .expect_err("post-kill observation failure is reported"); + assert!(failed_observation.to_string().contains("observation failed")); + assert!(failed_observation.to_string().contains("local polling reaper")); + } + + #[test] + fn reaper_startup_failure_is_reported_synchronously() { + let error = ProcessReaper::start_with(|job| { + drop(job); + Err(io::Error::other("injected reaper startup failure")) + }) + .expect_err("startup failure must be returned"); + assert!(error.to_string().contains("injected reaper startup failure")); + + let mut starts = 0; + let error = ProcessReaper::start_with(|job| { + starts += 1; + if starts == 1 { + thread::Builder::new().spawn(job).map(drop) + } else { + drop(job); + Err(io::Error::other("injected child-reaper startup failure")) + } + }) + .expect_err("child-reaper startup failure must be returned"); + assert!(error.to_string().contains("injected child-reaper startup failure")); + } + + #[test] + fn polling_reaper_checks_every_retained_group_and_eventually_collects_them() { + struct FakeGroup { + errors_remaining: usize, + error_kind: io::ErrorKind, + polls_remaining: usize, + collected: Arc, + } + + let (sender, receiver) = mpsc::channel(); + let collected = Arc::new(AtomicUsize::new(0)); + let reports = Arc::new(AtomicUsize::new(0)); + let first = FakeGroup { + errors_remaining: 2, + error_kind: io::ErrorKind::Interrupted, + polls_remaining: 20, + collected: Arc::clone(&collected), + }; + let second = FakeGroup { + errors_remaining: 0, + error_kind: io::ErrorKind::Other, + polls_remaining: 0, + collected: Arc::clone(&collected), + }; + let terminal = FakeGroup { + errors_remaining: 1, + error_kind: io::ErrorKind::Other, + polls_remaining: 0, + collected: Arc::clone(&collected), + }; + sender.send(first).expect("reaper receiver is connected"); + sender.send(second).expect("reaper receiver is connected"); + sender.send(terminal).expect("reaper receiver is connected"); + drop(sender); + + let (done_sender, done_receiver) = mpsc::channel(); + let worker = thread::spawn({ + let reports = Arc::clone(&reports); + move || { + poll_reaper( + &receiver, + |group| { + if group.errors_remaining > 0 { + group.errors_remaining -= 1; + return Err(io::Error::new(group.error_kind, "injected reaper observation failure")); + } + if group.polls_remaining == 0 { + group.collected.fetch_add(1, Ordering::SeqCst); + Ok(Some(successful_status())) + } else { + group.polls_remaining -= 1; + Ok(None) + } + }, + |_| { + reports.fetch_add(1, Ordering::SeqCst); + }, + ); + let _receiver_gone = done_sender.send(()); + } + }); + let deadline = Instant::now() + Duration::from_secs(1); + while collected.load(Ordering::SeqCst) == 0 && Instant::now() < deadline { + thread::sleep(Duration::from_millis(1)); + } + assert_eq!( + collected.load(Ordering::SeqCst), + 1, + "the ready group must be collected while another group remains pending" + ); + done_receiver + .recv_timeout(Duration::from_secs(1)) + .expect("the finite fake reaper must stop after the sender disconnects"); + worker.join().expect("the finite fake reaper exits"); + assert_eq!(collected.load(Ordering::SeqCst), 2); + assert_eq!(reports.load(Ordering::SeqCst), 1, "each failing group is reported once"); + } + + #[test] + fn reaper_observation_retention_distinguishes_transient_and_terminal_states() { + let mut reported = Vec::new(); + assert!(!retain_after_reaper_observation(Ok(Some(successful_status())), &mut |error| { + reported.push(error.kind()); + })); + assert!(retain_after_reaper_observation(Ok(None), &mut |error| { + reported.push(error.kind()); + })); + assert!(retain_after_reaper_observation( + Err(io::Error::new(io::ErrorKind::Interrupted, "interrupted")), + &mut |error| { + reported.push(error.kind()); + }, + )); + assert!(!retain_after_reaper_observation(Err(io::Error::other("terminal")), &mut |error| { + reported.push(error.kind()); + },)); + assert_eq!(reported, [io::ErrorKind::Other]); + } + + #[test] + fn failed_reaper_handoff_recovers_ownership_before_returning() { + let (connected_sender, connected_receiver) = mpsc::channel(); + let retained = Mutex::new(Vec::new()); + handoff_group(&connected_sender, &retained, "delivered group").expect("connected handoff succeeds"); + assert_eq!(connected_receiver.try_recv().expect("group is delivered"), "delivered group"); + assert!(retained.lock().expect("fallback ownership mutex is not poisoned").is_empty()); + + let (sender, receiver) = mpsc::channel(); + drop(receiver); + let retained = Mutex::new(Vec::new()); + let error = handoff_group(&sender, &retained, "owned group").expect_err("disconnected handoff is reported"); + assert_eq!(error.kind(), io::ErrorKind::BrokenPipe); + assert_eq!( + retained.lock().expect("fallback ownership mutex is not poisoned").as_slice(), + ["owned group"] + ); + } + + #[test] + fn fallback_handoff_reports_startup_failure_without_losing_ownership() { + let (sender, receiver) = mpsc::channel(); + drop(receiver); + let retained = Mutex::new(Vec::new()); + let error = handoff_group_with_fallback(&sender, &retained, "owned group", || { + Err(io::Error::other("injected fallback startup failure")) + }) + .expect_err("fallback startup failure is reported"); + assert_eq!(error.kind(), io::ErrorKind::BrokenPipe); + assert!(error.to_string().contains("injected fallback startup failure")); + assert_eq!( + retained.lock().expect("fallback ownership mutex is not poisoned").as_slice(), + ["owned group"] + ); + } + + #[test] + fn fallback_reaper_startup_and_polling_cover_every_state() { + let retained = FALLBACK_TEST_GROUPS.get_or_init(|| Mutex::new(Vec::new())); + retained.lock().expect("fallback ownership mutex is not poisoned").clear(); + FALLBACK_TEST_RUNNING.store(true, Ordering::Release); + FALLBACK_TEST_COLLECTED.store(0, Ordering::SeqCst); + FALLBACK_TEST_REPORTED.store(0, Ordering::SeqCst); + start_failed_handoff_reaper_with( + retained, + &FALLBACK_TEST_RUNNING, + observe_fallback_test_group, + report_fallback_test_error, + |_| { + panic!("an already-running fallback must not spawn another thread"); + }, + ) + .expect("an already-running fallback accepts more work"); + + FALLBACK_TEST_RUNNING.store(false, Ordering::Release); + retained.lock().expect("fallback ownership mutex is not poisoned").push(2); + let error = start_failed_handoff_reaper_with( + retained, + &FALLBACK_TEST_RUNNING, + observe_fallback_test_group, + report_fallback_test_error, + |job| { + drop(job); + Err(io::Error::other("injected fallback thread failure")) + }, + ) + .expect_err("fallback thread failure is reported"); + assert!(error.to_string().contains("injected fallback thread failure")); + assert!(!FALLBACK_TEST_RUNNING.load(Ordering::Acquire)); + assert_eq!(retained.lock().expect("fallback ownership mutex is not poisoned").len(), 1); + + retained.lock().expect("fallback ownership mutex is not poisoned").push(3); + start_failed_handoff_reaper_with( + retained, + &FALLBACK_TEST_RUNNING, + observe_fallback_test_group, + report_fallback_test_error, + |job| thread::Builder::new().spawn(job).map(drop), + ) + .expect("fallback polling thread starts"); + let deadline = Instant::now() + Duration::from_secs(1); + while FALLBACK_TEST_RUNNING.load(Ordering::Acquire) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(1)); + } + assert!( + !FALLBACK_TEST_RUNNING.load(Ordering::Acquire), + "fallback polling thread did not finish" + ); + assert_eq!(FALLBACK_TEST_COLLECTED.load(Ordering::SeqCst), 1); + assert_eq!(FALLBACK_TEST_REPORTED.load(Ordering::SeqCst), 1); + assert!(retained.lock().expect("fallback ownership mutex is not poisoned").is_empty()); + } + + #[test] + fn failed_handoff_storage_is_process_stable() { + assert!(std::ptr::eq(failed_handoffs(), failed_handoffs())); + assert!(std::ptr::eq(failed_child_handoffs(), failed_child_handoffs())); + } + + #[test] + #[cfg_attr(miri, ignore = "spawns a process group and a fallback reaper thread")] + fn disconnected_reaper_handoff_is_eventually_collected() { + let (sender, receiver) = mpsc::channel(); + drop(receiver); + let (child_sender, _child_receiver) = mpsc::channel(); + let reaper = ProcessReaper { + group_sender: sender, + child_sender, + }; + let mut command = Command::new("rustc"); + let _ = command.arg("--version").stdout(Stdio::null()).stderr(Stdio::null()); + let child = spawn_group(command).expect("spawn a short-lived process group"); + + let error = reaper + .handoff_group(child) + .expect_err("the disconnected primary reaper is reported"); + assert_eq!(error.kind(), io::ErrorKind::BrokenPipe); + let deadline = Instant::now() + Duration::from_secs(2); + while !failed_handoffs() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .is_empty() + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(10)); + } + + assert!( + failed_handoffs() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .is_empty(), + "the fallback reaper must eventually collect a recovered handoff" + ); + } + + #[test] + #[cfg_attr(miri, ignore = "spawns a child process and a fallback reaper thread")] + fn disconnected_child_reaper_handoff_is_eventually_collected() { + let (group_sender, _group_receiver) = mpsc::channel(); + let (child_sender, child_receiver) = mpsc::channel(); + drop(child_receiver); + let reaper = ProcessReaper { + group_sender, + child_sender, + }; + let mut command = Command::new("rustc"); + let child = command + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn a short-lived child"); + + let error = reaper.handoff_child(child).expect_err("the disconnected child reaper is reported"); + assert_eq!(error.kind(), io::ErrorKind::BrokenPipe); + let deadline = Instant::now() + Duration::from_secs(2); + while !failed_child_handoffs() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .is_empty() + && Instant::now() < deadline + { + thread::sleep(Duration::from_millis(10)); + } + assert!( + failed_child_handoffs() + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .is_empty(), + "the fallback child reaper must eventually collect a recovered handoff" + ); + } + + #[test] + #[cfg_attr(miri, ignore = "spawns process groups")] + fn real_group_execution_observes_completion_and_timeout() { + let reaper = test_reaper(); + let success = run_streamed_with_timeout(&invocation(&["rustc", "--version"]), Duration::from_secs(5), &reaper); + assert!(matches!(success, InvocationResult::Exited(status) if status.success())); + + let group = spawn_group(sleeping_test_command()).expect("spawn sleeping process group"); + let started = Instant::now(); + let error = terminate_group_bounded(group, &reaper).expect("killed process group is reaped"); + assert!(!error.success()); + assert!(started.elapsed() < Duration::from_secs(2)); + + let child = sleeping_test_command().spawn().expect("spawn sleeping direct child"); + let started = Instant::now(); + let status = terminate_child_bounded(child, &reaper).expect("killed direct child is reaped"); + assert!(!status.success()); + assert!(started.elapsed() < Duration::from_secs(2)); + + let mut quick = Command::new("rustc"); + let _ = quick.arg("--version").stdout(Stdio::null()).stderr(Stdio::null()); + let mut group = spawn_group(quick).expect("spawn quick process group"); + group.inner().wait().expect("quick leader exits"); + reaper.handoff_group(group).expect("completed group reaches the local reaper"); + drop(reaper); + thread::sleep(Duration::from_millis(100)); + } + + #[test] + #[cfg_attr(miri, ignore = "spawns and captures process groups")] + fn captured_runner_uses_group_control_and_keeps_output() { + let reaper = test_reaper(); + let mut untimed = run_captured(&invocation(&["rustc", "--version"]), None, &reaper); + assert!(matches!(untimed.result, InvocationResult::Exited(status) if status.success())); + assert!( + String::from_utf8(output_bytes(&mut untimed.stdout)) + .expect("rustc output is UTF-8") + .contains("rustc") + ); + + let timed = run_captured(&invocation(&["rustc", "--version"]), Some(Duration::from_secs(5)), &reaper); + assert!(matches!(timed.result, InvocationResult::Exited(status) if status.success())); + } + + #[test] + #[cfg_attr(miri, ignore = "spawns subprocesses")] + fn direct_runners_report_empty_and_unspawnable_commands() { + let reaper = test_reaper(); + let empty = invocation(&[]); + assert!( + matches!(run_streamed(&empty, &reaper), InvocationResult::Infrastructure(message) if message.contains("empty argument vector")) + ); + assert!(infrastructure_message(run_captured(&empty, None, &reaper)).contains("empty argument vector")); + assert!(matches!( + run_streamed_with_timeout(&empty, Duration::from_secs(1), &reaper), + InvocationResult::Infrastructure(message) if message.contains("empty argument vector") + )); + + let missing = invocation(&["__cargo_each_missing_program_for_unit_test__"]); + assert!( + matches!(run_streamed(&missing, &reaper), InvocationResult::Infrastructure(message) if message.contains("failed to spawn")) + ); + assert!(infrastructure_message(run_captured(&missing, None, &reaper)).contains("failed to spawn")); + + let injected = run_streamed_with_timeout_with(&invocation(&["rustc", "--version"]), Duration::from_secs(1), &reaper, |_| { + Err("injected group spawn failure".to_owned()) + }); + assert!(matches!( + injected, + InvocationResult::Infrastructure(message) if message.contains("injected group spawn failure") + )); + } + + #[test] + #[cfg_attr(miri, ignore = "spawns a child process")] + fn streamed_child_observation_returns_a_completed_status() { + let reaper = test_reaper(); + let mut command = Command::new("rustc"); + let mut child = command + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn short-lived child"); + let expected = child.wait().expect("wait for short-lived child"); + let mut control = StreamedChild { child, reaper: &reaper }; + assert_eq!( + observe_streamed_child(&mut control).expect("observe completed child"), + Some(expected) + ); + } + + #[test] + fn capture_setup_failure_is_reported_before_process_spawn() { + let reaper = test_reaper(); + let invocation = invocation(&["rustc", "--version"]); + let spawn_calls = Arc::new(AtomicUsize::new(0)); + let calls = Arc::clone(&spawn_calls); + let outcome = run_captured_with( + &invocation, + None, + &reaper, + |_| Err(io::Error::other("injected capture setup failure")), + move |_| { + calls.fetch_add(1, Ordering::SeqCst); + Err("spawn must not be reached".to_owned()) + }, + ); + assert!(infrastructure_message(outcome).contains("injected capture setup failure")); + assert_eq!(spawn_calls.load(Ordering::SeqCst), 0); + + let stderr_failure = run_captured_with( + &invocation, + None, + &reaper, + |stream| { + if stream == "stdout" { + Ok((Box::new(io::Cursor::new(Vec::new())), Stdio::null())) + } else { + Err(io::Error::other("injected stderr capture failure")) + } + }, + |_| Err("spawn must not be reached".to_owned()), + ); + assert!(infrastructure_message(stderr_failure).contains("injected stderr capture failure")); + + let spawn_failure = run_captured_with( + &invocation, + None, + &reaper, + |_| Ok((Box::new(io::Cursor::new(Vec::new())), Stdio::null())), + |_| Err("injected captured spawn failure".to_owned()), + ); + assert!(infrastructure_message(spawn_failure).contains("injected captured spawn failure")); + } + + #[test] + fn capture_finalization_and_emission_use_a_finite_snapshot() { + let source = FaultySnapshot { + cursor: io::Cursor::new(b"snapshot-later".to_vec()), + reported_length: 8, + fail_length: false, + fail_seek: false, + fail_read: false, + }; + let mut captured = finish_capture(Box::new(source), "stdout"); + assert!(captured.failure.is_none()); + assert_eq!(output_bytes(&mut captured.output), b"snapshot"); + + let failed_length = finish_capture( + Box::new(FaultySnapshot { + cursor: io::Cursor::new(Vec::new()), + reported_length: 0, + fail_length: true, + fail_seek: false, + fail_read: false, + }), + "stdout", + ); + assert!( + failed_length + .failure + .is_some_and(|failure| failure.contains("injected snapshot length failure")) + ); + + let failed_seek = finish_capture( + Box::new(FaultySnapshot { + cursor: io::Cursor::new(Vec::new()), + reported_length: 0, + fail_length: false, + fail_seek: true, + fail_read: false, + }), + "stderr", + ); + assert!( + failed_seek + .failure + .is_some_and(|failure| failure.contains("injected snapshot seek failure")) + ); + } + + #[test] + #[cfg_attr(miri, ignore = "uses filesystem-backed temporary files; Miri isolation forbids them")] + fn temporary_snapshot_seek_rewinds_the_independent_reader() { + let temporary = tempfile::NamedTempFile::new().expect("create named temporary capture"); + std::fs::write(temporary.path(), b"snapshot").expect("write temporary capture"); + let reader = temporary.reopen().expect("reopen temporary capture reader"); + let path = temporary.into_temp_path(); + let mut snapshot = TemporarySnapshot { _path: path, reader }; + let mut first = [0_u8; 1]; + snapshot.read_exact(&mut first).expect("read first snapshot byte"); + assert_eq!(first, [b's']); + + snapshot.seek(io::SeekFrom::Start(0)).expect("rewind snapshot reader"); + let mut output = Vec::new(); + snapshot.read_to_end(&mut output).expect("read rewound snapshot"); + assert_eq!(output, b"snapshot"); + } + + #[test] + fn capture_source_failures_become_infrastructure_failures() { + let mut outcome = BufferedOutcome { + stdout: CapturedOutput::Snapshot { + source: Box::new(FaultySnapshot { + cursor: io::Cursor::new(Vec::new()), + reported_length: 1, + fail_length: false, + fail_seek: false, + fail_read: true, + }), + length: 1, + }, + stderr: CapturedOutput::empty(), + result: InvocationResult::Exited(successful_status()), + }; + emit_buffered_to(&invocation(&["probe"]), &mut outcome, &mut Vec::new(), &mut Vec::new()).expect("destinations remain writable"); + assert!(infrastructure_message(outcome).contains("injected snapshot read failure")); + + let mut stderr_outcome = BufferedOutcome { + stdout: CapturedOutput::empty(), + stderr: CapturedOutput::Snapshot { + source: Box::new(FaultySnapshot { + cursor: io::Cursor::new(Vec::new()), + reported_length: 1, + fail_length: false, + fail_seek: false, + fail_read: true, + }), + length: 1, + }, + result: InvocationResult::Exited(successful_status()), + }; + emit_buffered_to(&invocation(&["probe"]), &mut stderr_outcome, &mut Vec::new(), &mut Vec::new()) + .expect("destinations remain writable"); + assert!(infrastructure_message(stderr_outcome).contains("injected snapshot read failure")); + + let mut short = CapturedOutput::Snapshot { + source: Box::new(io::Cursor::new(Vec::new())), + length: 1, + }; + let error = short.emit_to(&mut Vec::new()).expect_err("short snapshots are reported"); + assert!(matches!(error, OutputEmitError::Source(error) if error.kind() == io::ErrorKind::UnexpectedEof)); + } + + #[test] + fn capture_and_emission_preserve_all_failure_context() { + for (stdout, stderr, expected) in [ + (captured(b"out", Some("stdout failed")), captured(b"err", None), "stdout failed"), + (captured(b"out", None), captured(b"err", Some("stderr failed")), "stderr failed"), + ( + captured(b"out", Some("stdout failed")), + captured(b"err", Some("stderr failed")), + "stdout failed; stderr failed", + ), + ] { + let outcome = combine_captured_output(stdout, stderr, InvocationResult::Exited(successful_status())); + assert_eq!(infrastructure_message(outcome), expected); + } + + let mut outcome = BufferedOutcome { + stdout: captured(b"stdout", None).output, + stderr: CapturedOutput::empty(), + result: InvocationResult::Exited(successful_status()), + }; + let error = emit_buffered_to(&invocation(&["probe"]), &mut outcome, &mut FailingWriter, &mut Vec::new()) + .expect_err("destination failure propagates"); + assert!(error.to_string().contains("injected destination write failure")); + + let mut stderr_failure = BufferedOutcome { + stdout: CapturedOutput::empty(), + stderr: captured(b"stderr", None).output, + result: InvocationResult::Exited(successful_status()), + }; + let error = emit_buffered_to(&invocation(&["probe"]), &mut stderr_failure, &mut Vec::new(), &mut FailingWriter) + .expect_err("stderr destination failure propagates"); + assert!(error.to_string().contains("injected destination write failure")); + + for result in [ + InvocationResult::TimedOut(Duration::from_millis(10)), + InvocationResult::Infrastructure("infrastructure".to_owned()), + ] { + let mut diagnostic = BufferedOutcome { + stdout: CapturedOutput::empty(), + stderr: CapturedOutput::empty(), + result, + }; + let mut stderr = Vec::new(); + emit_buffered_to(&invocation(&["probe"]), &mut diagnostic, &mut Vec::new(), &mut stderr).expect("memory output succeeds"); + assert!(!stderr.is_empty()); + } + } + + #[test] + fn infrastructure_failure_merging_preserves_primary_context() { + assert!(matches!( + add_infrastructure_failure(InvocationResult::Exited(successful_status()), String::new()), + InvocationResult::Exited(status) if status.success() + )); + assert_eq!( + result_infrastructure_message(add_infrastructure_failure( + InvocationResult::TimedOut(Duration::from_millis(10)), + "drain failed".to_owned(), + )), + "invocation timed out after 10ms; drain failed" + ); + assert_eq!( + result_infrastructure_message(add_infrastructure_failure( + InvocationResult::Infrastructure("wait failed".to_owned()), + "drain failed".to_owned(), + )), + "wait failed; drain failed" + ); + } + + #[test] + fn worker_panics_and_disconnects_become_infrastructure_outcomes() { + let reaper = test_reaper(); + let plan = Plan { + invocations: vec![Invocation { + label: Some("panic-probe".to_owned()), + argv: vec![WORKER_PANIC_TEST_PROGRAM.to_owned()], + work_dir: None, + }], + }; + let code = execute_parallel(&plan, false, NonZeroUsize::new(2).expect("literal two is nonzero"), None, &reaper) + .expect("worker panic is represented as an outcome"); + assert_eq!(code, ExitCode::from(2)); + + let (sender, receiver) = mpsc::channel::(); + drop(sender); + let outcome = wait_for_worker(&mut vec![RunningWorker { + index: 4, + receiver, + thread: thread::spawn(|| {}), + }]) + .expect("disconnection is observable"); + assert_eq!(outcome.index, 4); + assert!(result_infrastructure_message(outcome.outcome.result).contains("without reporting")); + + let (sender, receiver) = mpsc::channel(); + let outcome = wait_for_worker(&mut vec![RunningWorker { + index: 5, + receiver, + thread: thread::spawn(move || { + sender + .send(BufferedOutcome::infrastructure("premature".to_owned())) + .expect("the receiver remains alive"); + panic!("panic after report"); + }), + }]) + .expect("reported panic is observable"); + assert!(result_infrastructure_message(outcome.outcome.result).contains("panic after report")); + + let (sender, receiver) = mpsc::channel::(); + drop(sender); + let outcome = wait_for_worker(&mut vec![RunningWorker { + index: 6, + receiver, + thread: thread::spawn(|| panic!("panic before report")), + }]) + .expect("unreported panic is observable"); + assert!(result_infrastructure_message(outcome.outcome.result).contains("panic before report")); + } + + #[test] + #[cfg_attr(miri, ignore = "spawns a rustc subprocess")] + fn worker_spawn_errors_are_local_and_scheduler_visible() { + let reaper = test_reaper(); + let error = spawn_worker(0, invocation(&[WORKER_SPAWN_ERROR_TEST_PROGRAM]), None, reaper.clone()) + .expect_err("the local seam rejects only its sentinel"); + assert!(error.to_string().contains("injected worker spawn failure")); + + let worker = spawn_worker(1, invocation(&["rustc", "--version"]), None, reaper).expect("ordinary program launches"); + let outcome = wait_for_worker(&mut vec![worker]).expect("ordinary worker reports"); + assert_eq!(outcome.index, 1); + assert!(!outcome.outcome.result.failed()); + } + + #[test] + fn helper_diagnostics_preserve_cleanup_and_reaper_context() { + assert_eq!(with_cleanup_failure("primary".to_owned(), &Ok::<_, io::Error>(())), "primary"); + assert_eq!( + with_cleanup_failure("primary".to_owned(), &Err::<(), _>(io::Error::other("cleanup"))), + "primary; process-group cleanup also failed: cleanup" + ); + assert!(with_reaper_handoff("deadline", &Ok(())).contains("local polling reaper")); + assert!(with_reaper_handoff("deadline", &Err(io::Error::other("thread unavailable"))).contains("thread unavailable")); + assert_eq!(panic_description(&"borrowed panic"), "borrowed panic"); + assert_eq!(panic_description(&"owned panic".to_owned()), "owned panic"); + assert_eq!(panic_description(&7_u8), "non-string panic payload"); + } + + #[test] + fn tree_outcome_retains_the_invocation_result() { + let outcome = TreeOutcome::new(InvocationResult::Infrastructure("outcome".to_owned())); + assert_eq!(result_infrastructure_message(outcome.result), "outcome"); } } diff --git a/crates/cargo-each/src/select.rs b/crates/cargo-each/src/select.rs index d882e6f87..d427f3fc3 100644 --- a/crates/cargo-each/src/select.rs +++ b/crates/cargo-each/src/select.rs @@ -6,26 +6,22 @@ //! //! Mirrors `cargo build`'s selection surface: `-p`/`--package` (with glob //! support and optional `@version` qualifier), `--workspace`/`--all`, and -//! `--exclude`, plus the `cargo-each`-specific `--none` (explicit empty set). -//! When nothing is named the default is cargo's `default-members`, exactly -//! like `cargo build`. -//! -//! A computed selection (e.g. an impact tier) is fed in as ordinary flags via -//! shell expansion by the caller; this module has no notion of files or -//! environment variables. +//! `--exclude`, plus a repeatable `--package-file` and the +//! `cargo-each`-specific `--none` (explicit empty set). When nothing is named +//! the default is cargo's `default-members`, exactly like `cargo build`. use std::collections::HashSet; +use std::fs; +use std::path::{Path, PathBuf}; use cargo_metadata::semver::Version; -use crate::error::{EachError, UnknownSelectorError}; +use crate::error::{EachError, InvalidPackageFileLineError, PackageFileReadError, PackageFileUtf8Error, UnknownSelectorError}; use crate::workspace::{Member, Workspace}; /// A parsed package selection, before it is resolved against a workspace. /// -/// Populated from command-line flags. A caller with a computed selection -/// (e.g. an impact tier) passes it as ordinary `-p` / `--workspace` / `--none` -/// flags via shell expansion. +/// Populated from command-line flags and package files. #[derive(Debug, Default, Clone)] pub(crate) struct Selection { /// `-p` / `--package` selectors (name, `name@version`, or glob). @@ -36,9 +32,42 @@ pub(crate) struct Selection { pub(crate) exclude: Vec, /// `--none`: explicitly resolve to the empty set. pub(crate) none: bool, + /// Whether at least one `--package-file` was present, even if every file + /// was empty. + pub(crate) package_file_supplied: bool, } impl Selection { + /// Build a selection from direct package specs and package files. + /// + /// Package files are always read and validated, even when `--none` or + /// `--workspace` will win selection precedence, so a broken declared input + /// never silently passes. + /// + /// # Errors + /// + /// Returns [`EachError`] when a package file cannot be read as UTF-8 or + /// contains a malformed nonempty line. + pub(crate) fn from_sources( + packages: &[String], + package_files: &[PathBuf], + all: bool, + exclude: &[String], + none: bool, + ) -> Result { + let mut combined = packages.to_vec(); + for path in package_files { + combined.extend(read_package_file(path)?); + } + Ok(Self { + packages: combined, + all, + exclude: exclude.to_vec(), + none, + package_file_supplied: !package_files.is_empty(), + }) + } + /// Whether the resolved set is the whole workspace with no narrowing. /// /// True when selected via `--workspace` / `--all` with no narrowing @@ -73,6 +102,8 @@ impl Selection { workspace.members.iter().collect() } else if !self.packages.is_empty() { resolve_selectors(workspace, &self.packages)? + } else if self.package_file_supplied { + Vec::new() } else { workspace .members @@ -93,6 +124,57 @@ impl Selection { } } +fn read_package_file(path: &Path) -> Result, EachError> { + let display = path.display().to_string(); + let bytes = fs::read(path).map_err(|error| PackageFileReadError::caused_by(display.clone(), error))?; + let contents = String::from_utf8(bytes).map_err(|error| PackageFileUtf8Error::caused_by(display.clone(), error))?; + contents + .strip_prefix('\u{feff}') + .unwrap_or(&contents) + .lines() + .enumerate() + .filter_map(|(index, line)| { + if line.is_empty() { + None + } else { + Some(validate_package_file_spec(&display, index + 1, line).map(str::to_owned)) + } + }) + .collect() +} + +fn validate_package_file_spec<'a>(path: &str, line: usize, spec: &'a str) -> Result<&'a str, EachError> { + let invalid = |reason: &str| InvalidPackageFileLineError::new(path.to_owned(), line, spec.to_owned(), reason.to_owned()).into(); + if spec.bytes().any(|byte| byte.is_ascii_whitespace()) { + return Err(invalid("leading, trailing, and embedded whitespace are not allowed")); + } + if spec.starts_with('#') { + return Err(invalid("comments are not supported")); + } + if spec.starts_with('-') { + return Err(invalid("command-line tokens are not package specs")); + } + let mut pieces = spec.split('@'); + let name = pieces.next().expect("split always yields at least one element"); + let version = pieces.next(); + if pieces.next().is_some() { + return Err(invalid("a package spec may contain at most one `@`")); + } + if name.is_empty() { + return Err(invalid("expected a package name or Unix glob, optionally followed by `@version`")); + } + if !name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'*' | b'?')) + { + return Err(invalid("expected a package name or Unix glob, optionally followed by `@version`")); + } + if version.is_some_and(str::is_empty) { + return Err(invalid("the version qualifier after `@` must not be empty")); + } + Ok(spec) +} + /// Resolve a list of selectors against the workspace, deduplicating and /// preserving the workspace's member order. Each selector must match at /// least one member. @@ -250,7 +332,6 @@ fn glob_matches(pattern: &str, name: &str) -> bool { #[cfg_attr(coverage_nightly, coverage(off))] mod tests { use std::collections::BTreeSet; - use std::path::PathBuf; use serde_json::Value; @@ -260,6 +341,7 @@ mod tests { Member { name: name.to_owned(), version: "0.1.0".to_owned(), + rust_version: Some("1.70.0".parse().expect("valid Rust version")), manifest_path: PathBuf::from(format!("/ws/{name}/Cargo.toml")), publishable: true, features: BTreeSet::new(), @@ -273,6 +355,7 @@ mod tests { Workspace { members: vec![member("alpha"), member("beta"), member("gamma")], default_member_names: defaults.iter().map(|s| (*s).to_owned()).collect(), + root_manifest_path: PathBuf::from("/ws/Cargo.toml"), } } @@ -307,6 +390,36 @@ mod tests { assert_eq!(names(&sel.resolve(&ws).expect("resolve")), ["alpha", "gamma"]); } + #[test] + fn an_empty_package_file_source_selects_nothing() { + let ws = workspace(&["alpha", "gamma"]); + let sel = Selection { + package_file_supplied: true, + ..Selection::default() + }; + assert!(sel.resolve(&ws).expect("resolve").is_empty()); + } + + #[test] + fn package_file_lines_reject_comments_tokens_and_whitespace() { + for spec in [ + "# alpha", + "--workspace", + " alpha", + "alpha ", + "alpha beta", + "@1", + "alpha!", + "alpha@", + "alpha@1@2", + ] { + validate_package_file_spec("packages.txt", 1, spec).expect_err(spec); + } + for spec in ["alpha", "alpha@1.2.3", "cargo-*", "?eta"] { + assert_eq!(validate_package_file_spec("packages.txt", 1, spec).expect(spec), spec); + } + } + #[test] fn package_glob_and_version_spec_match_on_name() { let ws = workspace(&["alpha", "beta", "gamma"]); diff --git a/crates/cargo-each/src/substitute.rs b/crates/cargo-each/src/substitute.rs index fe7d4e5aa..729d9efcf 100644 --- a/crates/cargo-each/src/substitute.rs +++ b/crates/cargo-each/src/substitute.rs @@ -13,6 +13,7 @@ //! - The once token (valid only in `--once` mode): `{packages}`. Must stand //! alone as a whole argument; it expands to the resolved selection flags, //! which is several tokens. +//! - The workspace token `{workspace-rust-version}`, valid in every mode. //! //! Using a token in the wrong mode is a usage error ([`PlaceholderMisuseError`]). //! @@ -23,7 +24,7 @@ //! of the contract (`cargo-each` never interprets the command beyond these //! fixed substitutions), not an oversight. -use crate::error::{EachError, PlaceholderMisuseError}; +use crate::error::{EachError, PlaceholderMisuseError, WorkspaceRustVersionError}; use crate::plan::Mode; /// Per-package placeholder tokens. @@ -32,6 +33,8 @@ const PER_PACKAGE_TOKENS: [&str; 4] = ["{name}", "{spec}", "{version}", "{manife const TARGET_TOKEN: &str = "{target}"; /// The once-mode placeholder token. const PACKAGES_TOKEN: &str = "{packages}"; +/// The workspace-wide Rust compatibility floor token. +const WORKSPACE_RUST_VERSION_TOKEN: &str = "{workspace-rust-version}"; /// The substitution context for one command invocation. #[derive(Debug, Clone)] @@ -46,6 +49,8 @@ pub(crate) enum Placeholders { version: String, /// `{manifest}` — absolute path to the member's `Cargo.toml`. manifest: String, + /// The root workspace Rust-version declaration, when requested. + workspace_rust_version: Option, }, /// Per-target mode: package facts plus the selected target name. Target { @@ -54,15 +59,66 @@ pub(crate) enum Placeholders { version: String, manifest: String, target: String, + workspace_rust_version: Option, }, /// Once mode: `{packages}` expands to these pre-computed selection flags. Once { /// The cargo selection flags for the resolved set (e.g. /// `["--workspace"]` or `["--package", "a@1", "--package", "b@2"]`). packages: Vec, + /// The root workspace Rust-version declaration, when requested. + workspace_rust_version: Option, }, } +impl Placeholders { + fn workspace_rust_version(&self) -> Option<&str> { + match self { + Self::Package { + workspace_rust_version, .. + } + | Self::Target { + workspace_rust_version, .. + } + | Self::Once { + workspace_rust_version, .. + } => workspace_rust_version.as_deref(), + } + } +} + +fn replace_arg<'a>(arg: &str, placeholders: &'a Placeholders, mut replacements: Vec<(&'static str, &'a str)>) -> Result { + if arg.contains(WORKSPACE_RUST_VERSION_TOKEN) { + let version = placeholders.workspace_rust_version().ok_or_else(|| { + WorkspaceRustVersionError::new("the command uses the placeholder but its root value was not resolved".to_owned()) + })?; + replacements.push((WORKSPACE_RUST_VERSION_TOKEN, version)); + } + + let mut rest = arg; + let mut replaced = String::with_capacity(arg.len()); + while let Some((offset, token, value)) = replacements + .iter() + .filter_map(|&(token, value)| rest.find(token).map(|offset| (offset, token, value))) + .min_by_key(|&(offset, _, _)| offset) + { + let (literal, token_and_rest) = rest.split_at(offset); + let (_token, remaining) = token_and_rest.split_at(token.len()); + replaced.push_str(literal); + replaced.push_str(value); + rest = remaining; + } + replaced.push_str(rest); + + Ok(replaced) +} + +/// Whether a command template uses the lazy workspace Rust-version token. +#[must_use] +pub(crate) fn uses_workspace_rust_version(args: &[String]) -> bool { + args.iter().any(|arg| arg.contains(WORKSPACE_RUST_VERSION_TOKEN)) +} + /// Validate that `args` only reference placeholders valid for the mode. /// /// Checks mode-consistency without expanding the tokens — the check factored @@ -133,6 +189,7 @@ pub(crate) fn substitute(args: &[String], placeholders: &Placeholders) -> Result spec, version, manifest, + .. } => { // The `{name}` / `{spec}` / … literals are cargo-each // placeholder tokens, not Rust format-string arguments. @@ -140,11 +197,11 @@ pub(crate) fn substitute(args: &[String], placeholders: &Placeholders) -> Result clippy::literal_string_with_formatting_args, reason = "cargo-each placeholder tokens, not format args" )] - let replaced = arg - .replace("{name}", name) - .replace("{spec}", spec) - .replace("{version}", version) - .replace("{manifest}", manifest); + let replaced = replace_arg( + arg, + placeholders, + vec![("{name}", name), ("{spec}", spec), ("{version}", version), ("{manifest}", manifest)], + )?; out.push(replaced); } Placeholders::Target { @@ -153,30 +210,37 @@ pub(crate) fn substitute(args: &[String], placeholders: &Placeholders) -> Result version, manifest, target, + .. } => { #[expect( clippy::literal_string_with_formatting_args, reason = "cargo-each placeholder tokens, not format args" )] - let replaced = arg - .replace("{name}", name) - .replace("{spec}", spec) - .replace("{version}", version) - .replace("{manifest}", manifest) - .replace(TARGET_TOKEN, target); + let replaced = replace_arg( + arg, + placeholders, + vec![ + ("{name}", name), + ("{spec}", spec), + ("{version}", version), + ("{manifest}", manifest), + (TARGET_TOKEN, target), + ], + )?; out.push(replaced); } - Placeholders::Once { packages } => { + Placeholders::Once { packages, .. } => { // Validation above guarantees each arg is either exactly // `{packages}` or contains no placeholder token at all. if arg == PACKAGES_TOKEN { out.extend(packages.iter().cloned()); } else { - out.push(arg.clone()); + out.push(replace_arg(arg, placeholders, Vec::new())?); } } } } + Ok(out) } @@ -191,6 +255,7 @@ mod tests { spec: "cargo-anvil@0.4.0".to_owned(), version: "0.4.0".to_owned(), manifest: "/ws/cargo-anvil/Cargo.toml".to_owned(), + workspace_rust_version: None, } } @@ -220,6 +285,7 @@ mod tests { fn once_expands_packages_token() { let ph = Placeholders::Once { packages: args(&["--package", "a@1", "--package", "b@2"]), + workspace_rust_version: None, }; let out = substitute(&args(&["clippy", "{packages}", "--all-targets"]), &ph).expect("substitute"); assert_eq!(out, ["clippy", "--package", "a@1", "--package", "b@2", "--all-targets"]); @@ -229,6 +295,7 @@ mod tests { fn once_rejects_per_package_token() { let ph = Placeholders::Once { packages: args(&["--workspace"]), + workspace_rust_version: None, }; let err = substitute(&args(&["test", "--package", "{name}"]), &ph).expect_err("misuse"); assert!(err.to_string().contains("{name}")); @@ -238,6 +305,7 @@ mod tests { fn once_rejects_target_token() { let ph = Placeholders::Once { packages: args(&["--workspace"]), + workspace_rust_version: None, }; let err = substitute(&args(&["test", "--test", "{target}"]), &ph).expect_err("misuse"); assert!(err.to_string().contains("{target}")); @@ -247,6 +315,7 @@ mod tests { fn once_rejects_embedded_packages_token() { let ph = Placeholders::Once { packages: args(&["--workspace"]), + workspace_rust_version: None, }; let err = substitute(&args(&["x={packages}"]), &ph).expect_err("misuse"); assert!(err.to_string().contains("stand alone")); @@ -260,6 +329,7 @@ mod tests { version: "0.4.0".to_owned(), manifest: "/ws/cargo-anvil/Cargo.toml".to_owned(), target: "loom".to_owned(), + workspace_rust_version: None, }; let out = substitute(&args(&["test", "-p", "{name}", "--test", "{target}"]), &ph).expect("substitute"); assert_eq!(out, ["test", "-p", "cargo-anvil", "--test", "loom"]); @@ -270,4 +340,102 @@ mod tests { let err = substitute(&args(&["echo", "{target}"]), &pkg()).expect_err("misuse"); assert!(err.to_string().contains("per-target")); } + + #[test] + fn workspace_rust_version_expands_in_every_mode() { + let command = args(&["rustup", "toolchain", "install", "{workspace-rust-version}"]); + let mut package = pkg(); + let Placeholders::Package { + workspace_rust_version, .. + } = &mut package + else { + unreachable!("pkg returns package placeholders"); + }; + *workspace_rust_version = Some("1.80".to_owned()); + assert_eq!( + substitute(&command, &package).expect("package substitution"), + ["rustup", "toolchain", "install", "1.80"] + ); + + let once = Placeholders::Once { + packages: args(&["--workspace"]), + workspace_rust_version: Some("1.80".to_owned()), + }; + assert_eq!( + substitute(&command, &once).expect("once substitution"), + ["rustup", "toolchain", "install", "1.80"] + ); + } + + #[test] + fn unresolved_workspace_rust_version_is_reported_in_package_and_target_modes() { + let command = args(&["echo", "{workspace-rust-version}"]); + let package_error = substitute(&command, &pkg()).expect_err("package value is unresolved"); + assert!(package_error.to_string().contains("root value was not resolved"), "{package_error}"); + + let target = Placeholders::Target { + name: "crate".to_owned(), + spec: "crate@1.0.0".to_owned(), + version: "1.0.0".to_owned(), + manifest: "/ws/crate/Cargo.toml".to_owned(), + target: "example".to_owned(), + workspace_rust_version: None, + }; + let target_error = substitute(&command, &target).expect_err("target value is unresolved"); + assert!(target_error.to_string().contains("root value was not resolved"), "{target_error}"); + } + + #[test] + fn package_values_are_not_rescanned_for_workspace_tokens() { + let placeholders = Placeholders::Package { + name: "crate".to_owned(), + spec: "crate@1.0.0".to_owned(), + version: "1.0.0".to_owned(), + manifest: "/ws/{workspace-rust-version}/crate/Cargo.toml".to_owned(), + workspace_rust_version: Some("1.80".to_owned()), + }; + assert_eq!( + substitute(&args(&["{workspace-rust-version}", "{manifest}"]), &placeholders).expect("substitute package placeholders"), + ["1.80", "/ws/{workspace-rust-version}/crate/Cargo.toml"] + ); + } + + #[test] + fn target_values_are_not_rescanned_for_workspace_tokens() { + let placeholders = Placeholders::Target { + name: "crate".to_owned(), + spec: "crate@1.0.0".to_owned(), + version: "1.0.0".to_owned(), + manifest: "/ws/{workspace-rust-version}/crate/Cargo.toml".to_owned(), + target: "example".to_owned(), + workspace_rust_version: Some("1.80".to_owned()), + }; + assert_eq!( + substitute(&args(&["{workspace-rust-version}", "{manifest}:{target}"]), &placeholders,) + .expect("substitute target placeholders"), + ["1.80", "/ws/{workspace-rust-version}/crate/Cargo.toml:example"] + ); + } + + #[test] + fn manifest_values_are_not_rescanned_for_target_tokens() { + let placeholders = Placeholders::Target { + name: "crate".to_owned(), + spec: "crate@1.0.0".to_owned(), + version: "1.0.0".to_owned(), + manifest: "/ws/{target}/crate/Cargo.toml".to_owned(), + target: "example".to_owned(), + workspace_rust_version: None, + }; + assert_eq!( + substitute(&args(&["{manifest}:{target}"]), &placeholders).expect("substitute target placeholders"), + ["/ws/{target}/crate/Cargo.toml:example"] + ); + } + + #[test] + fn detects_workspace_rust_version_usage() { + assert!(uses_workspace_rust_version(&args(&["tool", "v={workspace-rust-version}"]))); + assert!(!uses_workspace_rust_version(&args(&["tool", "{name}"]))); + } } diff --git a/crates/cargo-each/src/workspace.rs b/crates/cargo-each/src/workspace.rs index 8091d9f58..01009cfed 100644 --- a/crates/cargo-each/src/workspace.rs +++ b/crates/cargo-each/src/workspace.rs @@ -11,10 +11,11 @@ use std::collections::{BTreeSet, HashSet}; use std::path::{Path, PathBuf}; +use cargo_metadata::semver::Version; use cargo_metadata::{MetadataCommand, TargetKind}; use serde_json::Value; -use crate::error::{EachError, LoadMetadataError}; +use crate::error::{EachError, LoadMetadataError, WorkspaceManifestParseError, WorkspaceManifestReadError, WorkspaceRustVersionError}; /// A resolved view of the cargo workspace `cargo-each` is operating on. #[derive(Debug, Clone)] @@ -25,6 +26,8 @@ pub(crate) struct Workspace { /// or every member when unset). Used to resolve a selection that names /// no packages. pub(crate) default_member_names: HashSet, + /// Absolute path to the workspace root manifest. + pub(crate) root_manifest_path: PathBuf, } /// A single workspace member and the facts selection/filtering key on. @@ -34,6 +37,8 @@ pub(crate) struct Member { pub(crate) name: String, /// Package version, rendered (e.g. `0.3.0`). pub(crate) version: String, + /// The member's resolved minimum supported Rust version. + pub(crate) rust_version: Option, /// Absolute path to this member's `Cargo.toml`. pub(crate) manifest_path: PathBuf, /// Whether Cargo permits publishing this package. @@ -123,6 +128,7 @@ impl Workspace { Member { name: pkg.name.to_string(), version: pkg.version.to_string(), + rust_version: pkg.rust_version.clone(), manifest_path: pkg.manifest_path.clone().into_std_path_buf(), publishable: pkg.publish.as_ref().is_none_or(|registries| !registries.is_empty()), features: pkg.features.keys().cloned().collect(), @@ -139,12 +145,127 @@ impl Workspace { .iter() .map(|pkg| pkg.name.to_string()) .collect(); + let root_manifest_path = metadata.workspace_root.join("Cargo.toml").into_std_path_buf(); Ok(Self { members, default_member_names, + root_manifest_path, }) } + + /// Resolve and validate the workspace-wide Rust compatibility floor. + /// + /// This deliberately reads the root manifest only when the corresponding + /// placeholder is used. Ordinary selection and execution therefore do not + /// require a workspace Rust-version declaration. + /// + /// # Errors + /// + /// Returns [`EachError`] if the root declaration is absent or invalid, or + /// if any workspace member omits `rust-version` or requires a newer + /// compiler than the root floor. + pub(crate) fn workspace_rust_version(&self) -> Result { + let path = self.root_manifest_path.display().to_string(); + let text = std::fs::read_to_string(&self.root_manifest_path) + .map_err(|error| WorkspaceManifestReadError::caused_by(path.clone(), error))?; + let manifest: toml::Value = toml::from_str(&text).map_err(|error| WorkspaceManifestParseError::caused_by(path, error))?; + + let workspace_floor = manifest + .get("workspace") + .and_then(|workspace| workspace.get("package")) + .and_then(|package| package.get("rust-version")); + let package_floor = match self.members.as_slice() { + [member] if member.manifest_path == self.root_manifest_path => { + manifest.get("package").and_then(|package| package.get("rust-version")) + } + _ => None, + }; + let floor = workspace_floor.or(package_floor).ok_or_else(|| { + WorkspaceRustVersionError::new( + "the root manifest must declare `[workspace.package].rust-version`, or `[package].rust-version` for a single-package repository" + .to_owned(), + ) + })?; + let Some(floor) = floor.as_str() else { + return Err(WorkspaceRustVersionError::new("the root Rust version must be a string".to_owned()).into()); + }; + let parsed_floor = parse_rust_version(floor) + .map_err(|reason| WorkspaceRustVersionError::new(format!("root Rust version `{floor}` is invalid: {reason}")))?; + + for member in &self.members { + let Some(member_floor) = member.rust_version.as_ref() else { + return Err(WorkspaceRustVersionError::new(format!( + "workspace member `{}` does not expose a resolved `rust-version`", + member.name + )) + .into()); + }; + if member_floor.major != 1 { + return Err(invalid_member_rust_version(member, member_floor)); + } + if !member_floor.pre.is_empty() { + return Err(invalid_member_rust_version(member, member_floor)); + } + if !member_floor.build.is_empty() { + return Err(invalid_member_rust_version(member, member_floor)); + } + if member_floor > &parsed_floor { + return Err(WorkspaceRustVersionError::new(format!( + "workspace member `{}` requires Rust {}, newer than the root floor {floor}", + member.name, member_floor + )) + .into()); + } + } + + Ok(floor.to_owned()) + } +} + +fn invalid_member_rust_version(member: &Member, version: &Version) -> EachError { + WorkspaceRustVersionError::new(format!( + "workspace member `{}` exposes invalid Rust version `{version}`; expected a Rust 1.x toolchain version", + member.name + )) + .into() +} + +fn parse_rust_version(value: &str) -> Result { + if value.contains('-') { + return Err("pre-release and build metadata are not valid Rust toolchain versions".to_owned()); + } + if value.contains('+') { + return Err("pre-release and build metadata are not valid Rust toolchain versions".to_owned()); + } + let components: Vec<&str> = value.split('.').collect(); + if !(2..=3).contains(&components.len()) { + return Err("expected `major.minor` or `major.minor.patch`".to_owned()); + } + if components + .iter() + .any(|component| component.is_empty() || !component.bytes().all(|byte| byte.is_ascii_digit())) + { + return Err("expected `major.minor` or `major.minor.patch`".to_owned()); + } + if components + .iter() + .any(|component| component.strip_prefix('0').is_some_and(|remainder| !remainder.is_empty())) + { + return Err("numeric components must not contain leading zeroes".to_owned()); + } + let normalized = if components.len() == 2 { + format!("{value}.0") + } else { + value.to_owned() + }; + let parsed: Version = normalized + .parse() + .map_err(|error| format!("expected `major.minor` or `major.minor.patch`: {error}"))?; + if parsed.major != 1 { + return Err("expected a Rust 1.x toolchain version".to_owned()); + } + Ok(parsed) } /// Parse a supported Cargo target-kind spelling. @@ -159,8 +280,32 @@ pub(crate) fn parse_target_kind(kind: &str) -> Option { #[cfg(test)] #[cfg_attr(coverage_nightly, coverage(off))] mod tests { + use std::fs; + use super::*; + fn member(name: &str, manifest_path: PathBuf, rust_version: &str) -> Member { + Member { + name: name.to_owned(), + version: "0.1.0".to_owned(), + rust_version: Some(rust_version.parse().expect("the test Rust version is semver")), + manifest_path, + publishable: true, + features: BTreeSet::new(), + targets: Vec::new(), + dependencies: BTreeSet::new(), + metadata: Value::Null, + } + } + + fn workspace(root_manifest_path: PathBuf, members: Vec) -> Workspace { + Workspace { + members, + default_member_names: HashSet::new(), + root_manifest_path, + } + } + #[test] fn parses_every_supported_target_kind() { for kind in [ @@ -184,4 +329,90 @@ mod tests { fn rejects_unknown_target_kind() { assert_eq!(parse_target_kind("future-kind"), None); } + + #[test] + fn parses_cargo_rust_version_forms() { + assert_eq!(parse_rust_version("1.80").expect("minor form"), Version::new(1, 80, 0)); + assert_eq!(parse_rust_version("1.80.1").expect("patch form"), Version::new(1, 80, 1)); + assert_eq!( + parse_rust_version("1.80.0-beta"), + Err("pre-release and build metadata are not valid Rust toolchain versions".to_owned()) + ); + assert_eq!( + parse_rust_version("1.80+build"), + Err("pre-release and build metadata are not valid Rust toolchain versions".to_owned()) + ); + assert_eq!( + parse_rust_version("1"), + Err("expected `major.minor` or `major.minor.patch`".to_owned()) + ); + assert_eq!( + parse_rust_version("1.2.3.4"), + Err("expected `major.minor` or `major.minor.patch`".to_owned()) + ); + assert_eq!( + parse_rust_version("one.80"), + Err("expected `major.minor` or `major.minor.patch`".to_owned()) + ); + assert_eq!( + parse_rust_version("1.080"), + Err("numeric components must not contain leading zeroes".to_owned()) + ); + assert_eq!(parse_rust_version("2.0"), Err("expected a Rust 1.x toolchain version".to_owned())); + } + + #[test] + #[cfg_attr(miri, ignore = "uses temporary filesystem manifests")] + fn root_package_floor_requires_the_root_to_be_the_only_member() { + let temp = tempfile::tempdir().expect("create temporary workspace"); + let root = temp.path().join("Cargo.toml"); + fs::write(&root, "[package]\nname = \"root\"\nversion = \"0.1.0\"\nrust-version = \"1.80\"\n") + .expect("write root package manifest"); + let nested = member("nested", temp.path().join("nested/Cargo.toml"), "1.70.0"); + let error = workspace(root, vec![nested]) + .workspace_rust_version() + .expect_err("a nested sole member cannot use the root package floor"); + assert!(error.to_string().contains("[workspace.package].rust-version")); + } + + #[test] + #[cfg_attr(miri, ignore = "uses temporary filesystem manifests")] + fn invalid_resolved_member_versions_are_configuration_errors() { + let temp = tempfile::tempdir().expect("create temporary workspace"); + let root = temp.path().join("Cargo.toml"); + fs::write(&root, "[workspace]\n[workspace.package]\nrust-version = \"1.80\"\n").expect("write workspace manifest"); + + for version in ["2.0.0", "1.70.0-beta", "1.70.0+build"] { + let invalid = member("invalid", temp.path().join("invalid/Cargo.toml"), version); + let error = workspace(root.clone(), vec![invalid]) + .workspace_rust_version() + .expect_err("a non-Rust member version must fail"); + assert!(error.to_string().contains("invalid Rust version"), "{version}: {error}"); + } + } + + #[test] + #[cfg_attr(miri, ignore = "uses temporary filesystem manifests")] + fn workspace_rust_version_reports_manifest_io_and_shape_errors() { + let temp = tempfile::tempdir().expect("create temporary workspace"); + let missing = temp.path().join("missing.toml"); + let error = workspace(missing, Vec::new()) + .workspace_rust_version() + .expect_err("a missing root manifest must fail"); + assert!(error.to_string().contains("could not read workspace manifest")); + + let malformed = temp.path().join("malformed.toml"); + fs::write(&malformed, "[workspace").expect("write malformed root manifest"); + let error = workspace(malformed, Vec::new()) + .workspace_rust_version() + .expect_err("a malformed root manifest must fail"); + assert!(error.to_string().contains("could not parse workspace manifest")); + + let non_string = temp.path().join("non-string.toml"); + fs::write(&non_string, "[workspace]\n[workspace.package]\nrust-version = 180\n").expect("write non-string root floor"); + let error = workspace(non_string, Vec::new()) + .workspace_rust_version() + .expect_err("a non-string root floor must fail"); + assert!(error.to_string().contains("must be a string")); + } } diff --git a/crates/cargo-each/tests/cli.rs b/crates/cargo-each/tests/cli.rs index 2d4af518e..966dc1565 100644 --- a/crates/cargo-each/tests/cli.rs +++ b/crates/cargo-each/tests/cli.rs @@ -111,6 +111,228 @@ fn each(manifest: &Path) -> Command { cmd } +fn rust_version_fixture(root_floor: Option<&str>, members: &[(&str, Option<&str>)]) -> (TempDir, PathBuf) { + let tmp = tempfile::tempdir().expect("tempdir"); + let root = tmp.path(); + let member_names = members.iter().map(|(name, _)| format!("\"{name}\"")).collect::>().join(", "); + let workspace_package = root_floor.map_or_else(String::new, |floor| format!("\n[workspace.package]\nrust-version = \"{floor}\"\n")); + fs::write( + root.join("Cargo.toml"), + format!("[workspace]\nresolver = \"2\"\nmembers = [{member_names}]\n{workspace_package}"), + ) + .expect("write workspace root"); + for (name, rust_version) in members { + let declaration = match rust_version { + Some("workspace") => "rust-version.workspace = true\n".to_owned(), + Some(version) => format!("rust-version = \"{version}\"\n"), + None => String::new(), + }; + write_lib(root, name, "0.1.0", &declaration); + } + let manifest = root.join("Cargo.toml"); + (tmp, manifest) +} + +fn single_package_fixture(rust_version: &str) -> (TempDir, PathBuf) { + let tmp = tempfile::tempdir().expect("tempdir"); + let root = tmp.path(); + fs::create_dir_all(root.join("src")).expect("mkdir src"); + fs::write( + root.join("Cargo.toml"), + format!("[package]\nname = \"single\"\nversion = \"0.1.0\"\nedition = \"2021\"\nrust-version = \"{rust_version}\"\n"), + ) + .expect("write package manifest"); + fs::write(root.join("src/lib.rs"), "// fixture\n").expect("write lib"); + let manifest = root.join("Cargo.toml"); + (tmp, manifest) +} + +#[expect( + clippy::too_many_lines, + reason = "the embedded standalone probe stays together so rustc compiles one auditable cross-platform fixture" +)] +fn compile_execution_probe(directory: &Path) -> PathBuf { + let source = directory.join("execution-probe.rs"); + let executable = directory.join(format!("execution-probe{}", std::env::consts::EXE_SUFFIX)); + fs::write( + &source, + r#" +use std::env; +use std::fs::{self, OpenOptions}; +use std::io::{Read as _, Write as _}; +use std::process::{self, Command}; +use std::thread; +use std::time::{Duration, Instant}; + +fn append(path: &str, value: &str) { + let mut file = OpenOptions::new().create(true).append(true).open(path).expect("open log"); + writeln!(file, "{value}").expect("append log"); +} + +fn wait_for(path: &std::path::Path) { + let deadline = Instant::now() + Duration::from_secs(10); + while !path.exists() { + if Instant::now() >= deadline { + process::exit(90); + } + thread::sleep(Duration::from_millis(1)); + } +} + +fn main() { + let args: Vec = env::args().collect(); + match args[1].as_str() { + "ordered" => { + let name = &args[2]; + println!("{name}:start"); + thread::sleep(Duration::from_millis(if name == "alpha" { 250 } else { 20 })); + println!("{name}:end"); + append(&args[3], name); + } + "fail-order" => { + let name = &args[2]; + thread::sleep(Duration::from_millis(if name == "alpha" { 180 } else { 20 })); + process::exit(if name == "alpha" { 7 } else { 9 }); + } + "fail-stop" => { + let name = &args[2]; + let sync_dir = std::path::Path::new(&args[3]); + fs::write(sync_dir.join(format!("{name}.started")), "").expect("write start marker"); + match name.as_str() { + "alpha" => { + wait_for(&sync_dir.join("beta.started")); + process::exit(7); + } + "beta" => { + wait_for(&sync_dir.join("alpha.started")); + process::exit(9); + } + _ => process::exit(0), + } + } + "keep-going" => { + let name = &args[2]; + append(&args[3], name); + process::exit(if name == "alpha" { 7 } else { 0 }); + } + "large-output" => { + let name = &args[2]; + let size = 1_100_000; + let stdout_byte = if name == "alpha" { b'A' } else { b'B' }; + let stderr_byte = if name == "alpha" { b'C' } else { b'D' }; + let mut stdout = std::io::stdout().lock(); + writeln!(stdout, "{name}:stdout").expect("write stdout header"); + stdout.write_all(&vec![stdout_byte; size]).expect("write large stdout"); + let mut stderr = std::io::stderr().lock(); + writeln!(stderr, "{name}:stderr").expect("write stderr header"); + stderr.write_all(&vec![stderr_byte; size]).expect("write large stderr"); + process::exit(if name == "alpha" { 7 } else { 0 }); + } + "stdin" => { + let mut input = String::new(); + std::io::stdin().read_to_string(&mut input).expect("read stdin"); + println!("{}:stdin:{input}", args[2]); + } + "timeout-fail-fast" => { + if args[2] == "alpha" { + thread::sleep(Duration::from_secs(5)); + } else { + fs::write(&args[3], "later invocation ran").expect("write later marker"); + } + } + "timeout-keep-going" => { + if args[2] == "alpha" { + thread::sleep(Duration::from_secs(5)); + } else { + fs::write(&args[3], "later invocation ran").expect("write later marker"); + } + } + "tree-parent" => { + let marker = &args[2]; + Command::new(env::current_exe().expect("current exe")) + .arg("tree-child") + .arg(marker) + .spawn() + .expect("spawn tree child"); + thread::sleep(Duration::from_secs(5)); + } + "tree-child" => { + thread::sleep(Duration::from_millis(500)); + fs::write(&args[2], "survived").expect("write marker"); + } + "background-parent" => { + Command::new(env::current_exe().expect("current exe")) + .arg("background-child") + .arg(&args[2]) + .spawn() + .expect("spawn background child"); + } + "background-child" => { + thread::sleep(Duration::from_millis(100)); + fs::write(&args[2], "completed").expect("write background marker"); + } + "timed-background-parent" => { + Command::new(env::current_exe().expect("current exe")) + .arg("timed-background-child") + .arg(&args[2]) + .spawn() + .expect("spawn timed background child"); + } + "timed-background-child" => { + thread::sleep(Duration::from_millis(1200)); + fs::write(&args[2], "completed").expect("write timed background marker"); + } + "stubborn-background-parent" => { + Command::new(env::current_exe().expect("current exe")) + .arg("stubborn-background-child") + .arg(&args[2]) + .spawn() + .expect("spawn stubborn background child"); + } + "stubborn-background-child" => { + thread::sleep(Duration::from_secs(6)); + println!("late descendant output"); + fs::write(&args[2], "completed").expect("write stubborn background marker"); + } + "snapshot-offset-parent" => { + if args[2] == "alpha" { + let mut stdout = std::io::stdout().lock(); + stdout.write_all(&vec![b'A'; 1_100_000]).expect("write snapshot payload"); + stdout.flush().expect("flush snapshot payload"); + Command::new(env::current_exe().expect("current exe")) + .arg("snapshot-offset-child") + .arg(&args[4]) + .arg(&args[5]) + .spawn() + .expect("spawn snapshot offset child"); + fs::write(&args[3], "ready").expect("write snapshot-ready marker"); + } + } + "snapshot-offset-child" => { + wait_for(std::path::Path::new(&args[2])); + println!("late descendant output"); + fs::write(&args[3], "completed").expect("write snapshot child marker"); + } + other => panic!("unknown probe mode: {other}"), + } +} +"#, + ) + .expect("write execution probe"); + let output = std::process::Command::new("rustc") + .arg(&source) + .arg("-o") + .arg(&executable) + .output() + .expect("rustc must be available to compile the execution probe"); + assert!( + output.status.success(), + "failed to compile execution probe:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + executable +} + #[cfg(windows)] fn compile_probe(source: &Path, executable: &Path, marker: &str) { fs::write(source, format!("fn main() {{ println!(\"{marker}\"); }}\n")).expect("write probe source"); @@ -187,17 +409,191 @@ fn per_package_runs_once_per_selected_member() { .stdout(predicate::str::contains("echo alpha").and(predicate::str::contains("echo gamma"))); } +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn package_files_union_with_direct_packages_and_each_other() { + let (tmp, manifest) = fixture(); + let first = tmp.path().join("first.packages"); + let second = tmp.path().join("second.packages"); + fs::write(&first, "alpha\n\ngamma@0.1\n").expect("write first package file"); + fs::write(&second, "beta\n").expect("write second package file"); + + each(&manifest) + .arg("--package-file") + .arg(&first) + .arg("--package-file") + .arg(&second) + .args(["--package", "delta", "--dry-run", "--", "echo", "{name}"]) + .assert() + .success() + .stdout( + predicate::str::contains("echo alpha") + .and(predicate::str::contains("echo beta")) + .and(predicate::str::contains("echo delta")) + .and(predicate::str::contains("echo gamma")), + ); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn package_file_strips_one_leading_utf8_bom() { + let (tmp, manifest) = fixture(); + let packages = tmp.path().join("bom.packages"); + fs::write(&packages, "\u{feff}alpha\nbeta\n").expect("write BOM-prefixed package file"); + + each(&manifest) + .arg("--package-file") + .arg(packages) + .args(["--dry-run", "--", "echo", "{name}"]) + .assert() + .success() + .stdout(predicate::str::contains("echo alpha").and(predicate::str::contains("echo beta"))); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn present_empty_package_file_is_an_explicit_empty_selection() { + let (tmp, manifest) = default_members_fixture(); + let packages = tmp.path().join("empty.packages"); + fs::write(&packages, "").expect("write empty package file"); + + each(&manifest) + .arg("--package-file") + .arg(packages) + .args(["--dry-run", "--", "echo", "{name}:{workspace-rust-version}"]) + .assert() + .success() + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains("nothing to do")); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn package_file_precedence_matches_the_selection_contract() { + let (tmp, manifest) = fixture(); + let packages = tmp.path().join("alpha.packages"); + fs::write(&packages, "alpha\n").expect("write package file"); + + each(&manifest) + .arg("--package-file") + .arg(&packages) + .args(["--workspace", "--dry-run", "--", "echo", "{name}"]) + .assert() + .success() + .stdout( + predicate::str::contains("echo alpha") + .and(predicate::str::contains("echo beta")) + .and(predicate::str::contains("echo gamma")), + ); + + each(&manifest) + .arg("--package-file") + .arg(packages) + .args(["--none", "--dry-run", "--", "echo", "{name}"]) + .assert() + .success() + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains("nothing to do")); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn package_file_input_errors_fail_loudly() { + let (tmp, manifest) = fixture(); + let invalid_utf8 = tmp.path().join("invalid-utf8.packages"); + fs::write(&invalid_utf8, [0xFF, 0xFE]).expect("write invalid UTF-8"); + each(&manifest) + .arg("--package-file") + .arg(&invalid_utf8) + .args(["--dry-run", "--", "echo", "{name}"]) + .assert() + .failure() + .code(2) + .stderr(predicate::str::contains("not valid UTF-8")); + + let malformed = tmp.path().join("malformed.packages"); + fs::write(&malformed, "alpha\n--workspace\n").expect("write malformed package file"); + each(&manifest) + .arg("--package-file") + .arg(&malformed) + .args(["--dry-run", "--", "echo", "{name}"]) + .assert() + .failure() + .code(2) + .stderr(predicate::str::contains("line 2").and(predicate::str::contains("command-line tokens"))); + + let comment = tmp.path().join("comment.packages"); + fs::write(&comment, "#alpha\n").expect("write package file comment"); + each(&manifest) + .arg("--package-file") + .arg(&comment) + .args(["--dry-run", "--", "echo", "{name}"]) + .assert() + .failure() + .code(2) + .stderr(predicate::str::contains("comments are not supported")); + + let unmatched = tmp.path().join("unmatched.packages"); + fs::write(&unmatched, "does-not-exist\n").expect("write unmatched package file"); + each(&manifest) + .arg("--package-file") + .arg(&unmatched) + .args(["--dry-run", "--", "echo", "{name}"]) + .assert() + .failure() + .code(2) + .stderr(predicate::str::contains("did not match")); + + each(&manifest) + .arg("--package-file") + .arg(tmp.path().join("missing.packages")) + .args(["--dry-run", "--", "echo", "{name}"]) + .assert() + .failure() + .code(2) + .stderr(predicate::str::contains("could not read package file")); +} + #[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] #[test] fn none_is_a_successful_noop() { let (_tmp, manifest) = fixture(); each(&manifest) - .args(["--none", "--once", "--dry-run", "--", "cargo", "test", "{packages}"]) + .args([ + "--none", + "--once", + "--dry-run", + "--", + "cargo", + "test", + "{packages}", + "{workspace-rust-version}", + ]) .assert() .success() .stderr(predicate::str::contains("nothing to do")); } +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn empty_filtered_selection_skips_workspace_rust_version_resolution() { + let (_tmp, manifest) = fixture(); + each(&manifest) + .args([ + "--workspace", + "--filter", + "metadata:does-not-exist", + "--dry-run", + "--", + "echo", + "{name}:{workspace-rust-version}", + ]) + .assert() + .success() + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::contains("nothing to do")); +} + #[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] #[test] fn none_with_misused_placeholder_is_a_usage_error() { @@ -205,7 +601,7 @@ fn none_with_misused_placeholder_is_a_usage_error() { // usage error (exit 2), not a silent no-op. let (_tmp, manifest) = fixture(); each(&manifest) - .args(["--none", "--once", "--dry-run", "--", "echo", "{name}"]) + .args(["--none", "--once", "--dry-run", "--", "echo", "{name}", "{workspace-rust-version}"]) .assert() .failure() .code(2) @@ -865,3 +1261,512 @@ fn bare_invocation_runs_only_default_members() { .success() .stdout(predicate::str::contains("echo alpha").and(predicate::str::contains("echo beta").not())); } + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn workspace_rust_version_expands_in_all_modes_and_accepts_lower_members() { + let (_tmp, manifest) = rust_version_fixture(Some("1.80"), &[("alpha", Some("1.70")), ("beta", Some("workspace"))]); + each(&manifest) + .args(["--workspace", "--dry-run", "--", "echo", "{name}:{workspace-rust-version}"]) + .assert() + .success() + .stdout(predicate::str::contains("echo alpha:1.80").and(predicate::str::contains("echo beta:1.80"))); + + each(&manifest) + .args([ + "--workspace", + "--each-target", + "lib", + "--dry-run", + "--", + "echo", + "{target}:{workspace-rust-version}", + ]) + .assert() + .success() + .stdout(predicate::str::contains(":1.80")); + + each(&manifest) + .args([ + "--package", + "alpha", + "--once", + "--dry-run", + "--", + "echo", + "{workspace-rust-version}", + ]) + .assert() + .success() + .stdout(predicate::str::contains("echo 1.80")); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn workspace_rust_version_validation_is_lazy() { + let (_tmp, manifest) = rust_version_fixture(Some("1.80"), &[("alpha", Some("1.70")), ("beta", None)]); + each(&manifest) + .args(["--workspace", "--dry-run", "--", "echo", "{name}"]) + .assert() + .success(); + each(&manifest) + .args([ + "--package", + "alpha", + "--once", + "--dry-run", + "--", + "echo", + "{workspace-rust-version}", + ]) + .assert() + .failure() + .code(2) + .stderr(predicate::str::contains("beta").and(predicate::str::contains("rust-version"))); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn workspace_rust_version_rejects_newer_members() { + let (_tmp, manifest) = rust_version_fixture(Some("1.80"), &[("alpha", Some("1.81")), ("beta", Some("1.70"))]); + each(&manifest) + .args(["--workspace", "--once", "--dry-run", "--", "echo", "{workspace-rust-version}"]) + .assert() + .failure() + .code(2) + .stderr(predicate::str::contains("alpha").and(predicate::str::contains("newer than"))); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn workspace_rust_version_rejects_missing_or_invalid_root_floor() { + let (_missing, missing_manifest) = rust_version_fixture(None, &[("alpha", Some("1.70"))]); + each(&missing_manifest) + .args(["--workspace", "--once", "--dry-run", "--", "echo", "{workspace-rust-version}"]) + .assert() + .failure() + .code(2) + .stderr(predicate::str::contains("[workspace.package].rust-version")); + + let (_invalid, invalid_manifest) = rust_version_fixture(Some("2.0"), &[("alpha", Some("1.70"))]); + each(&invalid_manifest) + .args(["--workspace", "--dry-run", "--", "echo", "{name}"]) + .assert() + .success(); + each(&invalid_manifest) + .args(["--workspace", "--once", "--dry-run", "--", "echo", "{workspace-rust-version}"]) + .assert() + .failure() + .code(2) + .stderr(predicate::str::contains("2.0").and(predicate::str::contains("Rust 1.x"))); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn workspace_rust_version_uses_root_package_for_single_package_repository() { + let (_tmp, manifest) = single_package_fixture("1.75"); + each(&manifest) + .args(["--once", "--dry-run", "--", "echo", "{workspace-rust-version}"]) + .assert() + .success() + .stdout(predicate::str::contains("echo 1.75")); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn once_rejects_jobs_greater_than_one() { + let (_tmp, manifest) = fixture(); + each(&manifest) + .args(["--workspace", "--once", "--jobs", "2", "--dry-run", "--", "echo", "{packages}"]) + .assert() + .failure() + .code(2) + .stderr(predicate::str::contains("--jobs").and(predicate::str::contains("--once"))); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn jobs_help_documents_auto_and_default() { + Command::cargo_bin("cargo-each") + .expect("binary") + .args(["each", "--help"]) + .assert() + .success() + .stdout( + predicate::str::contains("--jobs ") + .and(predicate::str::contains("available parallelism")) + .and(predicate::str::contains("Defaults to 1")), + ); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn auto_jobs_is_accepted() { + let (_tmp, manifest) = fixture(); + each(&manifest) + .args(["--workspace", "--jobs", "auto", "--dry-run", "--", "echo", "{name}"]) + .assert() + .success(); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn default_jobs_runs_one_invocation_at_a_time() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + let completion_log = tmp.path().join("completion.log"); + each(&manifest) + .args(["-p", "alpha", "-p", "beta", "--"]) + .arg(probe) + .args(["ordered", "{name}"]) + .arg(&completion_log) + .assert() + .success(); + assert_eq!( + fs::read_to_string(completion_log).expect("completion log"), + "alpha\nbeta\n", + "the default must finish each invocation before launching the next" + ); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn requested_parallelism_with_one_invocation_preserves_inherited_stdin() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + each(&manifest) + .args(["-p", "alpha", "--jobs", "2", "--"]) + .arg(probe) + .args(["stdin", "{name}"]) + .write_stdin("inherited-input") + .assert() + .success() + .stdout(predicate::str::contains("alpha:stdin:inherited-input")); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn genuinely_parallel_children_receive_null_stdin() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + each(&manifest) + .args(["-p", "alpha", "-p", "beta", "--jobs", "2", "--"]) + .arg(probe) + .args(["stdin", "{name}"]) + .write_stdin("must-not-reach-children") + .assert() + .success() + .stdout( + predicate::str::contains("alpha:stdin:\n") + .and(predicate::str::contains("beta:stdin:\n")) + .and(predicate::str::contains("must-not-reach-children").not()), + ); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn parallel_output_is_buffered_in_plan_order() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + let completion_log = tmp.path().join("completion.log"); + let output = each(&manifest) + .args(["-p", "alpha", "-p", "beta", "--jobs", "2", "--"]) + .arg(&probe) + .args(["ordered", "{name}"]) + .arg(&completion_log) + .output() + .expect("run cargo-each"); + assert!(output.status.success(), "stderr: {}", String::from_utf8_lossy(&output.stderr)); + let stdout = String::from_utf8(output.stdout).expect("UTF-8 probe output"); + assert_eq!(stdout, "alpha:start\nalpha:end\nbeta:start\nbeta:end\n"); + assert_eq!( + fs::read_to_string(completion_log).expect("completion log"), + "beta\nalpha\n", + "the probe must finish out of order to prove cargo-each reordered complete blocks" + ); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn parallel_fail_fast_chooses_failure_by_plan_order() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + each(&manifest) + .args(["-p", "alpha", "-p", "beta", "--jobs", "2", "--"]) + .arg(probe) + .args(["fail-order", "{name}"]) + .assert() + .failure() + .code(7); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn parallel_fail_fast_stops_launching_new_work() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + let sync_dir = tmp.path().join("fail-stop"); + fs::create_dir(&sync_dir).expect("create synchronization directory"); + each(&manifest) + .args(["--workspace", "--jobs", "2", "--"]) + .arg(probe) + .args(["fail-stop", "{name}"]) + .arg(&sync_dir) + .assert() + .failure() + .code(7); + assert!(sync_dir.join("alpha.started").exists(), "the first initial worker must start"); + assert!(sync_dir.join("beta.started").exists(), "the second initial worker must start"); + for name in ["delta", "epsilon", "gamma"] { + assert!( + !sync_dir.join(format!("{name}.started")).exists(), + "{name} must not launch after either synchronized initial worker fails" + ); + } +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn parallel_keep_going_runs_the_complete_plan() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + let launch_log = tmp.path().join("launch.log"); + each(&manifest) + .args(["--workspace", "--jobs", "2", "--keep-going", "--"]) + .arg(probe) + .args(["keep-going", "{name}"]) + .arg(&launch_log) + .assert() + .failure() + .code(1); + let launched = fs::read_to_string(launch_log).expect("launch log"); + for name in ["alpha", "beta", "delta", "epsilon", "gamma"] { + assert!(launched.contains(name), "{name} must run under --keep-going:\n{launched}"); + } +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn parallel_tempfile_capture_preserves_large_stdout_and_stderr_in_plan_order() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + let output = each(&manifest) + .args(["-p", "alpha", "-p", "beta", "--jobs", "2", "--keep-going", "--"]) + .arg(probe) + .args(["large-output", "{name}"]) + .output() + .expect("run cargo-each with large output"); + + assert_eq!(output.status.code(), Some(1)); + let stdout = String::from_utf8(output.stdout).expect("probe stdout is ASCII"); + let stderr = String::from_utf8(output.stderr).expect("probe stderr is ASCII"); + assert!(stdout.find("alpha:stdout").expect("alpha stdout header") < stdout.find("beta:stdout").expect("beta stdout header")); + assert!(stderr.find("alpha:stderr").expect("alpha stderr header") < stderr.find("beta:stderr").expect("beta stderr header")); + assert_eq!(stdout.bytes().filter(|byte| *byte == b'A').count(), 1_100_000); + assert_eq!(stdout.bytes().filter(|byte| *byte == b'B').count(), 1_100_000); + assert_eq!(stderr.bytes().filter(|byte| *byte == b'C').count(), 1_100_000); + assert_eq!(stderr.bytes().filter(|byte| *byte == b'D').count(), 1_100_000); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn parallel_snapshot_reads_do_not_move_descendant_write_offsets() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + let ready = tmp.path().join("snapshot-ready"); + let release = tmp.path().join("snapshot-release"); + let completed = tmp.path().join("snapshot-child-completed"); + let mut command = std::process::Command::new(assert_cmd::cargo::cargo_bin!("cargo-each")); + let _ = command + .arg("each") + .arg("--manifest-path") + .arg(&manifest) + .args(["-p", "alpha", "-p", "beta", "--jobs", "2", "--"]) + .arg(probe) + .args(["snapshot-offset-parent", "{name}"]) + .arg(&ready) + .arg(&release) + .arg(&completed) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + let child = command.spawn().expect("spawn cargo-each snapshot-offset probe"); + + let ready_deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !ready.exists() && std::time::Instant::now() < ready_deadline { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert!(ready.exists(), "the snapshot leader did not become ready"); + std::thread::sleep(std::time::Duration::from_millis(100)); + fs::write(&release, "release").expect("release the background writer"); + + let output = child.wait_with_output().expect("collect cargo-each snapshot-offset output"); + assert!( + output.status.success(), + "snapshot-offset probe failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8(output.stdout).expect("snapshot-offset probe output is ASCII"); + assert_eq!(stdout.bytes().filter(|byte| *byte == b'A').count(), 1_100_000); + assert!( + !stdout.contains("late descendant output"), + "bytes appended after snapshot finalization leaked into the finite snapshot" + ); + + let completed_deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + while !completed.exists() && std::time::Instant::now() < completed_deadline { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert!(completed.exists(), "the released background writer did not complete"); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn parallel_without_timeout_preserves_ordinary_background_descendants() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + let marker = tmp.path().join("background-completed"); + each(&manifest) + .args(["-p", "alpha", "--jobs", "2", "--"]) + .arg(probe) + .arg("background-parent") + .arg(&marker) + .assert() + .success(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + while !marker.exists() && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert!( + marker.exists(), + "parallel execution without --timeout must not kill an ordinary background descendant" + ); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn parallel_descendant_inheriting_capture_handles_does_not_delay_snapshot() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + let marker = tmp.path().join("stubborn-background-completed"); + let started = std::time::Instant::now(); + let mut command = std::process::Command::new(assert_cmd::cargo::cargo_bin!("cargo-each")); + let _ = command + .arg("each") + .arg("--manifest-path") + .arg(&manifest) + .args(["-p", "alpha", "-p", "beta", "--jobs", "2", "--"]) + .arg(probe) + .arg("stubborn-background-parent") + .arg(&marker); + let status = command.status().expect("run cargo-each with descendant-held capture handles"); + + assert_eq!(status.code(), Some(0)); + assert!( + started.elapsed() < std::time::Duration::from_secs(4), + "capture finalization waited for an inherited handle to close: {:?}", + started.elapsed() + ); + assert!( + !marker.exists(), + "cargo-each waited for the ordinary background descendant instead of finalizing a file snapshot" + ); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(7); + while !marker.exists() && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert!( + marker.exists(), + "untimed execution must preserve the ordinary background descendant" + ); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn timed_invocation_preserves_leader_success_while_background_descendant_remains() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + let marker = tmp.path().join("timed-background-completed"); + let output = each(&manifest) + .args(["-p", "alpha", "--timeout", "500ms", "--"]) + .arg(probe) + .arg("timed-background-parent") + .arg(&marker) + .output() + .expect("run timed invocation with ordinary background descendant"); + assert!( + output.status.success(), + "leader success must win before the deadline; stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); + while !marker.exists() && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert!( + marker.exists(), + "ordinary background descendant must remain alive after leader success" + ); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn sequential_timeout_fail_fast_does_not_run_later_members() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + let later_marker = tmp.path().join("later-invocation"); + let output = each(&manifest) + .args(["-p", "alpha", "-p", "beta", "--timeout", "50ms", "--"]) + .arg(probe) + .args(["timeout-fail-fast", "{name}"]) + .arg(&later_marker) + .output() + .expect("run cargo-each timeout fail-fast"); + assert_eq!(output.status.code(), Some(1), "stderr: {}", String::from_utf8_lossy(&output.stderr)); + assert!(String::from_utf8_lossy(&output.stderr).contains("timed out after 50ms")); + assert!(!later_marker.exists(), "fail-fast must not launch the later member"); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn sequential_timeout_keep_going_runs_later_members() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + let later_marker = tmp.path().join("later-invocation"); + let output = each(&manifest) + .args(["-p", "alpha", "-p", "beta", "--timeout", "1s", "--keep-going", "--"]) + .arg(probe) + .args(["timeout-keep-going", "{name}"]) + .arg(&later_marker) + .output() + .expect("run cargo-each timeout keep-going"); + assert_eq!(output.status.code(), Some(1), "stderr: {}", String::from_utf8_lossy(&output.stderr)); + assert!(String::from_utf8_lossy(&output.stderr).contains("timed out after 1s")); + assert!( + later_marker.exists(), + "--keep-going must launch the member after a timed-out invocation" + ); +} + +#[cfg_attr(miri, ignore = "spawns the cargo-each binary and cargo subprocesses; miri supports neither")] +#[test] +fn timeout_kills_an_ordinary_descendant_in_the_process_group() { + let (tmp, manifest) = fixture(); + let probe = compile_execution_probe(tmp.path()); + let marker = tmp.path().join("grandchild-survived"); + let output = each(&manifest) + .args(["-p", "alpha", "-p", "beta", "--jobs", "2", "--timeout", "50ms", "--"]) + .arg(probe) + .arg("tree-parent") + .arg(&marker) + .output() + .expect("run cargo-each tree timeout"); + assert_eq!(output.status.code(), Some(1), "stderr: {}", String::from_utf8_lossy(&output.stderr)); + assert!(String::from_utf8_lossy(&output.stderr).contains("timed out after 50ms")); + std::thread::sleep(std::time::Duration::from_millis(700)); + assert!( + !marker.exists(), + "a timed-out ordinary descendant must be terminated with its process group" + ); +}