Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 38 additions & 5 deletions test/bench/nushell_ci_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,36 @@ def test_failed_test_stops_before_doctests_and_save
assert calls.grep(/^boringcache /).all? { it.include?("--skip-save") }
end

def test_doctests_run_after_the_binary_is_kept_and_target_is_pruned
kept = nil
_, status = run_ci { |dir| kept = File.read(File.join(dir, "out/nu")) }
assert status.success?
assert_equal "nu", kept
end

def test_prune_keeps_libraries_and_removes_executables
Dir.mktmpdir("nu-prune") do |dir|
profile = File.join(dir, "ci")
deps = File.join(profile, "deps")
FileUtils.mkdir_p([deps, File.join(profile, "examples"), File.join(profile, "build/serde-1111111111111111"), File.join(profile, ".fingerprint/nu-2222222222222222"), File.join(deps, "nu-3333333333333333.dSYM")])
kept = %w[libnu_protocol-aaaaaaaaaaaaaaaa.rlib libnu_protocol-aaaaaaaaaaaaaaaa.rmeta nu_protocol-aaaaaaaaaaaaaaaa.d
nu_derive_value-bbbbbbbbbbbbbbbb.dll nu_derive_value-bbbbbbbbbbbbbbbb.dll.lib nu_derive_value-bbbbbbbbbbbbbbbb.pdb
libserde_derive-cccccccccccccccc.so serde_derive-cccccccccccccccc.d]
removed = %w[nu_protocol-dddddddddddddddd nu_protocol-dddddddddddddddd.d tests-eeeeeeeeeeeeeeee.exe tests-eeeeeeeeeeeeeeee.pdb]
(kept + removed).each { File.write(File.join(deps, it), "") }
%w[nu nu.exe nu.pdb nu.d .cargo-lock].each { File.write(File.join(profile, it), "") }
_, status = Open3.capture2e("ruby", File.join(Bench::ROOT, "tools/cargo/nushell/overlay/prune.rb"), profile)
assert status.success?
assert_equal kept.sort, Dir.children(deps).sort
assert_equal %w[.cargo-lock .fingerprint build deps], Dir.children(profile).sort
end
end

def test_prune_accepts_a_missing_target
_, status = Open3.capture2e("ruby", File.join(Bench::ROOT, "tools/cargo/nushell/overlay/prune.rb"), File.join(Dir.tmpdir, "missing-nu-target"))
assert status.success?
end

def test_output_check_accepts_native_and_windows_executables
["nu", "nu.exe"].each do |name|
output_check(name:) do |dir, status|
Expand All @@ -53,11 +83,10 @@ def check_script
def output_check(name:, exit_status: 0)
Dir.mktmpdir("nu-check") do |dir|
upstream = File.join(dir, "upstream")
FileUtils.mkdir_p(File.join(upstream, "target/ci"))
FileUtils.mkdir_p([upstream, File.join(dir, "out")])
Bench.git("init", "--quiet", chdir: upstream)
File.write(File.join(upstream, ".git/info/exclude"), "/target/\n")
if name
binary = File.join(upstream, "target/ci", name)
binary = File.join(dir, "out", name)
File.write(binary, "#!/usr/bin/env bash\nexit #{exit_status}\n")
File.chmod(0o755, binary)
end
Expand All @@ -66,8 +95,11 @@ def output_check(name:, exit_status: 0)
end
end

def run_ci(extra_env = {})
def run_ci(extra_env = {}, &inspect)
Dir.mktmpdir("nushell-ci") do |dir|
upstream = File.join(dir, "upstream")
FileUtils.mkdir_p(File.join(upstream, "target/ci"))
File.write(File.join(upstream, "target/ci/nu"), "nu")
File.write(File.join(dir, "cargo"), <<~BASH)
#!/usr/bin/env bash
printf 'cargo %s\n' "$*" >> "$CALLS"
Expand All @@ -84,7 +116,8 @@ def run_ci(extra_env = {})
File.chmod(0o755, File.join(dir, "cargo"), File.join(dir, "boringcache"))
env = { "PATH" => "#{dir}:#{ENV.fetch('PATH')}", "CALLS" => File.join(dir, "calls"),
"BENCH_BORINGCACHE" => nil, "FAIL_COMMAND" => nil, **extra_env }
_, status = Open3.capture2e(env, "bash", File.join(Bench::ROOT, "tools/cargo/nushell/overlay/ci.sh"))
_, status = Open3.capture2e(env, "bash", File.join(Bench::ROOT, "tools/cargo/nushell/overlay/ci.sh"), chdir: upstream)
inspect&.call(dir)
[File.readlines(env.fetch("CALLS"), chomp: true), status]
end
end
Expand Down
19 changes: 16 additions & 3 deletions tools/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -233,8 +233,14 @@ Fresh builds use that starting source for both cold and warm phases.
Every lane runs the root job's Cargo build, tests and doctests, with its `ci`
profile and `.cargo/ci.toml`. Source files are unchanged. Rust is 1.97.1 and
sccache is 0.17.0. Incremental compilation is off in all lanes so sccache can
cache compiler outputs. The output check runs `nu --version` and verifies the
checkout is clean. The lint job, standard-library and Python virtualenv tests,
cache compiler outputs. After the tests, every lane copies `nu` out of
`target` and removes the workspace's test executables, binaries, examples and
their debug files from `target/ci`. Libraries and dependency outputs stay, so
the doctests reuse them and the plus lanes cache them. Upstream stopped caching
workspace crates for the same reason: they are rebuilt on most commits and make
the Windows cache large. The doctests run last, because a BoringCache Cargo
command publishes its cache when it ends. The output check runs the copied
`nu --version` and verifies the checkout is clean. The lint job, standard-library and Python virtualenv tests,
fuzz jobs, WASM job and artifact upload are outside this measurement.

The base lanes compare BoringCache and GitHub Actions compiler caches. The plus
Expand All @@ -252,10 +258,17 @@ platforms. Rolling runs include every configured platform. Series 2 started
from `168db974bad0181a35f4ccd735db0a20cb999d24` with new cache scopes and `nu`
cache tags, and ran formatting, check and Clippy before the build, as the root
job did then. Series 3 follows upstream's split root job from its new starting
source with new cache scopes. Earlier records remain available.
source with new cache scopes. Its Windows plus lanes cached every workspace
executable and debug file: `gha-plus` ran out of disk on every step from 7 to
12 ([run 43537134309](https://github.com/boringcache/benchmarks/actions/runs/43537134309)),
and BoringCache plus failed at step 7 when a debug file passed the linker's
size limit (`LNK1140`, [run 43522482334](https://github.com/boringcache/benchmarks/actions/runs/43522482334)).
Series 4 starts from the same source with new cache scopes and removes those
files before the cache is saved. Earlier records remain available.

Sources: [Nushell root CI](https://github.com/nushell/nushell/blob/396a7588af811612ebfc6de5b090d4f5674d6e51/.github/workflows/ci.yml),
[lint job split](https://github.com/nushell/nushell/pull/19453),
[workspace crates left out of the cache](https://github.com/nushell/nushell/pull/19452),
[runner comparison](https://github.com/nushell/nushell/pull/18810),
[osquery context](https://github.com/osquery/osquery/issues/9104#issuecomment-6626417651),
[sccache GitHub cache](https://github.com/mozilla/sccache/blob/v0.17.0/docs/GHA.md),
Expand Down
2 changes: 1 addition & 1 deletion tools/cargo/nushell/case.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
repo = "nushell/nushell"
branch = "main"
start_sha = "396a7588af811612ebfc6de5b090d4f5674d6e51"
rolling_series = 3
rolling_series = 4
directory = "upstream"
script = ["bash", "../overlay/ci.sh"]
prepare = [
Expand Down
2 changes: 1 addition & 1 deletion tools/cargo/nushell/overlay/check.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
set -euo pipefail

binary=upstream/target/ci/nu
binary=out/nu
if [ -f "$binary.exe" ]; then binary="$binary.exe"; fi
"$binary" --version
test -z "$(git -C upstream status --porcelain=v1 --untracked-files=all)"
7 changes: 7 additions & 0 deletions tools/cargo/nushell/overlay/ci.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
set -euo pipefail

overlay=$(cd "$(dirname "$0")" && pwd)

cargo_step() {
local restore=$1 save=$2
shift 2
Expand All @@ -15,4 +17,9 @@ cargo_step() {

cargo_step restore no-save build --workspace --all-targets --profile ci
cargo_step no-restore no-save test --workspace --all-targets --profile ci
mkdir -p ../out
for binary in target/ci/nu target/ci/nu.exe; do
[ ! -f "$binary" ] || cp "$binary" ../out/
done
ruby "$overlay/prune.rb" target/ci
cargo_step no-restore save test --workspace --doc --profile ci
21 changes: 21 additions & 0 deletions tools/cargo/nushell/overlay/prune.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
require "fileutils"
require "set"

profile = ARGV.fetch(0)
exit unless File.directory?(profile)

Dir.children(profile).each do |name|
path = File.join(profile, name)
FileUtils.rm_rf(path) if name == "examples" || (File.file?(path) && !name.start_with?("."))
end

deps = File.join(profile, "deps")
exit unless File.directory?(deps)

hash = ->(name) { name[/-(\h{16})(?:\.|\z)/, 1] }
names = Dir.children(deps)
libraries = names.grep(/\.(?:rlib|rmeta|so|dylib|dll)\z/).filter_map { hash.(it) }.to_set
names.each do |name|
id = hash.(name)
FileUtils.rm_rf(File.join(deps, name)) if id && !libraries.include?(id)
end
Loading