Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/actions/phase/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,10 @@ runs:
BENCH_CACHE_RESTORED_KEY: ${{ steps.actions-cache-restore.outputs.cache-matched-key || steps.dance-restore.outputs.cache-matched-key }}
run: bundle exec ruby bin/bench record "$BENCH_CASE" --lane "$BENCH_LANE" --exit-status "$([ "$BUILD_FAILED" = true ] && echo 1 || echo 0)"

- if: steps.prepare.outputs.ready == 'true' && contains(inputs.setup, 'cache-dance') && inputs.phase != 'warm'
shell: bash
run: bundle exec ruby bin/bench prune-cache-dance "$BENCH_CASE" --lane "$BENCH_LANE"

- if: always() && steps.sampler.outcome == 'success'
shell: bash
env:
Expand Down
16 changes: 15 additions & 1 deletion lib/bench/cli.rb
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ class CLI
bin/bench start <tool>/<case> --lane LANE
bin/bench build <tool>/<case> --lane LANE
bin/bench record <tool>/<case> --lane LANE --exit-status N
bin/bench prune-cache-dance <tool>/<case> --lane LANE
bin/bench probe <tool>/<case> --lane LANE [--runner KEY] [--run-id ID]
bin/bench label <runner>
bin/bench matrix <project|tool/case> [--tool TOOL] [--lane LANES] [--runner KEYS] [--rolling | --sample | --cli TAG]
Expand Down Expand Up @@ -40,6 +41,7 @@ def call
when "start" then resumed { it.start; 0 }
when "build" then resumed(&:build)
when "record" then record
when "prune-cache-dance" then resumed { prune_cache_dance(it) }
when "probe" then probe
when "label" then label
when "matrix" then matrix
Expand Down Expand Up @@ -110,6 +112,18 @@ def record
1
end

def prune_cache_dance(phase_run)
return 0 unless phase_run.kase.setups(phase_run.lane).include?("cache-dance")

root = cache_dance_root
phase_run.prune_cache_dance(root).each { @out.puts "cache-dance: removed #{it.delete_prefix("#{File.expand_path(root)}/")}" }
0
end

def cache_dance_root
File.join(ENV.fetch("RUNNER_TEMP", "/tmp"), "cache-dance")
end

def failed?(record)
record["exit_status"].nonzero? || record["output_ok"] == false
end
Expand Down Expand Up @@ -398,7 +412,7 @@ def export_to_github(kase, lane, phase_run, continues: false)
outputs.merge!("tool" => kase.tool.name, "provider" => lane.provider, "build_dir" => phase_run.build_dir,
"restore_key" => restore_keys.join("\n"), "cache_key" => [base_key, phase_run.step, (ENV["GITHUB_RUN_ID"] if phase_run.phase == "sample")].compact.join("-"),
"artifact_key" => ["#{kase.id.tr("/", "-")}-#{phase_run.scope}", phase_run.step].compact.join("-"),
"cache_paths" => phase_run.cache_paths.join("\n"), "cache_dance_map" => JSON.generate(kase.setups(lane).include?("cache-dance") ? phase_run.cache_dance_map(File.join(ENV.fetch("RUNNER_TEMP", "/tmp"), "cache-dance")) : {}), "setup" => kase.setups(lane).join(" "), "boringcache_version" => @catalog.versions.fetch("boringcache"))
"cache_paths" => phase_run.cache_paths.join("\n"), "cache_dance_map" => JSON.generate(kase.setups(lane).include?("cache-dance") ? phase_run.cache_dance_map(cache_dance_root) : {}), "setup" => kase.setups(lane).join(" "), "boringcache_version" => @catalog.versions.fetch("boringcache"))
File.open(ENV.fetch("GITHUB_ENV"), "a") { |file| phase_run.exported_env.each { |key, value| file.puts "#{key}=#{value}" } }
end
File.open(ENV.fetch("GITHUB_OUTPUT"), "a") { |file| outputs.each { |key, value| file.puts "#{key}<<BENCH_EOF\n#{value}\nBENCH_EOF" } }
Expand Down
21 changes: 21 additions & 0 deletions lib/bench/phase_run.rb
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,27 @@ def cache_dance_map(root)
end.uniq.to_h { [File.join(root, it), { "id" => it, "target" => "/var/cache-target" }] }
end

def prune_cache_dance(root)
keep = cache_dance_map(root).keys.map { File.expand_path(it) }
return [] if keep.empty? || !File.directory?(root)

stale = []
visit = lambda do |dir|
Dir.children(dir).sort.each do |name|
path = File.join(dir, name)
next if keep.include?(path)

if File.directory?(path) && !File.symlink?(path) && keep.any? { it.start_with?("#{path}/") }
visit.(path)
else
stale << path
end
end
end
visit.(File.expand_path(root))
stale.each { FileUtils.rm_rf(it) }
end

def cache_paths
paths = [*lane.paths, *kase.shared].map { File.expand_path(it, workspace.dir) }
paths.empty? ? paths : paths + kase.exclude.map { "!#{File.expand_path(it, workspace.dir)}" }
Expand Down
51 changes: 48 additions & 3 deletions test/bench/euro_office_preparation_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@
require_relative "../../tools/docker/shared/euro-office-cache"

class EuroOfficePreparationTest < Minitest::Test
THIRD_PARTY = {
"common.cmake" => "--only=libxml2-wasm\n",
"Common/3dParty/build_3rdparty.py" => "def build(): pass\n",
"Common/3dParty/libxml2/nc-build.py" => "VERSION = '2.15.0'\n",
}.freeze

def test_plus_preserves_source_and_adds_compiler_and_mount_caches
prepare_core do |dir, source, output, status|
assert status.success?, output
Expand All @@ -26,10 +32,39 @@ def test_plus_turns_on_the_upstream_ccache_switch_of_the_build_script_core
assert_includes dockerfile, " ARG CORE_CCACHE=1\n"
assert_includes dockerfile, "--mount=type=cache,target=/ccache,id=core-ccache"
assert_includes dockerfile, "id=vcpkg-binary-cache"
assert_includes dockerfile, " RUN --mount=type=cache,id=build-cache-#{EuroOfficeCache.third_party_key(File.join(dir, "upstream/core"))},target=/build-cache \\\n"
assert_equal source.lines.size + 2, dockerfile.lines.size
end
end

def test_third_party_key_follows_the_builder_and_its_dependency_list
Dir.mktmpdir("euro-office-core") do |core|
write_third_party(core)
File.write(File.join(core, "CMakeLists.txt"), "project(core)\n")
key = EuroOfficeCache.third_party_key(core)
assert_match(/\A\h{12}\z/, key)
File.write(File.join(core, "CMakeLists.txt"), "project(core)\nadd_subdirectory(x2t)\n")
assert_equal key, EuroOfficeCache.third_party_key(core)

File.write(File.join(core, "common.cmake"), "--only=libxml2-wasm,libjpeg-turbo-wasm\n")
listed = EuroOfficeCache.third_party_key(core)
refute_equal key, listed
File.write(File.join(core, "Common/3dParty/libxml2/nc-build.py"), "VERSION = '2.15.1'\n")
bumped = EuroOfficeCache.third_party_key(core)
refute_equal listed, bumped
write_third_party(core, "Common/3dParty/zlib-ng/nc-build.py" => "VERSION = '2.3.2'\n")
refute_equal bumped, EuroOfficeCache.third_party_key(core)
end
end

def test_core_without_the_third_party_builder_stops_preparation
prepare_core(source: build_script_core_source, third_party: {}) do |dir, _, output, status|
refute status.success?
assert_includes output, "core third-party builder changed"
refute File.exist?(File.join(dir, "overlay/core.Dockerfile"))
end
end

def test_build_script_core_without_its_ccache_switch_stops_preparation
prepare_core(source: build_script_core_source.sub(" ARG CORE_CCACHE\n", "")) do |dir, _, output, status|
refute status.success?
Expand Down Expand Up @@ -98,7 +133,8 @@ def test_combined_cache_keeps_build_state_without_archiving_remote_compiler_resu
assert_includes core, "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
assert_includes core, "target=/build-cache"
refute_includes core, "target=/root/.cache/ccache"
assert_includes wasm, "target=/build-cache-wasm"
key = EuroOfficeCache.third_party_key(File.join(dir, "upstream/core"))
assert_includes wasm, "--mount=type=cache,id=wasm-build-cache-${CACHE_BUST}-#{key},target=/build-cache-wasm \\\n"
assert_includes wasm, "target=/em-cache"
refute_includes wasm, "target=/ccache"
assert_includes wasm, "COPY bench-ccache/ccache /usr/local/bin/ccache"
Expand All @@ -123,7 +159,7 @@ def test_combined_cache_leaves_the_build_script_core_compiler_cache_remote
assert status.success?, output
core = File.read(File.join(dir, "overlay/core.Dockerfile"))
assert_includes core, " ARG CORE_CCACHE=1\n"
assert_includes core, "target=/build-cache"
assert_includes core, "id=build-cache-#{EuroOfficeCache.third_party_key(File.join(dir, "upstream/core"))},target=/build-cache"
refute_includes core, "target=/ccache"
end
end
Expand All @@ -139,6 +175,7 @@ def test_native_control_keeps_the_build_script_core_compiler_cache_mount
def prepare_standalone(resolve: true, plus: false, local_cache: false, core: core_source)
Dir.mktmpdir("euro-office") do |dir|
%w[upstream/core/.docker overlay bin].each { FileUtils.mkdir_p(File.join(dir, it)) }
write_third_party(File.join(dir, "upstream/core"))
File.write(File.join(dir, "upstream/VERSION"), "9.3.6-rc.1\n")
File.write(File.join(dir, ".boringcache.toml"), "[adapters.docker]\ntool-cache = #{plus ? '["ccache:test"]' : '[]'}\n")
File.write(File.join(dir, "upstream/core/.docker/core.bake.Dockerfile"), core)
Expand Down Expand Up @@ -172,9 +209,10 @@ def prepare_standalone(resolve: true, plus: false, local_cache: false, core: cor
end
end

def prepare_core(plus: true, replace: nil, source: core_source)
def prepare_core(plus: true, replace: nil, source: core_source, third_party: THIRD_PARTY)
Dir.mktmpdir("euro-office") do |dir|
FileUtils.mkdir_p(File.join(dir, "upstream/core/.docker"))
write_third_party(File.join(dir, "upstream/core"), third_party)
FileUtils.mkdir_p(File.join(dir, "overlay"))
source = source.sub("cmake --build .", replace) if replace
File.write(File.join(dir, "upstream/core/.docker/core.bake.Dockerfile"), source)
Expand Down Expand Up @@ -218,6 +256,13 @@ def build_script_core_source
DOCKERFILE
end

def write_third_party(core, files = THIRD_PARTY)
files.each do |path, content|
FileUtils.mkdir_p(File.dirname(File.join(core, path)))
File.write(File.join(core, path), content)
end
end

def stage_stub(dir)
helper = File.join(dir, "tools/docker/shared/stage-ccache-binary.sh")
FileUtils.mkdir_p(File.dirname(helper))
Expand Down
62 changes: 62 additions & 0 deletions test/bench/phase_run_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -453,6 +453,68 @@ def test_cache_dance_reads_resolved_bake_targets_and_mount_arguments
assert_equal ["wasm-2", "ccache-#{arch}", "/deps"], run.cache_dance_map("/cache").values.map { it.fetch("id") }
end

def test_cache_dance_prune_drops_restored_mounts_the_dockerfile_no_longer_declares
write "tools/demo/app/.boringcache.toml", <<~TOML
[adapters.demo]
tag = "demo-app"
command = ["docker", "buildx", "build", "--file", "overlay/app.Dockerfile", "upstream"]
TOML
write "tools/demo/app/overlay/app.Dockerfile", <<~'DOCKERFILE'
FROM scratch
RUN --mount=type=cache,id=build-cache-new,target=/build-cache \
--mount=type=cache,target=/go/pkg/mod true
DOCKERFILE
kase = catalog.find_case("demo/app")
run = Bench::PhaseRun.new(kase, kase.tool.lane("boringcache-demo"), phase: "cold", runner: "local", scope: "test",
work_root: File.join(@root, ".work"), results_dir: File.join(@root, "results"))
with_fixture_env { run.prepare }
root = File.join(@root, "cache-dance")
%w[build-cache-new/state build-cache/state go/pkg/mod/cache go/pkg/old/cache go/other].each do |path|
FileUtils.mkdir_p(File.dirname(File.join(root, path)))
File.write(File.join(root, path), "x")
end
files = -> { Dir.glob("**/*", base: root).select { File.file?(File.join(root, it)) }.sort }

assert_equal %w[build-cache go/other go/pkg/old].map { File.join(root, it) }, run.prune_cache_dance(root)
assert_equal %w[build-cache-new/state go/pkg/mod/cache], files.()

File.write(File.join(@root, ".work/demo/app/overlay/app.Dockerfile"), "FROM scratch\nRUN true\n")
assert_equal [], run.prune_cache_dance(root)
assert_equal %w[build-cache-new/state go/pkg/mod/cache], files.()
end

def test_prune_cache_dance_only_touches_cache_dance_lanes
write "tools/demo/app/.boringcache.toml", <<~TOML
workspace = "boringcache/benchmarks"
[adapters.demo]
tag = "demo-app"
command = ["docker", "buildx", "build", "--file", "overlay/app.Dockerfile", "upstream"]
TOML
write "tools/demo/app/overlay/app.Dockerfile", "FROM scratch\nRUN --mount=type=cache,id=deps-2,target=/deps true\n"
write "tools/demo/lanes/remote.toml", %(provider = "remote"\nlevel = "base"\nsecrets = ["REMOTE_TOKEN"]\nsetup = ["cache-dance"]\n[env]\nDEMO_CACHE = "${REMOTE_TOKEN}"\n)
root = File.join(@root, "runner/cache-dance")
stale = File.join(root, "deps-1/state")
prune = lambda do |lane|
FileUtils.mkdir_p([File.dirname(stale), File.join(root, "deps-2")])
File.write(stale, "x")
out = StringIO.new
with_fixture_env do
ENV.update("GITHUB_OUTPUT" => File.join(@root, "github_output"), "GITHUB_ENV" => File.join(@root, "github_env"), "RUNNER_TEMP" => File.join(@root, "runner"))
assert_equal 0, cli("prepare", "demo/app", "--lane", lane, "--phase", "cold", "--run-id", "r60", "--github")
assert_equal 0, cli("prune-cache-dance", "demo/app", "--lane", lane, out:)
ensure
%w[GITHUB_OUTPUT GITHUB_ENV RUNNER_TEMP].each { ENV.delete(it) }
end
out.string
end

assert_equal "", prune.("boringcache-demo")
assert File.exist?(stale)
assert_equal "cache-dance: removed deps-1\n", prune.("remote")
refute File.exist?(File.dirname(stale))
assert File.directory?(File.join(root, "deps-2"))
end

private
def cli(*argv, out: StringIO.new)
Bench::CLI.new([*argv, "--results", File.join(@root, "tmp/results"), "--work", File.join(@root, ".work")], catalog:, out:).call
Expand Down
10 changes: 8 additions & 2 deletions tools/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ TURBO_CACHE = "local:rw,remote:r"
| `env`, `warm.env` | environment for the build, and extra environment for warm |
| `paths` | GitHub Actions cache lanes: tool cache paths to save and restore, relative to the run directory |
| `args`, `warm.args` | arguments appended to the plan command, and the warm replacement (for example `--cache-from`/`--cache-to`) |
| `setup` | workflow setup the lane needs: `buildx` (Buildx builder), `actions-runtime` (Actions cache credentials for type=gha and sccache), `cache-dance` (restore the plan Dockerfile cache mounts from the Actions cache after the timer starts, recording the matched key, and save them after the job), `namespace-cache` (mount the case `shared` paths from the runner profile Namespace cache volume), `ghcr` (log in to GHCR), `depot` (Depot CLI), `vercel` (Vercel Remote Cache token through OIDC), `namespace` (Namespace CLI and remote Buildx builder), `nix` (Nix), `cachix` (Cachix CLI and substituter, no push), `bitrise` (Bitrise Build Cache CLI; the lane activates Bazel, Gradle or Xcode), `nativelink` (local NativeLink server backed by R2) before the timer; `kache` and `mbx` run their actions after the timer starts because they restore cache data |
| `setup` | workflow setup the lane needs: `buildx` (Buildx builder), `actions-runtime` (Actions cache credentials for type=gha and sccache), `cache-dance` (restore the plan Dockerfile cache mounts from the Actions cache after the timer starts, recording the matched key, and save them after the job; restored mount directories that the Dockerfile no longer declares are deleted after the record so they are not saved again), `namespace-cache` (mount the case `shared` paths from the runner profile Namespace cache volume), `ghcr` (log in to GHCR), `depot` (Depot CLI), `vercel` (Vercel Remote Cache token through OIDC), `namespace` (Namespace CLI and remote Buildx builder), `nix` (Nix), `cachix` (Cachix CLI and substituter, no push), `bitrise` (Bitrise Build Cache CLI; the lane activates Bazel, Gradle or Xcode), `nativelink` (local NativeLink server backed by R2) before the timer; `kache` and `mbx` run their actions after the timer starts because they restore cache data |
| `phases` | BoringCache lanes only: run `boringcache <tool> --phase restore`, then the plan command, then `--phase save` (no save on warm), for upstream scripts that run several tool commands, such as Zed's bundle script with Cargo |
| `program`, `replaces` | `program` replaces the leading `replaces` words of the plan command, for example `depot` for `docker buildx` or `mbx` for `cargo` |
| `wrap`, `cold.wrap` | command prefix around the build, for example `cachix watch-exec <cache> --` on cold |
Expand Down Expand Up @@ -153,6 +153,10 @@ Core revisions that declare `CORE_CCACHE` compile through ccache themselves.
For those, the generated Dockerfile copies the pinned ccache binaries and sets
`CORE_CCACHE=1`. It keeps upstream's `core-ccache` mount only for lanes that
archive local compiler caches.
CMake runs upstream's third-party builder only once per build directory, so
the generated Dockerfile names the build-state mount after a hash of
`core/common.cmake` and `core/Common/3dParty`. A change to the builder or its
dependency list starts the plus lanes with an empty build directory.
All lanes retain the dependency downloads and builds defined at each revision.
The history crosses upstream's migration to anonymous S3 dependency downloads.
`euro-office` builds the upstream `standalone` Bake target, including packaging,
Expand All @@ -169,7 +173,9 @@ lanes use the same pinned ccache binaries, enable native C/C++ launchers and
retain the existing WASM launchers. The BoringCache lane stores compiler results
remotely without archiving local ccache directories. Both plus lanes persist
native build state, vcpkg packages, WASM build state, the Emscripten cache and
npm downloads. The native control also archives its two local compiler caches.
npm downloads. The native and WASM build-state mounts are named after the same
third-party builder hash. The native control also archives its two local
compiler caches.
Cache-dance discovers mounts across the resolved Bake targets and expands their
build arguments. Its extraction and archive saving remain part of the reported
elapsed time. The original layer-only and isolated-core records stay separate.
Expand Down
2 changes: 1 addition & 1 deletion tools/docker/euro-office-core/overlay/prepare.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,5 @@

source = File.read("upstream/core/.docker/core.bake.Dockerfile")
require File.join(ENV.fetch("BENCH_ROOT"), "tools/docker/shared/euro-office-cache")
File.write("overlay/core.Dockerfile", EuroOfficeCache.core(source))
File.write("overlay/core.Dockerfile", EuroOfficeCache.core(source, third_party: EuroOfficeCache.third_party_key("upstream/core")))
system("bash", File.join(ENV.fetch("BENCH_ROOT"), "tools/docker/shared/stage-ccache-binary.sh"), "upstream/bench-ccache", exception: true)
Loading
Loading