diff --git a/.github/actions/phase/action.yml b/.github/actions/phase/action.yml index 183fc7bc77..a58418ced6 100644 --- a/.github/actions/phase/action.yml +++ b/.github/actions/phase/action.yml @@ -300,6 +300,10 @@ runs: BENCH_CACHE_RESTORED_KEY: ${{ steps.actions-cache-restore.outputs.cache-matched-key || steps.dance-restore.outputs.cache-matched-key }} run: bundle exec ruby bin/bench record "$BENCH_CASE" --lane "$BENCH_LANE" --exit-status "$([ "$BUILD_FAILED" = true ] && echo 1 || echo 0)" + - if: steps.prepare.outputs.ready == 'true' && contains(inputs.setup, 'cache-dance') && inputs.phase != 'warm' + shell: bash + run: bundle exec ruby bin/bench prune-cache-dance "$BENCH_CASE" --lane "$BENCH_LANE" + - if: always() && steps.sampler.outcome == 'success' shell: bash env: diff --git a/lib/bench/cli.rb b/lib/bench/cli.rb index a902e8c09c..5c109a8b6b 100644 --- a/lib/bench/cli.rb +++ b/lib/bench/cli.rb @@ -10,6 +10,7 @@ class CLI bin/bench start / --lane LANE bin/bench build / --lane LANE bin/bench record / --lane LANE --exit-status N + bin/bench prune-cache-dance / --lane LANE bin/bench probe / --lane LANE [--runner KEY] [--run-id ID] bin/bench label bin/bench matrix [--tool TOOL] [--lane LANES] [--runner KEYS] [--rolling | --sample | --cli TAG] @@ -40,6 +41,7 @@ def call when "start" then resumed { it.start; 0 } when "build" then resumed(&:build) when "record" then record + when "prune-cache-dance" then resumed { prune_cache_dance(it) } when "probe" then probe when "label" then label when "matrix" then matrix @@ -110,6 +112,18 @@ def record 1 end + def prune_cache_dance(phase_run) + return 0 unless phase_run.kase.setups(phase_run.lane).include?("cache-dance") + + root = cache_dance_root + phase_run.prune_cache_dance(root).each { @out.puts "cache-dance: removed #{it.delete_prefix("#{File.expand_path(root)}/")}" } + 0 + end + + def cache_dance_root + File.join(ENV.fetch("RUNNER_TEMP", "/tmp"), "cache-dance") + end + def failed?(record) record["exit_status"].nonzero? || record["output_ok"] == false end @@ -398,7 +412,7 @@ def export_to_github(kase, lane, phase_run, continues: false) outputs.merge!("tool" => kase.tool.name, "provider" => lane.provider, "build_dir" => phase_run.build_dir, "restore_key" => restore_keys.join("\n"), "cache_key" => [base_key, phase_run.step, (ENV["GITHUB_RUN_ID"] if phase_run.phase == "sample")].compact.join("-"), "artifact_key" => ["#{kase.id.tr("/", "-")}-#{phase_run.scope}", phase_run.step].compact.join("-"), - "cache_paths" => phase_run.cache_paths.join("\n"), "cache_dance_map" => JSON.generate(kase.setups(lane).include?("cache-dance") ? phase_run.cache_dance_map(File.join(ENV.fetch("RUNNER_TEMP", "/tmp"), "cache-dance")) : {}), "setup" => kase.setups(lane).join(" "), "boringcache_version" => @catalog.versions.fetch("boringcache")) + "cache_paths" => phase_run.cache_paths.join("\n"), "cache_dance_map" => JSON.generate(kase.setups(lane).include?("cache-dance") ? phase_run.cache_dance_map(cache_dance_root) : {}), "setup" => kase.setups(lane).join(" "), "boringcache_version" => @catalog.versions.fetch("boringcache")) File.open(ENV.fetch("GITHUB_ENV"), "a") { |file| phase_run.exported_env.each { |key, value| file.puts "#{key}=#{value}" } } end File.open(ENV.fetch("GITHUB_OUTPUT"), "a") { |file| outputs.each { |key, value| file.puts "#{key}< it, "target" => "/var/cache-target" }] } end + def prune_cache_dance(root) + keep = cache_dance_map(root).keys.map { File.expand_path(it) } + return [] if keep.empty? || !File.directory?(root) + + stale = [] + visit = lambda do |dir| + Dir.children(dir).sort.each do |name| + path = File.join(dir, name) + next if keep.include?(path) + + if File.directory?(path) && !File.symlink?(path) && keep.any? { it.start_with?("#{path}/") } + visit.(path) + else + stale << path + end + end + end + visit.(File.expand_path(root)) + stale.each { FileUtils.rm_rf(it) } + end + def cache_paths paths = [*lane.paths, *kase.shared].map { File.expand_path(it, workspace.dir) } paths.empty? ? paths : paths + kase.exclude.map { "!#{File.expand_path(it, workspace.dir)}" } diff --git a/test/bench/euro_office_preparation_test.rb b/test/bench/euro_office_preparation_test.rb index daa633d79c..010f02b96f 100644 --- a/test/bench/euro_office_preparation_test.rb +++ b/test/bench/euro_office_preparation_test.rb @@ -3,6 +3,12 @@ require_relative "../../tools/docker/shared/euro-office-cache" class EuroOfficePreparationTest < Minitest::Test + THIRD_PARTY = { + "common.cmake" => "--only=libxml2-wasm\n", + "Common/3dParty/build_3rdparty.py" => "def build(): pass\n", + "Common/3dParty/libxml2/nc-build.py" => "VERSION = '2.15.0'\n", + }.freeze + def test_plus_preserves_source_and_adds_compiler_and_mount_caches prepare_core do |dir, source, output, status| assert status.success?, output @@ -26,10 +32,39 @@ def test_plus_turns_on_the_upstream_ccache_switch_of_the_build_script_core assert_includes dockerfile, " ARG CORE_CCACHE=1\n" assert_includes dockerfile, "--mount=type=cache,target=/ccache,id=core-ccache" assert_includes dockerfile, "id=vcpkg-binary-cache" + assert_includes dockerfile, " RUN --mount=type=cache,id=build-cache-#{EuroOfficeCache.third_party_key(File.join(dir, "upstream/core"))},target=/build-cache \\\n" assert_equal source.lines.size + 2, dockerfile.lines.size end end + def test_third_party_key_follows_the_builder_and_its_dependency_list + Dir.mktmpdir("euro-office-core") do |core| + write_third_party(core) + File.write(File.join(core, "CMakeLists.txt"), "project(core)\n") + key = EuroOfficeCache.third_party_key(core) + assert_match(/\A\h{12}\z/, key) + File.write(File.join(core, "CMakeLists.txt"), "project(core)\nadd_subdirectory(x2t)\n") + assert_equal key, EuroOfficeCache.third_party_key(core) + + File.write(File.join(core, "common.cmake"), "--only=libxml2-wasm,libjpeg-turbo-wasm\n") + listed = EuroOfficeCache.third_party_key(core) + refute_equal key, listed + File.write(File.join(core, "Common/3dParty/libxml2/nc-build.py"), "VERSION = '2.15.1'\n") + bumped = EuroOfficeCache.third_party_key(core) + refute_equal listed, bumped + write_third_party(core, "Common/3dParty/zlib-ng/nc-build.py" => "VERSION = '2.3.2'\n") + refute_equal bumped, EuroOfficeCache.third_party_key(core) + end + end + + def test_core_without_the_third_party_builder_stops_preparation + prepare_core(source: build_script_core_source, third_party: {}) do |dir, _, output, status| + refute status.success? + assert_includes output, "core third-party builder changed" + refute File.exist?(File.join(dir, "overlay/core.Dockerfile")) + end + end + def test_build_script_core_without_its_ccache_switch_stops_preparation prepare_core(source: build_script_core_source.sub(" ARG CORE_CCACHE\n", "")) do |dir, _, output, status| refute status.success? @@ -98,7 +133,8 @@ def test_combined_cache_keeps_build_state_without_archiving_remote_compiler_resu assert_includes core, "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache" assert_includes core, "target=/build-cache" refute_includes core, "target=/root/.cache/ccache" - assert_includes wasm, "target=/build-cache-wasm" + key = EuroOfficeCache.third_party_key(File.join(dir, "upstream/core")) + assert_includes wasm, "--mount=type=cache,id=wasm-build-cache-${CACHE_BUST}-#{key},target=/build-cache-wasm \\\n" assert_includes wasm, "target=/em-cache" refute_includes wasm, "target=/ccache" assert_includes wasm, "COPY bench-ccache/ccache /usr/local/bin/ccache" @@ -123,7 +159,7 @@ def test_combined_cache_leaves_the_build_script_core_compiler_cache_remote assert status.success?, output core = File.read(File.join(dir, "overlay/core.Dockerfile")) assert_includes core, " ARG CORE_CCACHE=1\n" - assert_includes core, "target=/build-cache" + assert_includes core, "id=build-cache-#{EuroOfficeCache.third_party_key(File.join(dir, "upstream/core"))},target=/build-cache" refute_includes core, "target=/ccache" end end @@ -139,6 +175,7 @@ def test_native_control_keeps_the_build_script_core_compiler_cache_mount def prepare_standalone(resolve: true, plus: false, local_cache: false, core: core_source) Dir.mktmpdir("euro-office") do |dir| %w[upstream/core/.docker overlay bin].each { FileUtils.mkdir_p(File.join(dir, it)) } + write_third_party(File.join(dir, "upstream/core")) File.write(File.join(dir, "upstream/VERSION"), "9.3.6-rc.1\n") File.write(File.join(dir, ".boringcache.toml"), "[adapters.docker]\ntool-cache = #{plus ? '["ccache:test"]' : '[]'}\n") File.write(File.join(dir, "upstream/core/.docker/core.bake.Dockerfile"), core) @@ -172,9 +209,10 @@ def prepare_standalone(resolve: true, plus: false, local_cache: false, core: cor end end - def prepare_core(plus: true, replace: nil, source: core_source) + def prepare_core(plus: true, replace: nil, source: core_source, third_party: THIRD_PARTY) Dir.mktmpdir("euro-office") do |dir| FileUtils.mkdir_p(File.join(dir, "upstream/core/.docker")) + write_third_party(File.join(dir, "upstream/core"), third_party) FileUtils.mkdir_p(File.join(dir, "overlay")) source = source.sub("cmake --build .", replace) if replace File.write(File.join(dir, "upstream/core/.docker/core.bake.Dockerfile"), source) @@ -218,6 +256,13 @@ def build_script_core_source DOCKERFILE end + def write_third_party(core, files = THIRD_PARTY) + files.each do |path, content| + FileUtils.mkdir_p(File.dirname(File.join(core, path))) + File.write(File.join(core, path), content) + end + end + def stage_stub(dir) helper = File.join(dir, "tools/docker/shared/stage-ccache-binary.sh") FileUtils.mkdir_p(File.dirname(helper)) diff --git a/test/bench/phase_run_test.rb b/test/bench/phase_run_test.rb index fe2e673f97..b492e8b78d 100644 --- a/test/bench/phase_run_test.rb +++ b/test/bench/phase_run_test.rb @@ -453,6 +453,68 @@ def test_cache_dance_reads_resolved_bake_targets_and_mount_arguments assert_equal ["wasm-2", "ccache-#{arch}", "/deps"], run.cache_dance_map("/cache").values.map { it.fetch("id") } end + def test_cache_dance_prune_drops_restored_mounts_the_dockerfile_no_longer_declares + write "tools/demo/app/.boringcache.toml", <<~TOML + [adapters.demo] + tag = "demo-app" + command = ["docker", "buildx", "build", "--file", "overlay/app.Dockerfile", "upstream"] + TOML + write "tools/demo/app/overlay/app.Dockerfile", <<~'DOCKERFILE' + FROM scratch + RUN --mount=type=cache,id=build-cache-new,target=/build-cache \ + --mount=type=cache,target=/go/pkg/mod true + DOCKERFILE + kase = catalog.find_case("demo/app") + run = Bench::PhaseRun.new(kase, kase.tool.lane("boringcache-demo"), phase: "cold", runner: "local", scope: "test", + work_root: File.join(@root, ".work"), results_dir: File.join(@root, "results")) + with_fixture_env { run.prepare } + root = File.join(@root, "cache-dance") + %w[build-cache-new/state build-cache/state go/pkg/mod/cache go/pkg/old/cache go/other].each do |path| + FileUtils.mkdir_p(File.dirname(File.join(root, path))) + File.write(File.join(root, path), "x") + end + files = -> { Dir.glob("**/*", base: root).select { File.file?(File.join(root, it)) }.sort } + + assert_equal %w[build-cache go/other go/pkg/old].map { File.join(root, it) }, run.prune_cache_dance(root) + assert_equal %w[build-cache-new/state go/pkg/mod/cache], files.() + + File.write(File.join(@root, ".work/demo/app/overlay/app.Dockerfile"), "FROM scratch\nRUN true\n") + assert_equal [], run.prune_cache_dance(root) + assert_equal %w[build-cache-new/state go/pkg/mod/cache], files.() + end + + def test_prune_cache_dance_only_touches_cache_dance_lanes + write "tools/demo/app/.boringcache.toml", <<~TOML + workspace = "boringcache/benchmarks" + [adapters.demo] + tag = "demo-app" + command = ["docker", "buildx", "build", "--file", "overlay/app.Dockerfile", "upstream"] + TOML + write "tools/demo/app/overlay/app.Dockerfile", "FROM scratch\nRUN --mount=type=cache,id=deps-2,target=/deps true\n" + write "tools/demo/lanes/remote.toml", %(provider = "remote"\nlevel = "base"\nsecrets = ["REMOTE_TOKEN"]\nsetup = ["cache-dance"]\n[env]\nDEMO_CACHE = "${REMOTE_TOKEN}"\n) + root = File.join(@root, "runner/cache-dance") + stale = File.join(root, "deps-1/state") + prune = lambda do |lane| + FileUtils.mkdir_p([File.dirname(stale), File.join(root, "deps-2")]) + File.write(stale, "x") + out = StringIO.new + with_fixture_env do + ENV.update("GITHUB_OUTPUT" => File.join(@root, "github_output"), "GITHUB_ENV" => File.join(@root, "github_env"), "RUNNER_TEMP" => File.join(@root, "runner")) + assert_equal 0, cli("prepare", "demo/app", "--lane", lane, "--phase", "cold", "--run-id", "r60", "--github") + assert_equal 0, cli("prune-cache-dance", "demo/app", "--lane", lane, out:) + ensure + %w[GITHUB_OUTPUT GITHUB_ENV RUNNER_TEMP].each { ENV.delete(it) } + end + out.string + end + + assert_equal "", prune.("boringcache-demo") + assert File.exist?(stale) + assert_equal "cache-dance: removed deps-1\n", prune.("remote") + refute File.exist?(File.dirname(stale)) + assert File.directory?(File.join(root, "deps-2")) + end + private def cli(*argv, out: StringIO.new) Bench::CLI.new([*argv, "--results", File.join(@root, "tmp/results"), "--work", File.join(@root, ".work")], catalog:, out:).call diff --git a/tools/README.md b/tools/README.md index 67913131c7..382ebca0ad 100644 --- a/tools/README.md +++ b/tools/README.md @@ -67,7 +67,7 @@ TURBO_CACHE = "local:rw,remote:r" | `env`, `warm.env` | environment for the build, and extra environment for warm | | `paths` | GitHub Actions cache lanes: tool cache paths to save and restore, relative to the run directory | | `args`, `warm.args` | arguments appended to the plan command, and the warm replacement (for example `--cache-from`/`--cache-to`) | -| `setup` | workflow setup the lane needs: `buildx` (Buildx builder), `actions-runtime` (Actions cache credentials for type=gha and sccache), `cache-dance` (restore the plan Dockerfile cache mounts from the Actions cache after the timer starts, recording the matched key, and save them after the job), `namespace-cache` (mount the case `shared` paths from the runner profile Namespace cache volume), `ghcr` (log in to GHCR), `depot` (Depot CLI), `vercel` (Vercel Remote Cache token through OIDC), `namespace` (Namespace CLI and remote Buildx builder), `nix` (Nix), `cachix` (Cachix CLI and substituter, no push), `bitrise` (Bitrise Build Cache CLI; the lane activates Bazel, Gradle or Xcode), `nativelink` (local NativeLink server backed by R2) before the timer; `kache` and `mbx` run their actions after the timer starts because they restore cache data | +| `setup` | workflow setup the lane needs: `buildx` (Buildx builder), `actions-runtime` (Actions cache credentials for type=gha and sccache), `cache-dance` (restore the plan Dockerfile cache mounts from the Actions cache after the timer starts, recording the matched key, and save them after the job; restored mount directories that the Dockerfile no longer declares are deleted after the record so they are not saved again), `namespace-cache` (mount the case `shared` paths from the runner profile Namespace cache volume), `ghcr` (log in to GHCR), `depot` (Depot CLI), `vercel` (Vercel Remote Cache token through OIDC), `namespace` (Namespace CLI and remote Buildx builder), `nix` (Nix), `cachix` (Cachix CLI and substituter, no push), `bitrise` (Bitrise Build Cache CLI; the lane activates Bazel, Gradle or Xcode), `nativelink` (local NativeLink server backed by R2) before the timer; `kache` and `mbx` run their actions after the timer starts because they restore cache data | | `phases` | BoringCache lanes only: run `boringcache --phase restore`, then the plan command, then `--phase save` (no save on warm), for upstream scripts that run several tool commands, such as Zed's bundle script with Cargo | | `program`, `replaces` | `program` replaces the leading `replaces` words of the plan command, for example `depot` for `docker buildx` or `mbx` for `cargo` | | `wrap`, `cold.wrap` | command prefix around the build, for example `cachix watch-exec --` on cold | @@ -153,6 +153,10 @@ Core revisions that declare `CORE_CCACHE` compile through ccache themselves. For those, the generated Dockerfile copies the pinned ccache binaries and sets `CORE_CCACHE=1`. It keeps upstream's `core-ccache` mount only for lanes that archive local compiler caches. +CMake runs upstream's third-party builder only once per build directory, so +the generated Dockerfile names the build-state mount after a hash of +`core/common.cmake` and `core/Common/3dParty`. A change to the builder or its +dependency list starts the plus lanes with an empty build directory. All lanes retain the dependency downloads and builds defined at each revision. The history crosses upstream's migration to anonymous S3 dependency downloads. `euro-office` builds the upstream `standalone` Bake target, including packaging, @@ -169,7 +173,9 @@ lanes use the same pinned ccache binaries, enable native C/C++ launchers and retain the existing WASM launchers. The BoringCache lane stores compiler results remotely without archiving local ccache directories. Both plus lanes persist native build state, vcpkg packages, WASM build state, the Emscripten cache and -npm downloads. The native control also archives its two local compiler caches. +npm downloads. The native and WASM build-state mounts are named after the same +third-party builder hash. The native control also archives its two local +compiler caches. Cache-dance discovers mounts across the resolved Bake targets and expands their build arguments. Its extraction and archive saving remain part of the reported elapsed time. The original layer-only and isolated-core records stay separate. diff --git a/tools/docker/euro-office-core/overlay/prepare.rb b/tools/docker/euro-office-core/overlay/prepare.rb index fd0ece1a66..9a089b9732 100644 --- a/tools/docker/euro-office-core/overlay/prepare.rb +++ b/tools/docker/euro-office-core/overlay/prepare.rb @@ -5,5 +5,5 @@ source = File.read("upstream/core/.docker/core.bake.Dockerfile") require File.join(ENV.fetch("BENCH_ROOT"), "tools/docker/shared/euro-office-cache") -File.write("overlay/core.Dockerfile", EuroOfficeCache.core(source)) +File.write("overlay/core.Dockerfile", EuroOfficeCache.core(source, third_party: EuroOfficeCache.third_party_key("upstream/core"))) system("bash", File.join(ENV.fetch("BENCH_ROOT"), "tools/docker/shared/stage-ccache-binary.sh"), "upstream/bench-ccache", exception: true) diff --git a/tools/docker/shared/euro-office-cache.rb b/tools/docker/shared/euro-office-cache.rb index a1a0bc80f5..0141a3871d 100644 --- a/tools/docker/shared/euro-office-cache.rb +++ b/tools/docker/shared/euro-office-cache.rb @@ -1,8 +1,22 @@ +require "digest" +require "find" + module EuroOfficeCache CCACHE_BINARIES = "COPY bench-ccache/ccache /usr/local/bin/ccache\nCOPY bench-ccache/ccache-storage-http /usr/local/bin/ccache-storage-http\n".freeze + THIRD_PARTY_INPUTS = ["common.cmake", "Common/3dParty"].freeze + + def self.third_party_key(core) + builder, recipes = THIRD_PARTY_INPUTS.map { File.join(core, it) } + abort "core third-party builder changed: expected #{THIRD_PARTY_INPUTS.join(" and ")}" unless File.file?(builder) && File.directory?(recipes) + + files = [builder, *Find.find(recipes).select { File.file?(it) }].map { it.delete_prefix("#{core}/") }.sort + files.each_with_object(Digest::SHA256.new) do |path, digest| + digest << path << "\0" << Digest::SHA256.file(File.join(core, path)).hexdigest << "\0" + end.hexdigest[0, 12] + end - def self.core(source, local_cache: true) - return build_script_core(source, local_cache:) if source.include?("ARG CORE_CCACHE\n") + def self.core(source, third_party:, local_cache: true) + return build_script_core(source, third_party:, local_cache:) if source.include?("ARG CORE_CCACHE\n") replacements = { "FROM core-base AS core\n" => "FROM core-base AS core\n#{CCACHE_BINARIES}ENV CCACHE_DIR=/root/.cache/ccache\n", @@ -15,19 +29,21 @@ def self.core(source, local_cache: true) replace(source, replacements, "core") end - def self.build_script_core(source, local_cache:) + def self.build_script_core(source, third_party:, local_cache:) replacements = { "FROM core-base AS core\n" => "FROM core-base AS core\n#{CCACHE_BINARIES}", "ARG CORE_CCACHE\n" => "ARG CORE_CCACHE=1\n", + "RUN --mount=type=cache,target=/build-cache \\\n" => "RUN --mount=type=cache,id=build-cache-#{third_party},target=/build-cache \\\n", } replacements[" --mount=type=cache,target=/ccache,id=core-ccache \\\n"] = "" unless local_cache replace(source, replacements, "core") end - def self.wasm(source, local_cache: true) + def self.wasm(source, third_party:, local_cache: true) replacements = { " COPY core /core\n" => " COPY bench-ccache/ccache /usr/local/bin/ccache\n COPY bench-ccache/ccache-storage-http /usr/local/bin/ccache-storage-http\n ENV CCACHE_DIR=/ccache\n COPY core /core\n", " cmake --build . -- -j$(nproc)\n" => " ccache --zero-stats\n cmake --build . -- -j$(nproc)\n", + "id=wasm-build-cache-${CACHE_BUST},target=/build-cache-wasm" => "id=wasm-build-cache-${CACHE_BUST}-#{third_party},target=/build-cache-wasm", } replacements[" --mount=type=cache,id=wasm-ccache,target=/ccache \\\n"] = "" unless local_cache replace(source, replacements, "core-wasm") diff --git a/tools/docker/shared/prepare-euro-office.rb b/tools/docker/shared/prepare-euro-office.rb index 71d9bac2a7..0aea0eb033 100644 --- a/tools/docker/shared/prepare-euro-office.rb +++ b/tools/docker/shared/prepare-euro-office.rb @@ -40,10 +40,11 @@ if Array(plan["tool-cache"]).any? require_relative "euro-office-cache" local_cache = ENV["EURO_OFFICE_LOCAL_CCACHE"] == "1" + third_party = EuroOfficeCache.third_party_key("upstream/core") { "core" => :core, "core-wasm" => :wasm }.each do |name, method| target = definition.fetch("target").fetch(name) path = File.join(target.fetch("context"), target.fetch("dockerfile")) - File.write("overlay/#{name}.Dockerfile", EuroOfficeCache.public_send(method, File.read(path), local_cache:)) + File.write("overlay/#{name}.Dockerfile", EuroOfficeCache.public_send(method, File.read(path), third_party:, local_cache:)) target["dockerfile"] = "../overlay/#{name}.Dockerfile" end system("bash", File.join(ENV.fetch("BENCH_ROOT"), "tools/docker/shared/stage-ccache-binary.sh"), "upstream/bench-ccache", exception: true)